]> git.karo-electronics.de Git - karo-tx-linux.git/blob - drivers/gpu/drm/i915/intel_lrc.c
Merge tag 'topic/dp-quirks-2017-05-31' of git://anongit.freedesktop.org/git/drm-intel...
[karo-tx-linux.git] / drivers / gpu / drm / i915 / intel_lrc.c
1 /*
2  * Copyright © 2014 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Ben Widawsky <ben@bwidawsk.net>
25  *    Michel Thierry <michel.thierry@intel.com>
26  *    Thomas Daniel <thomas.daniel@intel.com>
27  *    Oscar Mateo <oscar.mateo@intel.com>
28  *
29  */
30
31 /**
32  * DOC: Logical Rings, Logical Ring Contexts and Execlists
33  *
34  * Motivation:
35  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
36  * These expanded contexts enable a number of new abilities, especially
37  * "Execlists" (also implemented in this file).
38  *
39  * One of the main differences with the legacy HW contexts is that logical
40  * ring contexts incorporate many more things to the context's state, like
41  * PDPs or ringbuffer control registers:
42  *
43  * The reason why PDPs are included in the context is straightforward: as
44  * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
45  * contained there mean you don't need to do a ppgtt->switch_mm yourself,
46  * instead, the GPU will do it for you on the context switch.
47  *
48  * But, what about the ringbuffer control registers (head, tail, etc..)?
49  * shouldn't we just need a set of those per engine command streamer? This is
50  * where the name "Logical Rings" starts to make sense: by virtualizing the
51  * rings, the engine cs shifts to a new "ring buffer" with every context
52  * switch. When you want to submit a workload to the GPU you: A) choose your
53  * context, B) find its appropriate virtualized ring, C) write commands to it
54  * and then, finally, D) tell the GPU to switch to that context.
55  *
56  * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
57  * to a contexts is via a context execution list, ergo "Execlists".
58  *
59  * LRC implementation:
60  * Regarding the creation of contexts, we have:
61  *
62  * - One global default context.
63  * - One local default context for each opened fd.
64  * - One local extra context for each context create ioctl call.
65  *
66  * Now that ringbuffers belong per-context (and not per-engine, like before)
67  * and that contexts are uniquely tied to a given engine (and not reusable,
68  * like before) we need:
69  *
70  * - One ringbuffer per-engine inside each context.
71  * - One backing object per-engine inside each context.
72  *
73  * The global default context starts its life with these new objects fully
74  * allocated and populated. The local default context for each opened fd is
75  * more complex, because we don't know at creation time which engine is going
76  * to use them. To handle this, we have implemented a deferred creation of LR
77  * contexts:
78  *
79  * The local context starts its life as a hollow or blank holder, that only
80  * gets populated for a given engine once we receive an execbuffer. If later
81  * on we receive another execbuffer ioctl for the same context but a different
82  * engine, we allocate/populate a new ringbuffer and context backing object and
83  * so on.
84  *
85  * Finally, regarding local contexts created using the ioctl call: as they are
86  * only allowed with the render ring, we can allocate & populate them right
87  * away (no need to defer anything, at least for now).
88  *
89  * Execlists implementation:
90  * Execlists are the new method by which, on gen8+ hardware, workloads are
91  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
92  * This method works as follows:
93  *
94  * When a request is committed, its commands (the BB start and any leading or
95  * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
96  * for the appropriate context. The tail pointer in the hardware context is not
97  * updated at this time, but instead, kept by the driver in the ringbuffer
98  * structure. A structure representing this request is added to a request queue
99  * for the appropriate engine: this structure contains a copy of the context's
100  * tail after the request was written to the ring buffer and a pointer to the
101  * context itself.
102  *
103  * If the engine's request queue was empty before the request was added, the
104  * queue is processed immediately. Otherwise the queue will be processed during
105  * a context switch interrupt. In any case, elements on the queue will get sent
106  * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
107  * globally unique 20-bits submission ID.
108  *
109  * When execution of a request completes, the GPU updates the context status
110  * buffer with a context complete event and generates a context switch interrupt.
111  * During the interrupt handling, the driver examines the events in the buffer:
112  * for each context complete event, if the announced ID matches that on the head
113  * of the request queue, then that request is retired and removed from the queue.
114  *
115  * After processing, if any requests were retired and the queue is not empty
116  * then a new execution list can be submitted. The two requests at the front of
117  * the queue are next to be submitted but since a context may not occur twice in
118  * an execution list, if subsequent requests have the same ID as the first then
119  * the two requests must be combined. This is done simply by discarding requests
120  * at the head of the queue until either only one requests is left (in which case
121  * we use a NULL second context) or the first two requests have unique IDs.
122  *
123  * By always executing the first two requests in the queue the driver ensures
124  * that the GPU is kept as busy as possible. In the case where a single context
125  * completes but a second context is still executing, the request for this second
126  * context will be at the head of the queue when we remove the first one. This
127  * request will then be resubmitted along with a new request for a different context,
128  * which will cause the hardware to continue executing the second request and queue
129  * the new request (the GPU detects the condition of a context getting preempted
130  * with the same context and optimizes the context switch flow by not doing
131  * preemption, but just sampling the new tail pointer).
132  *
133  */
134 #include <linux/interrupt.h>
135
136 #include <drm/drmP.h>
137 #include <drm/i915_drm.h>
138 #include "i915_drv.h"
139 #include "intel_mocs.h"
140
141 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
142 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
143 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
144
145 #define RING_EXECLIST_QFULL             (1 << 0x2)
146 #define RING_EXECLIST1_VALID            (1 << 0x3)
147 #define RING_EXECLIST0_VALID            (1 << 0x4)
148 #define RING_EXECLIST_ACTIVE_STATUS     (3 << 0xE)
149 #define RING_EXECLIST1_ACTIVE           (1 << 0x11)
150 #define RING_EXECLIST0_ACTIVE           (1 << 0x12)
151
152 #define GEN8_CTX_STATUS_IDLE_ACTIVE     (1 << 0)
153 #define GEN8_CTX_STATUS_PREEMPTED       (1 << 1)
154 #define GEN8_CTX_STATUS_ELEMENT_SWITCH  (1 << 2)
155 #define GEN8_CTX_STATUS_ACTIVE_IDLE     (1 << 3)
156 #define GEN8_CTX_STATUS_COMPLETE        (1 << 4)
157 #define GEN8_CTX_STATUS_LITE_RESTORE    (1 << 15)
158
159 #define GEN8_CTX_STATUS_COMPLETED_MASK \
160          (GEN8_CTX_STATUS_ACTIVE_IDLE | \
161           GEN8_CTX_STATUS_PREEMPTED | \
162           GEN8_CTX_STATUS_ELEMENT_SWITCH)
163
164 #define CTX_LRI_HEADER_0                0x01
165 #define CTX_CONTEXT_CONTROL             0x02
166 #define CTX_RING_HEAD                   0x04
167 #define CTX_RING_TAIL                   0x06
168 #define CTX_RING_BUFFER_START           0x08
169 #define CTX_RING_BUFFER_CONTROL         0x0a
170 #define CTX_BB_HEAD_U                   0x0c
171 #define CTX_BB_HEAD_L                   0x0e
172 #define CTX_BB_STATE                    0x10
173 #define CTX_SECOND_BB_HEAD_U            0x12
174 #define CTX_SECOND_BB_HEAD_L            0x14
175 #define CTX_SECOND_BB_STATE             0x16
176 #define CTX_BB_PER_CTX_PTR              0x18
177 #define CTX_RCS_INDIRECT_CTX            0x1a
178 #define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
179 #define CTX_LRI_HEADER_1                0x21
180 #define CTX_CTX_TIMESTAMP               0x22
181 #define CTX_PDP3_UDW                    0x24
182 #define CTX_PDP3_LDW                    0x26
183 #define CTX_PDP2_UDW                    0x28
184 #define CTX_PDP2_LDW                    0x2a
185 #define CTX_PDP1_UDW                    0x2c
186 #define CTX_PDP1_LDW                    0x2e
187 #define CTX_PDP0_UDW                    0x30
188 #define CTX_PDP0_LDW                    0x32
189 #define CTX_LRI_HEADER_2                0x41
190 #define CTX_R_PWR_CLK_STATE             0x42
191 #define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
192
193 #define CTX_REG(reg_state, pos, reg, val) do { \
194         (reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \
195         (reg_state)[(pos)+1] = (val); \
196 } while (0)
197
198 #define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do {                \
199         const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n)); \
200         reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
201         reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
202 } while (0)
203
204 #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \
205         reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \
206         reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \
207 } while (0)
208
209 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT        0x17
210 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT        0x26
211
212 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
213 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
214
215 #define WA_TAIL_DWORDS 2
216
217 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
218                                             struct intel_engine_cs *engine);
219 static void execlists_init_reg_state(u32 *reg_state,
220                                      struct i915_gem_context *ctx,
221                                      struct intel_engine_cs *engine,
222                                      struct intel_ring *ring);
223
224 /**
225  * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
226  * @dev_priv: i915 device private
227  * @enable_execlists: value of i915.enable_execlists module parameter.
228  *
229  * Only certain platforms support Execlists (the prerequisites being
230  * support for Logical Ring Contexts and Aliasing PPGTT or better).
231  *
232  * Return: 1 if Execlists is supported and has to be enabled.
233  */
234 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists)
235 {
236         /* On platforms with execlist available, vGPU will only
237          * support execlist mode, no ring buffer mode.
238          */
239         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv))
240                 return 1;
241
242         if (INTEL_GEN(dev_priv) >= 9)
243                 return 1;
244
245         if (enable_execlists == 0)
246                 return 0;
247
248         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) &&
249             USES_PPGTT(dev_priv) &&
250             i915.use_mmio_flip >= 0)
251                 return 1;
252
253         return 0;
254 }
255
256 /**
257  * intel_lr_context_descriptor_update() - calculate & cache the descriptor
258  *                                        descriptor for a pinned context
259  * @ctx: Context to work on
260  * @engine: Engine the descriptor will be used with
261  *
262  * The context descriptor encodes various attributes of a context,
263  * including its GTT address and some flags. Because it's fairly
264  * expensive to calculate, we'll just do it once and cache the result,
265  * which remains valid until the context is unpinned.
266  *
267  * This is what a descriptor looks like, from LSB to MSB::
268  *
269  *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx->desc_template)
270  *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
271  *      bits 32-52:    ctx ID, a globally unique tag
272  *      bits 53-54:    mbz, reserved for use by hardware
273  *      bits 55-63:    group ID, currently unused and set to 0
274  */
275 static void
276 intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
277                                    struct intel_engine_cs *engine)
278 {
279         struct intel_context *ce = &ctx->engine[engine->id];
280         u64 desc;
281
282         BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH));
283
284         desc = ctx->desc_template;                              /* bits  0-11 */
285         desc |= i915_ggtt_offset(ce->state) + LRC_PPHWSP_PN * PAGE_SIZE;
286                                                                 /* bits 12-31 */
287         desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;           /* bits 32-52 */
288
289         ce->lrc_desc = desc;
290 }
291
292 uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
293                                      struct intel_engine_cs *engine)
294 {
295         return ctx->engine[engine->id].lrc_desc;
296 }
297
298 static inline void
299 execlists_context_status_change(struct drm_i915_gem_request *rq,
300                                 unsigned long status)
301 {
302         /*
303          * Only used when GVT-g is enabled now. When GVT-g is disabled,
304          * The compiler should eliminate this function as dead-code.
305          */
306         if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
307                 return;
308
309         atomic_notifier_call_chain(&rq->engine->context_status_notifier,
310                                    status, rq);
311 }
312
313 static void
314 execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
315 {
316         ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
317         ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
318         ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
319         ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
320 }
321
322 static u64 execlists_update_context(struct drm_i915_gem_request *rq)
323 {
324         struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
325         struct i915_hw_ppgtt *ppgtt =
326                 rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
327         u32 *reg_state = ce->lrc_reg_state;
328
329         assert_ring_tail_valid(rq->ring, rq->tail);
330         reg_state[CTX_RING_TAIL+1] = rq->tail;
331
332         /* True 32b PPGTT with dynamic page allocation: update PDP
333          * registers and point the unallocated PDPs to scratch page.
334          * PML4 is allocated during ppgtt init, so this is not needed
335          * in 48-bit mode.
336          */
337         if (ppgtt && !i915_vm_is_48bit(&ppgtt->base))
338                 execlists_update_context_pdps(ppgtt, reg_state);
339
340         return ce->lrc_desc;
341 }
342
343 static void execlists_submit_ports(struct intel_engine_cs *engine)
344 {
345         struct drm_i915_private *dev_priv = engine->i915;
346         struct execlist_port *port = engine->execlist_port;
347         u32 __iomem *elsp =
348                 dev_priv->regs + i915_mmio_reg_offset(RING_ELSP(engine));
349         u64 desc[2];
350
351         GEM_BUG_ON(port[0].count > 1);
352         if (!port[0].count)
353                 execlists_context_status_change(port[0].request,
354                                                 INTEL_CONTEXT_SCHEDULE_IN);
355         desc[0] = execlists_update_context(port[0].request);
356         GEM_DEBUG_EXEC(port[0].context_id = upper_32_bits(desc[0]));
357         port[0].count++;
358
359         if (port[1].request) {
360                 GEM_BUG_ON(port[1].count);
361                 execlists_context_status_change(port[1].request,
362                                                 INTEL_CONTEXT_SCHEDULE_IN);
363                 desc[1] = execlists_update_context(port[1].request);
364                 GEM_DEBUG_EXEC(port[1].context_id = upper_32_bits(desc[1]));
365                 port[1].count = 1;
366         } else {
367                 desc[1] = 0;
368         }
369         GEM_BUG_ON(desc[0] == desc[1]);
370
371         /* You must always write both descriptors in the order below. */
372         writel(upper_32_bits(desc[1]), elsp);
373         writel(lower_32_bits(desc[1]), elsp);
374
375         writel(upper_32_bits(desc[0]), elsp);
376         /* The context is automatically loaded after the following */
377         writel(lower_32_bits(desc[0]), elsp);
378 }
379
380 static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
381 {
382         return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
383                 i915_gem_context_force_single_submission(ctx));
384 }
385
386 static bool can_merge_ctx(const struct i915_gem_context *prev,
387                           const struct i915_gem_context *next)
388 {
389         if (prev != next)
390                 return false;
391
392         if (ctx_single_port_submission(prev))
393                 return false;
394
395         return true;
396 }
397
398 static void execlists_dequeue(struct intel_engine_cs *engine)
399 {
400         struct drm_i915_gem_request *last;
401         struct execlist_port *port = engine->execlist_port;
402         struct rb_node *rb;
403         bool submit = false;
404
405         last = port->request;
406         if (last)
407                 /* WaIdleLiteRestore:bdw,skl
408                  * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL
409                  * as we resubmit the request. See gen8_emit_breadcrumb()
410                  * for where we prepare the padding after the end of the
411                  * request.
412                  */
413                 last->tail = last->wa_tail;
414
415         GEM_BUG_ON(port[1].request);
416
417         /* Hardware submission is through 2 ports. Conceptually each port
418          * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
419          * static for a context, and unique to each, so we only execute
420          * requests belonging to a single context from each ring. RING_HEAD
421          * is maintained by the CS in the context image, it marks the place
422          * where it got up to last time, and through RING_TAIL we tell the CS
423          * where we want to execute up to this time.
424          *
425          * In this list the requests are in order of execution. Consecutive
426          * requests from the same context are adjacent in the ringbuffer. We
427          * can combine these requests into a single RING_TAIL update:
428          *
429          *              RING_HEAD...req1...req2
430          *                                    ^- RING_TAIL
431          * since to execute req2 the CS must first execute req1.
432          *
433          * Our goal then is to point each port to the end of a consecutive
434          * sequence of requests as being the most optimal (fewest wake ups
435          * and context switches) submission.
436          */
437
438         spin_lock_irq(&engine->timeline->lock);
439         rb = engine->execlist_first;
440         while (rb) {
441                 struct drm_i915_gem_request *cursor =
442                         rb_entry(rb, typeof(*cursor), priotree.node);
443
444                 /* Can we combine this request with the current port? It has to
445                  * be the same context/ringbuffer and not have any exceptions
446                  * (e.g. GVT saying never to combine contexts).
447                  *
448                  * If we can combine the requests, we can execute both by
449                  * updating the RING_TAIL to point to the end of the second
450                  * request, and so we never need to tell the hardware about
451                  * the first.
452                  */
453                 if (last && !can_merge_ctx(cursor->ctx, last->ctx)) {
454                         /* If we are on the second port and cannot combine
455                          * this request with the last, then we are done.
456                          */
457                         if (port != engine->execlist_port)
458                                 break;
459
460                         /* If GVT overrides us we only ever submit port[0],
461                          * leaving port[1] empty. Note that we also have
462                          * to be careful that we don't queue the same
463                          * context (even though a different request) to
464                          * the second port.
465                          */
466                         if (ctx_single_port_submission(last->ctx) ||
467                             ctx_single_port_submission(cursor->ctx))
468                                 break;
469
470                         GEM_BUG_ON(last->ctx == cursor->ctx);
471
472                         i915_gem_request_assign(&port->request, last);
473                         port++;
474                 }
475
476                 rb = rb_next(rb);
477                 rb_erase(&cursor->priotree.node, &engine->execlist_queue);
478                 RB_CLEAR_NODE(&cursor->priotree.node);
479                 cursor->priotree.priority = INT_MAX;
480
481                 __i915_gem_request_submit(cursor);
482                 trace_i915_gem_request_in(cursor, port - engine->execlist_port);
483                 last = cursor;
484                 submit = true;
485         }
486         if (submit) {
487                 i915_gem_request_assign(&port->request, last);
488                 engine->execlist_first = rb;
489         }
490         spin_unlock_irq(&engine->timeline->lock);
491
492         if (submit)
493                 execlists_submit_ports(engine);
494 }
495
496 static bool execlists_elsp_idle(struct intel_engine_cs *engine)
497 {
498         return !engine->execlist_port[0].request;
499 }
500
501 static bool execlists_elsp_ready(const struct intel_engine_cs *engine)
502 {
503         const struct execlist_port *port = engine->execlist_port;
504
505         return port[0].count + port[1].count < 2;
506 }
507
508 /*
509  * Check the unread Context Status Buffers and manage the submission of new
510  * contexts to the ELSP accordingly.
511  */
512 static void intel_lrc_irq_handler(unsigned long data)
513 {
514         struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
515         struct execlist_port *port = engine->execlist_port;
516         struct drm_i915_private *dev_priv = engine->i915;
517
518         intel_uncore_forcewake_get(dev_priv, engine->fw_domains);
519
520         /* Prefer doing test_and_clear_bit() as a two stage operation to avoid
521          * imposing the cost of a locked atomic transaction when submitting a
522          * new request (outside of the context-switch interrupt).
523          */
524         while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) {
525                 u32 __iomem *csb_mmio =
526                         dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
527                 u32 __iomem *buf =
528                         dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0));
529                 unsigned int head, tail;
530
531                 /* The write will be ordered by the uncached read (itself
532                  * a memory barrier), so we do not need another in the form
533                  * of a locked instruction. The race between the interrupt
534                  * handler and the split test/clear is harmless as we order
535                  * our clear before the CSB read. If the interrupt arrived
536                  * first between the test and the clear, we read the updated
537                  * CSB and clear the bit. If the interrupt arrives as we read
538                  * the CSB or later (i.e. after we had cleared the bit) the bit
539                  * is set and we do a new loop.
540                  */
541                 __clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
542                 head = readl(csb_mmio);
543                 tail = GEN8_CSB_WRITE_PTR(head);
544                 head = GEN8_CSB_READ_PTR(head);
545                 while (head != tail) {
546                         unsigned int status;
547
548                         if (++head == GEN8_CSB_ENTRIES)
549                                 head = 0;
550
551                         /* We are flying near dragons again.
552                          *
553                          * We hold a reference to the request in execlist_port[]
554                          * but no more than that. We are operating in softirq
555                          * context and so cannot hold any mutex or sleep. That
556                          * prevents us stopping the requests we are processing
557                          * in port[] from being retired simultaneously (the
558                          * breadcrumb will be complete before we see the
559                          * context-switch). As we only hold the reference to the
560                          * request, any pointer chasing underneath the request
561                          * is subject to a potential use-after-free. Thus we
562                          * store all of the bookkeeping within port[] as
563                          * required, and avoid using unguarded pointers beneath
564                          * request itself. The same applies to the atomic
565                          * status notifier.
566                          */
567
568                         status = readl(buf + 2 * head);
569                         if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
570                                 continue;
571
572                         /* Check the context/desc id for this event matches */
573                         GEM_DEBUG_BUG_ON(readl(buf + 2 * head + 1) !=
574                                          port[0].context_id);
575
576                         GEM_BUG_ON(port[0].count == 0);
577                         if (--port[0].count == 0) {
578                                 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
579                                 GEM_BUG_ON(!i915_gem_request_completed(port[0].request));
580                                 execlists_context_status_change(port[0].request,
581                                                                 INTEL_CONTEXT_SCHEDULE_OUT);
582
583                                 trace_i915_gem_request_out(port[0].request);
584                                 i915_gem_request_put(port[0].request);
585                                 port[0] = port[1];
586                                 memset(&port[1], 0, sizeof(port[1]));
587                         }
588
589                         GEM_BUG_ON(port[0].count == 0 &&
590                                    !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
591                 }
592
593                 writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
594                        csb_mmio);
595         }
596
597         if (execlists_elsp_ready(engine))
598                 execlists_dequeue(engine);
599
600         intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
601 }
602
603 static bool insert_request(struct i915_priotree *pt, struct rb_root *root)
604 {
605         struct rb_node **p, *rb;
606         bool first = true;
607
608         /* most positive priority is scheduled first, equal priorities fifo */
609         rb = NULL;
610         p = &root->rb_node;
611         while (*p) {
612                 struct i915_priotree *pos;
613
614                 rb = *p;
615                 pos = rb_entry(rb, typeof(*pos), node);
616                 if (pt->priority > pos->priority) {
617                         p = &rb->rb_left;
618                 } else {
619                         p = &rb->rb_right;
620                         first = false;
621                 }
622         }
623         rb_link_node(&pt->node, rb, p);
624         rb_insert_color(&pt->node, root);
625
626         return first;
627 }
628
629 static void execlists_submit_request(struct drm_i915_gem_request *request)
630 {
631         struct intel_engine_cs *engine = request->engine;
632         unsigned long flags;
633
634         /* Will be called from irq-context when using foreign fences. */
635         spin_lock_irqsave(&engine->timeline->lock, flags);
636
637         if (insert_request(&request->priotree, &engine->execlist_queue)) {
638                 engine->execlist_first = &request->priotree.node;
639                 if (execlists_elsp_ready(engine))
640                         tasklet_hi_schedule(&engine->irq_tasklet);
641         }
642
643         spin_unlock_irqrestore(&engine->timeline->lock, flags);
644 }
645
646 static struct intel_engine_cs *
647 pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
648 {
649         struct intel_engine_cs *engine =
650                 container_of(pt, struct drm_i915_gem_request, priotree)->engine;
651
652         GEM_BUG_ON(!locked);
653
654         if (engine != locked) {
655                 spin_unlock(&locked->timeline->lock);
656                 spin_lock(&engine->timeline->lock);
657         }
658
659         return engine;
660 }
661
662 static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
663 {
664         struct intel_engine_cs *engine;
665         struct i915_dependency *dep, *p;
666         struct i915_dependency stack;
667         LIST_HEAD(dfs);
668
669         if (prio <= READ_ONCE(request->priotree.priority))
670                 return;
671
672         /* Need BKL in order to use the temporary link inside i915_dependency */
673         lockdep_assert_held(&request->i915->drm.struct_mutex);
674
675         stack.signaler = &request->priotree;
676         list_add(&stack.dfs_link, &dfs);
677
678         /* Recursively bump all dependent priorities to match the new request.
679          *
680          * A naive approach would be to use recursion:
681          * static void update_priorities(struct i915_priotree *pt, prio) {
682          *      list_for_each_entry(dep, &pt->signalers_list, signal_link)
683          *              update_priorities(dep->signal, prio)
684          *      insert_request(pt);
685          * }
686          * but that may have unlimited recursion depth and so runs a very
687          * real risk of overunning the kernel stack. Instead, we build
688          * a flat list of all dependencies starting with the current request.
689          * As we walk the list of dependencies, we add all of its dependencies
690          * to the end of the list (this may include an already visited
691          * request) and continue to walk onwards onto the new dependencies. The
692          * end result is a topological list of requests in reverse order, the
693          * last element in the list is the request we must execute first.
694          */
695         list_for_each_entry_safe(dep, p, &dfs, dfs_link) {
696                 struct i915_priotree *pt = dep->signaler;
697
698                 /* Within an engine, there can be no cycle, but we may
699                  * refer to the same dependency chain multiple times
700                  * (redundant dependencies are not eliminated) and across
701                  * engines.
702                  */
703                 list_for_each_entry(p, &pt->signalers_list, signal_link) {
704                         GEM_BUG_ON(p->signaler->priority < pt->priority);
705                         if (prio > READ_ONCE(p->signaler->priority))
706                                 list_move_tail(&p->dfs_link, &dfs);
707                 }
708
709                 list_safe_reset_next(dep, p, dfs_link);
710         }
711
712         engine = request->engine;
713         spin_lock_irq(&engine->timeline->lock);
714
715         /* Fifo and depth-first replacement ensure our deps execute before us */
716         list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
717                 struct i915_priotree *pt = dep->signaler;
718
719                 INIT_LIST_HEAD(&dep->dfs_link);
720
721                 engine = pt_lock_engine(pt, engine);
722
723                 if (prio <= pt->priority)
724                         continue;
725
726                 pt->priority = prio;
727                 if (!RB_EMPTY_NODE(&pt->node)) {
728                         rb_erase(&pt->node, &engine->execlist_queue);
729                         if (insert_request(pt, &engine->execlist_queue))
730                                 engine->execlist_first = &pt->node;
731                 }
732         }
733
734         spin_unlock_irq(&engine->timeline->lock);
735
736         /* XXX Do we need to preempt to make room for us and our deps? */
737 }
738
739 static int execlists_context_pin(struct intel_engine_cs *engine,
740                                  struct i915_gem_context *ctx)
741 {
742         struct intel_context *ce = &ctx->engine[engine->id];
743         unsigned int flags;
744         void *vaddr;
745         int ret;
746
747         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
748
749         if (ce->pin_count++)
750                 return 0;
751         GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
752
753         if (!ce->state) {
754                 ret = execlists_context_deferred_alloc(ctx, engine);
755                 if (ret)
756                         goto err;
757         }
758         GEM_BUG_ON(!ce->state);
759
760         flags = PIN_GLOBAL | PIN_HIGH;
761         if (ctx->ggtt_offset_bias)
762                 flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
763
764         ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN, flags);
765         if (ret)
766                 goto err;
767
768         vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
769         if (IS_ERR(vaddr)) {
770                 ret = PTR_ERR(vaddr);
771                 goto unpin_vma;
772         }
773
774         ret = intel_ring_pin(ce->ring, ctx->ggtt_offset_bias);
775         if (ret)
776                 goto unpin_map;
777
778         intel_lr_context_descriptor_update(ctx, engine);
779
780         ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
781         ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
782                 i915_ggtt_offset(ce->ring->vma);
783
784         ce->state->obj->mm.dirty = true;
785
786         i915_gem_context_get(ctx);
787         return 0;
788
789 unpin_map:
790         i915_gem_object_unpin_map(ce->state->obj);
791 unpin_vma:
792         __i915_vma_unpin(ce->state);
793 err:
794         ce->pin_count = 0;
795         return ret;
796 }
797
798 static void execlists_context_unpin(struct intel_engine_cs *engine,
799                                     struct i915_gem_context *ctx)
800 {
801         struct intel_context *ce = &ctx->engine[engine->id];
802
803         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
804         GEM_BUG_ON(ce->pin_count == 0);
805
806         if (--ce->pin_count)
807                 return;
808
809         intel_ring_unpin(ce->ring);
810
811         i915_gem_object_unpin_map(ce->state->obj);
812         i915_vma_unpin(ce->state);
813
814         i915_gem_context_put(ctx);
815 }
816
817 static int execlists_request_alloc(struct drm_i915_gem_request *request)
818 {
819         struct intel_engine_cs *engine = request->engine;
820         struct intel_context *ce = &request->ctx->engine[engine->id];
821         u32 *cs;
822         int ret;
823
824         GEM_BUG_ON(!ce->pin_count);
825
826         /* Flush enough space to reduce the likelihood of waiting after
827          * we start building the request - in which case we will just
828          * have to repeat work.
829          */
830         request->reserved_space += EXECLISTS_REQUEST_SIZE;
831
832         GEM_BUG_ON(!ce->ring);
833         request->ring = ce->ring;
834
835         if (i915.enable_guc_submission) {
836                 /*
837                  * Check that the GuC has space for the request before
838                  * going any further, as the i915_add_request() call
839                  * later on mustn't fail ...
840                  */
841                 ret = i915_guc_wq_reserve(request);
842                 if (ret)
843                         goto err;
844         }
845
846         cs = intel_ring_begin(request, 0);
847         if (IS_ERR(cs)) {
848                 ret = PTR_ERR(cs);
849                 goto err_unreserve;
850         }
851
852         if (!ce->initialised) {
853                 ret = engine->init_context(request);
854                 if (ret)
855                         goto err_unreserve;
856
857                 ce->initialised = true;
858         }
859
860         /* Note that after this point, we have committed to using
861          * this request as it is being used to both track the
862          * state of engine initialisation and liveness of the
863          * golden renderstate above. Think twice before you try
864          * to cancel/unwind this request now.
865          */
866
867         request->reserved_space -= EXECLISTS_REQUEST_SIZE;
868         return 0;
869
870 err_unreserve:
871         if (i915.enable_guc_submission)
872                 i915_guc_wq_unreserve(request);
873 err:
874         return ret;
875 }
876
877 /*
878  * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
879  * PIPE_CONTROL instruction. This is required for the flush to happen correctly
880  * but there is a slight complication as this is applied in WA batch where the
881  * values are only initialized once so we cannot take register value at the
882  * beginning and reuse it further; hence we save its value to memory, upload a
883  * constant value with bit21 set and then we restore it back with the saved value.
884  * To simplify the WA, a constant value is formed by using the default value
885  * of this register. This shouldn't be a problem because we are only modifying
886  * it for a short period and this batch in non-premptible. We can ofcourse
887  * use additional instructions that read the actual value of the register
888  * at that time and set our bit of interest but it makes the WA complicated.
889  *
890  * This WA is also required for Gen9 so extracting as a function avoids
891  * code duplication.
892  */
893 static u32 *
894 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)
895 {
896         *batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
897         *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
898         *batch++ = i915_ggtt_offset(engine->scratch) + 256;
899         *batch++ = 0;
900
901         *batch++ = MI_LOAD_REGISTER_IMM(1);
902         *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
903         *batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES;
904
905         batch = gen8_emit_pipe_control(batch,
906                                        PIPE_CONTROL_CS_STALL |
907                                        PIPE_CONTROL_DC_FLUSH_ENABLE,
908                                        0);
909
910         *batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
911         *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
912         *batch++ = i915_ggtt_offset(engine->scratch) + 256;
913         *batch++ = 0;
914
915         return batch;
916 }
917
918 /*
919  * Typically we only have one indirect_ctx and per_ctx batch buffer which are
920  * initialized at the beginning and shared across all contexts but this field
921  * helps us to have multiple batches at different offsets and select them based
922  * on a criteria. At the moment this batch always start at the beginning of the page
923  * and at this point we don't have multiple wa_ctx batch buffers.
924  *
925  * The number of WA applied are not known at the beginning; we use this field
926  * to return the no of DWORDS written.
927  *
928  * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
929  * so it adds NOOPs as padding to make it cacheline aligned.
930  * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
931  * makes a complete batch buffer.
932  */
933 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
934 {
935         /* WaDisableCtxRestoreArbitration:bdw,chv */
936         *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
937
938         /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
939         if (IS_BROADWELL(engine->i915))
940                 batch = gen8_emit_flush_coherentl3_wa(engine, batch);
941
942         /* WaClearSlmSpaceAtContextSwitch:bdw,chv */
943         /* Actual scratch location is at 128 bytes offset */
944         batch = gen8_emit_pipe_control(batch,
945                                        PIPE_CONTROL_FLUSH_L3 |
946                                        PIPE_CONTROL_GLOBAL_GTT_IVB |
947                                        PIPE_CONTROL_CS_STALL |
948                                        PIPE_CONTROL_QW_WRITE,
949                                        i915_ggtt_offset(engine->scratch) +
950                                        2 * CACHELINE_BYTES);
951
952         /* Pad to end of cacheline */
953         while ((unsigned long)batch % CACHELINE_BYTES)
954                 *batch++ = MI_NOOP;
955
956         /*
957          * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because
958          * execution depends on the length specified in terms of cache lines
959          * in the register CTX_RCS_INDIRECT_CTX
960          */
961
962         return batch;
963 }
964
965 /*
966  *  This batch is started immediately after indirect_ctx batch. Since we ensure
967  *  that indirect_ctx ends on a cacheline this batch is aligned automatically.
968  *
969  *  The number of DWORDS written are returned using this field.
970  *
971  *  This batch is terminated with MI_BATCH_BUFFER_END and so we need not add padding
972  *  to align it with cacheline as padding after MI_BATCH_BUFFER_END is redundant.
973  */
974 static u32 *gen8_init_perctx_bb(struct intel_engine_cs *engine, u32 *batch)
975 {
976         /* WaDisableCtxRestoreArbitration:bdw,chv */
977         *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
978         *batch++ = MI_BATCH_BUFFER_END;
979
980         return batch;
981 }
982
983 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
984 {
985         /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */
986         batch = gen8_emit_flush_coherentl3_wa(engine, batch);
987
988         /* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */
989         *batch++ = MI_LOAD_REGISTER_IMM(1);
990         *batch++ = i915_mmio_reg_offset(COMMON_SLICE_CHICKEN2);
991         *batch++ = _MASKED_BIT_DISABLE(
992                         GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE);
993         *batch++ = MI_NOOP;
994
995         /* WaClearSlmSpaceAtContextSwitch:kbl */
996         /* Actual scratch location is at 128 bytes offset */
997         if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) {
998                 batch = gen8_emit_pipe_control(batch,
999                                                PIPE_CONTROL_FLUSH_L3 |
1000                                                PIPE_CONTROL_GLOBAL_GTT_IVB |
1001                                                PIPE_CONTROL_CS_STALL |
1002                                                PIPE_CONTROL_QW_WRITE,
1003                                                i915_ggtt_offset(engine->scratch)
1004                                                + 2 * CACHELINE_BYTES);
1005         }
1006
1007         /* WaMediaPoolStateCmdInWABB:bxt,glk */
1008         if (HAS_POOLED_EU(engine->i915)) {
1009                 /*
1010                  * EU pool configuration is setup along with golden context
1011                  * during context initialization. This value depends on
1012                  * device type (2x6 or 3x6) and needs to be updated based
1013                  * on which subslice is disabled especially for 2x6
1014                  * devices, however it is safe to load default
1015                  * configuration of 3x6 device instead of masking off
1016                  * corresponding bits because HW ignores bits of a disabled
1017                  * subslice and drops down to appropriate config. Please
1018                  * see render_state_setup() in i915_gem_render_state.c for
1019                  * possible configurations, to avoid duplication they are
1020                  * not shown here again.
1021                  */
1022                 *batch++ = GEN9_MEDIA_POOL_STATE;
1023                 *batch++ = GEN9_MEDIA_POOL_ENABLE;
1024                 *batch++ = 0x00777000;
1025                 *batch++ = 0;
1026                 *batch++ = 0;
1027                 *batch++ = 0;
1028         }
1029
1030         /* Pad to end of cacheline */
1031         while ((unsigned long)batch % CACHELINE_BYTES)
1032                 *batch++ = MI_NOOP;
1033
1034         return batch;
1035 }
1036
1037 static u32 *gen9_init_perctx_bb(struct intel_engine_cs *engine, u32 *batch)
1038 {
1039         *batch++ = MI_BATCH_BUFFER_END;
1040
1041         return batch;
1042 }
1043
1044 #define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE)
1045
1046 static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
1047 {
1048         struct drm_i915_gem_object *obj;
1049         struct i915_vma *vma;
1050         int err;
1051
1052         obj = i915_gem_object_create(engine->i915, CTX_WA_BB_OBJ_SIZE);
1053         if (IS_ERR(obj))
1054                 return PTR_ERR(obj);
1055
1056         vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
1057         if (IS_ERR(vma)) {
1058                 err = PTR_ERR(vma);
1059                 goto err;
1060         }
1061
1062         err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH);
1063         if (err)
1064                 goto err;
1065
1066         engine->wa_ctx.vma = vma;
1067         return 0;
1068
1069 err:
1070         i915_gem_object_put(obj);
1071         return err;
1072 }
1073
1074 static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine)
1075 {
1076         i915_vma_unpin_and_release(&engine->wa_ctx.vma);
1077 }
1078
1079 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
1080
1081 static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1082 {
1083         struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1084         struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx,
1085                                             &wa_ctx->per_ctx };
1086         wa_bb_func_t wa_bb_fn[2];
1087         struct page *page;
1088         void *batch, *batch_ptr;
1089         unsigned int i;
1090         int ret;
1091
1092         if (WARN_ON(engine->id != RCS || !engine->scratch))
1093                 return -EINVAL;
1094
1095         switch (INTEL_GEN(engine->i915)) {
1096         case 9:
1097                 wa_bb_fn[0] = gen9_init_indirectctx_bb;
1098                 wa_bb_fn[1] = gen9_init_perctx_bb;
1099                 break;
1100         case 8:
1101                 wa_bb_fn[0] = gen8_init_indirectctx_bb;
1102                 wa_bb_fn[1] = gen8_init_perctx_bb;
1103                 break;
1104         default:
1105                 MISSING_CASE(INTEL_GEN(engine->i915));
1106                 return 0;
1107         }
1108
1109         ret = lrc_setup_wa_ctx(engine);
1110         if (ret) {
1111                 DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
1112                 return ret;
1113         }
1114
1115         page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
1116         batch = batch_ptr = kmap_atomic(page);
1117
1118         /*
1119          * Emit the two workaround batch buffers, recording the offset from the
1120          * start of the workaround batch buffer object for each and their
1121          * respective sizes.
1122          */
1123         for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) {
1124                 wa_bb[i]->offset = batch_ptr - batch;
1125                 if (WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, CACHELINE_BYTES))) {
1126                         ret = -EINVAL;
1127                         break;
1128                 }
1129                 batch_ptr = wa_bb_fn[i](engine, batch_ptr);
1130                 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);
1131         }
1132
1133         BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE);
1134
1135         kunmap_atomic(batch);
1136         if (ret)
1137                 lrc_destroy_wa_ctx(engine);
1138
1139         return ret;
1140 }
1141
1142 static u32 port_seqno(struct execlist_port *port)
1143 {
1144         return port->request ? port->request->global_seqno : 0;
1145 }
1146
1147 static int gen8_init_common_ring(struct intel_engine_cs *engine)
1148 {
1149         struct drm_i915_private *dev_priv = engine->i915;
1150         int ret;
1151
1152         ret = intel_mocs_init_engine(engine);
1153         if (ret)
1154                 return ret;
1155
1156         intel_engine_reset_breadcrumbs(engine);
1157         intel_engine_init_hangcheck(engine);
1158
1159         I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
1160         I915_WRITE(RING_MODE_GEN7(engine),
1161                    _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
1162         I915_WRITE(RING_HWS_PGA(engine->mmio_base),
1163                    engine->status_page.ggtt_offset);
1164         POSTING_READ(RING_HWS_PGA(engine->mmio_base));
1165
1166         DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name);
1167
1168         /* After a GPU reset, we may have requests to replay */
1169         clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
1170         if (!i915.enable_guc_submission && !execlists_elsp_idle(engine)) {
1171                 DRM_DEBUG_DRIVER("Restarting %s from requests [0x%x, 0x%x]\n",
1172                                  engine->name,
1173                                  port_seqno(&engine->execlist_port[0]),
1174                                  port_seqno(&engine->execlist_port[1]));
1175                 engine->execlist_port[0].count = 0;
1176                 engine->execlist_port[1].count = 0;
1177                 execlists_submit_ports(engine);
1178         }
1179
1180         return 0;
1181 }
1182
1183 static int gen8_init_render_ring(struct intel_engine_cs *engine)
1184 {
1185         struct drm_i915_private *dev_priv = engine->i915;
1186         int ret;
1187
1188         ret = gen8_init_common_ring(engine);
1189         if (ret)
1190                 return ret;
1191
1192         /* We need to disable the AsyncFlip performance optimisations in order
1193          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1194          * programmed to '1' on all products.
1195          *
1196          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
1197          */
1198         I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1199
1200         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1201
1202         return init_workarounds_ring(engine);
1203 }
1204
1205 static int gen9_init_render_ring(struct intel_engine_cs *engine)
1206 {
1207         int ret;
1208
1209         ret = gen8_init_common_ring(engine);
1210         if (ret)
1211                 return ret;
1212
1213         return init_workarounds_ring(engine);
1214 }
1215
1216 static void reset_common_ring(struct intel_engine_cs *engine,
1217                               struct drm_i915_gem_request *request)
1218 {
1219         struct execlist_port *port = engine->execlist_port;
1220         struct intel_context *ce;
1221
1222         /* If the request was innocent, we leave the request in the ELSP
1223          * and will try to replay it on restarting. The context image may
1224          * have been corrupted by the reset, in which case we may have
1225          * to service a new GPU hang, but more likely we can continue on
1226          * without impact.
1227          *
1228          * If the request was guilty, we presume the context is corrupt
1229          * and have to at least restore the RING register in the context
1230          * image back to the expected values to skip over the guilty request.
1231          */
1232         if (!request || request->fence.error != -EIO)
1233                 return;
1234
1235         /* We want a simple context + ring to execute the breadcrumb update.
1236          * We cannot rely on the context being intact across the GPU hang,
1237          * so clear it and rebuild just what we need for the breadcrumb.
1238          * All pending requests for this context will be zapped, and any
1239          * future request will be after userspace has had the opportunity
1240          * to recreate its own state.
1241          */
1242         ce = &request->ctx->engine[engine->id];
1243         execlists_init_reg_state(ce->lrc_reg_state,
1244                                  request->ctx, engine, ce->ring);
1245
1246         /* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
1247         ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
1248                 i915_ggtt_offset(ce->ring->vma);
1249         ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
1250
1251         request->ring->head = request->postfix;
1252         intel_ring_update_space(request->ring);
1253
1254         /* Catch up with any missed context-switch interrupts */
1255         if (request->ctx != port[0].request->ctx) {
1256                 i915_gem_request_put(port[0].request);
1257                 port[0] = port[1];
1258                 memset(&port[1], 0, sizeof(port[1]));
1259         }
1260
1261         GEM_BUG_ON(request->ctx != port[0].request->ctx);
1262
1263         /* Reset WaIdleLiteRestore:bdw,skl as well */
1264         request->tail =
1265                 intel_ring_wrap(request->ring,
1266                                 request->wa_tail - WA_TAIL_DWORDS*sizeof(u32));
1267         assert_ring_tail_valid(request->ring, request->tail);
1268 }
1269
1270 static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
1271 {
1272         struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt;
1273         struct intel_engine_cs *engine = req->engine;
1274         const int num_lri_cmds = GEN8_3LVL_PDPES * 2;
1275         u32 *cs;
1276         int i;
1277
1278         cs = intel_ring_begin(req, num_lri_cmds * 2 + 2);
1279         if (IS_ERR(cs))
1280                 return PTR_ERR(cs);
1281
1282         *cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds);
1283         for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
1284                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
1285
1286                 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i));
1287                 *cs++ = upper_32_bits(pd_daddr);
1288                 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i));
1289                 *cs++ = lower_32_bits(pd_daddr);
1290         }
1291
1292         *cs++ = MI_NOOP;
1293         intel_ring_advance(req, cs);
1294
1295         return 0;
1296 }
1297
1298 static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1299                               u64 offset, u32 len,
1300                               const unsigned int flags)
1301 {
1302         u32 *cs;
1303         int ret;
1304
1305         /* Don't rely in hw updating PDPs, specially in lite-restore.
1306          * Ideally, we should set Force PD Restore in ctx descriptor,
1307          * but we can't. Force Restore would be a second option, but
1308          * it is unsafe in case of lite-restore (because the ctx is
1309          * not idle). PML4 is allocated during ppgtt init so this is
1310          * not needed in 48-bit.*/
1311         if (req->ctx->ppgtt &&
1312             (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings) &&
1313             !i915_vm_is_48bit(&req->ctx->ppgtt->base) &&
1314             !intel_vgpu_active(req->i915)) {
1315                 ret = intel_logical_ring_emit_pdps(req);
1316                 if (ret)
1317                         return ret;
1318
1319                 req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine);
1320         }
1321
1322         cs = intel_ring_begin(req, 4);
1323         if (IS_ERR(cs))
1324                 return PTR_ERR(cs);
1325
1326         /* FIXME(BDW): Address space and security selectors. */
1327         *cs++ = MI_BATCH_BUFFER_START_GEN8 |
1328                 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)) |
1329                 (flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
1330         *cs++ = lower_32_bits(offset);
1331         *cs++ = upper_32_bits(offset);
1332         *cs++ = MI_NOOP;
1333         intel_ring_advance(req, cs);
1334
1335         return 0;
1336 }
1337
1338 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
1339 {
1340         struct drm_i915_private *dev_priv = engine->i915;
1341         I915_WRITE_IMR(engine,
1342                        ~(engine->irq_enable_mask | engine->irq_keep_mask));
1343         POSTING_READ_FW(RING_IMR(engine->mmio_base));
1344 }
1345
1346 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
1347 {
1348         struct drm_i915_private *dev_priv = engine->i915;
1349         I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1350 }
1351
1352 static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode)
1353 {
1354         u32 cmd, *cs;
1355
1356         cs = intel_ring_begin(request, 4);
1357         if (IS_ERR(cs))
1358                 return PTR_ERR(cs);
1359
1360         cmd = MI_FLUSH_DW + 1;
1361
1362         /* We always require a command barrier so that subsequent
1363          * commands, such as breadcrumb interrupts, are strictly ordered
1364          * wrt the contents of the write cache being flushed to memory
1365          * (and thus being coherent from the CPU).
1366          */
1367         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1368
1369         if (mode & EMIT_INVALIDATE) {
1370                 cmd |= MI_INVALIDATE_TLB;
1371                 if (request->engine->id == VCS)
1372                         cmd |= MI_INVALIDATE_BSD;
1373         }
1374
1375         *cs++ = cmd;
1376         *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
1377         *cs++ = 0; /* upper addr */
1378         *cs++ = 0; /* value */
1379         intel_ring_advance(request, cs);
1380
1381         return 0;
1382 }
1383
1384 static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
1385                                   u32 mode)
1386 {
1387         struct intel_engine_cs *engine = request->engine;
1388         u32 scratch_addr =
1389                 i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
1390         bool vf_flush_wa = false, dc_flush_wa = false;
1391         u32 *cs, flags = 0;
1392         int len;
1393
1394         flags |= PIPE_CONTROL_CS_STALL;
1395
1396         if (mode & EMIT_FLUSH) {
1397                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
1398                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1399                 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
1400                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
1401         }
1402
1403         if (mode & EMIT_INVALIDATE) {
1404                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
1405                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
1406                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1407                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1408                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
1409                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
1410                 flags |= PIPE_CONTROL_QW_WRITE;
1411                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
1412
1413                 /*
1414                  * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
1415                  * pipe control.
1416                  */
1417                 if (IS_GEN9(request->i915))
1418                         vf_flush_wa = true;
1419
1420                 /* WaForGAMHang:kbl */
1421                 if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0))
1422                         dc_flush_wa = true;
1423         }
1424
1425         len = 6;
1426
1427         if (vf_flush_wa)
1428                 len += 6;
1429
1430         if (dc_flush_wa)
1431                 len += 12;
1432
1433         cs = intel_ring_begin(request, len);
1434         if (IS_ERR(cs))
1435                 return PTR_ERR(cs);
1436
1437         if (vf_flush_wa)
1438                 cs = gen8_emit_pipe_control(cs, 0, 0);
1439
1440         if (dc_flush_wa)
1441                 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,
1442                                             0);
1443
1444         cs = gen8_emit_pipe_control(cs, flags, scratch_addr);
1445
1446         if (dc_flush_wa)
1447                 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);
1448
1449         intel_ring_advance(request, cs);
1450
1451         return 0;
1452 }
1453
1454 /*
1455  * Reserve space for 2 NOOPs at the end of each request to be
1456  * used as a workaround for not being allowed to do lite
1457  * restore with HEAD==TAIL (WaIdleLiteRestore).
1458  */
1459 static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs)
1460 {
1461         *cs++ = MI_NOOP;
1462         *cs++ = MI_NOOP;
1463         request->wa_tail = intel_ring_offset(request, cs);
1464 }
1465
1466 static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
1467 {
1468         /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
1469         BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
1470
1471         *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
1472         *cs++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT;
1473         *cs++ = 0;
1474         *cs++ = request->global_seqno;
1475         *cs++ = MI_USER_INTERRUPT;
1476         *cs++ = MI_NOOP;
1477         request->tail = intel_ring_offset(request, cs);
1478         assert_ring_tail_valid(request->ring, request->tail);
1479
1480         gen8_emit_wa_tail(request, cs);
1481 }
1482
1483 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
1484
1485 static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
1486                                         u32 *cs)
1487 {
1488         /* We're using qword write, seqno should be aligned to 8 bytes. */
1489         BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
1490
1491         /* w/a for post sync ops following a GPGPU operation we
1492          * need a prior CS_STALL, which is emitted by the flush
1493          * following the batch.
1494          */
1495         *cs++ = GFX_OP_PIPE_CONTROL(6);
1496         *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
1497                 PIPE_CONTROL_QW_WRITE;
1498         *cs++ = intel_hws_seqno_address(request->engine);
1499         *cs++ = 0;
1500         *cs++ = request->global_seqno;
1501         /* We're thrashing one dword of HWS. */
1502         *cs++ = 0;
1503         *cs++ = MI_USER_INTERRUPT;
1504         *cs++ = MI_NOOP;
1505         request->tail = intel_ring_offset(request, cs);
1506         assert_ring_tail_valid(request->ring, request->tail);
1507
1508         gen8_emit_wa_tail(request, cs);
1509 }
1510
1511 static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS;
1512
1513 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
1514 {
1515         int ret;
1516
1517         ret = intel_ring_workarounds_emit(req);
1518         if (ret)
1519                 return ret;
1520
1521         ret = intel_rcs_context_init_mocs(req);
1522         /*
1523          * Failing to program the MOCS is non-fatal.The system will not
1524          * run at peak performance. So generate an error and carry on.
1525          */
1526         if (ret)
1527                 DRM_ERROR("MOCS failed to program: expect performance issues.\n");
1528
1529         return i915_gem_render_state_emit(req);
1530 }
1531
1532 /**
1533  * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
1534  * @engine: Engine Command Streamer.
1535  */
1536 void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
1537 {
1538         struct drm_i915_private *dev_priv;
1539
1540         /*
1541          * Tasklet cannot be active at this point due intel_mark_active/idle
1542          * so this is just for documentation.
1543          */
1544         if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state)))
1545                 tasklet_kill(&engine->irq_tasklet);
1546
1547         dev_priv = engine->i915;
1548
1549         if (engine->buffer) {
1550                 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
1551         }
1552
1553         if (engine->cleanup)
1554                 engine->cleanup(engine);
1555
1556         if (engine->status_page.vma) {
1557                 i915_gem_object_unpin_map(engine->status_page.vma->obj);
1558                 engine->status_page.vma = NULL;
1559         }
1560
1561         intel_engine_cleanup_common(engine);
1562
1563         lrc_destroy_wa_ctx(engine);
1564         engine->i915 = NULL;
1565         dev_priv->engine[engine->id] = NULL;
1566         kfree(engine);
1567 }
1568
1569 static void execlists_set_default_submission(struct intel_engine_cs *engine)
1570 {
1571         engine->submit_request = execlists_submit_request;
1572         engine->schedule = execlists_schedule;
1573         engine->irq_tasklet.func = intel_lrc_irq_handler;
1574 }
1575
1576 static void
1577 logical_ring_default_vfuncs(struct intel_engine_cs *engine)
1578 {
1579         /* Default vfuncs which can be overriden by each engine. */
1580         engine->init_hw = gen8_init_common_ring;
1581         engine->reset_hw = reset_common_ring;
1582
1583         engine->context_pin = execlists_context_pin;
1584         engine->context_unpin = execlists_context_unpin;
1585
1586         engine->request_alloc = execlists_request_alloc;
1587
1588         engine->emit_flush = gen8_emit_flush;
1589         engine->emit_breadcrumb = gen8_emit_breadcrumb;
1590         engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
1591
1592         engine->set_default_submission = execlists_set_default_submission;
1593
1594         engine->irq_enable = gen8_logical_ring_enable_irq;
1595         engine->irq_disable = gen8_logical_ring_disable_irq;
1596         engine->emit_bb_start = gen8_emit_bb_start;
1597 }
1598
1599 static inline void
1600 logical_ring_default_irqs(struct intel_engine_cs *engine)
1601 {
1602         unsigned shift = engine->irq_shift;
1603         engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
1604         engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
1605 }
1606
1607 static int
1608 lrc_setup_hws(struct intel_engine_cs *engine, struct i915_vma *vma)
1609 {
1610         const int hws_offset = LRC_PPHWSP_PN * PAGE_SIZE;
1611         void *hws;
1612
1613         /* The HWSP is part of the default context object in LRC mode. */
1614         hws = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1615         if (IS_ERR(hws))
1616                 return PTR_ERR(hws);
1617
1618         engine->status_page.page_addr = hws + hws_offset;
1619         engine->status_page.ggtt_offset = i915_ggtt_offset(vma) + hws_offset;
1620         engine->status_page.vma = vma;
1621
1622         return 0;
1623 }
1624
1625 static void
1626 logical_ring_setup(struct intel_engine_cs *engine)
1627 {
1628         struct drm_i915_private *dev_priv = engine->i915;
1629         enum forcewake_domains fw_domains;
1630
1631         intel_engine_setup_common(engine);
1632
1633         /* Intentionally left blank. */
1634         engine->buffer = NULL;
1635
1636         fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
1637                                                     RING_ELSP(engine),
1638                                                     FW_REG_WRITE);
1639
1640         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
1641                                                      RING_CONTEXT_STATUS_PTR(engine),
1642                                                      FW_REG_READ | FW_REG_WRITE);
1643
1644         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
1645                                                      RING_CONTEXT_STATUS_BUF_BASE(engine),
1646                                                      FW_REG_READ);
1647
1648         engine->fw_domains = fw_domains;
1649
1650         tasklet_init(&engine->irq_tasklet,
1651                      intel_lrc_irq_handler, (unsigned long)engine);
1652
1653         logical_ring_default_vfuncs(engine);
1654         logical_ring_default_irqs(engine);
1655 }
1656
1657 static int
1658 logical_ring_init(struct intel_engine_cs *engine)
1659 {
1660         struct i915_gem_context *dctx = engine->i915->kernel_context;
1661         int ret;
1662
1663         ret = intel_engine_init_common(engine);
1664         if (ret)
1665                 goto error;
1666
1667         /* And setup the hardware status page. */
1668         ret = lrc_setup_hws(engine, dctx->engine[engine->id].state);
1669         if (ret) {
1670                 DRM_ERROR("Failed to set up hws %s: %d\n", engine->name, ret);
1671                 goto error;
1672         }
1673
1674         return 0;
1675
1676 error:
1677         intel_logical_ring_cleanup(engine);
1678         return ret;
1679 }
1680
1681 int logical_render_ring_init(struct intel_engine_cs *engine)
1682 {
1683         struct drm_i915_private *dev_priv = engine->i915;
1684         int ret;
1685
1686         logical_ring_setup(engine);
1687
1688         if (HAS_L3_DPF(dev_priv))
1689                 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1690
1691         /* Override some for render ring. */
1692         if (INTEL_GEN(dev_priv) >= 9)
1693                 engine->init_hw = gen9_init_render_ring;
1694         else
1695                 engine->init_hw = gen8_init_render_ring;
1696         engine->init_context = gen8_init_rcs_context;
1697         engine->emit_flush = gen8_emit_flush_render;
1698         engine->emit_breadcrumb = gen8_emit_breadcrumb_render;
1699         engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz;
1700
1701         ret = intel_engine_create_scratch(engine, PAGE_SIZE);
1702         if (ret)
1703                 return ret;
1704
1705         ret = intel_init_workaround_bb(engine);
1706         if (ret) {
1707                 /*
1708                  * We continue even if we fail to initialize WA batch
1709                  * because we only expect rare glitches but nothing
1710                  * critical to prevent us from using GPU
1711                  */
1712                 DRM_ERROR("WA batch buffer initialization failed: %d\n",
1713                           ret);
1714         }
1715
1716         return logical_ring_init(engine);
1717 }
1718
1719 int logical_xcs_ring_init(struct intel_engine_cs *engine)
1720 {
1721         logical_ring_setup(engine);
1722
1723         return logical_ring_init(engine);
1724 }
1725
1726 static u32
1727 make_rpcs(struct drm_i915_private *dev_priv)
1728 {
1729         u32 rpcs = 0;
1730
1731         /*
1732          * No explicit RPCS request is needed to ensure full
1733          * slice/subslice/EU enablement prior to Gen9.
1734         */
1735         if (INTEL_GEN(dev_priv) < 9)
1736                 return 0;
1737
1738         /*
1739          * Starting in Gen9, render power gating can leave
1740          * slice/subslice/EU in a partially enabled state. We
1741          * must make an explicit request through RPCS for full
1742          * enablement.
1743         */
1744         if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) {
1745                 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
1746                 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) <<
1747                         GEN8_RPCS_S_CNT_SHIFT;
1748                 rpcs |= GEN8_RPCS_ENABLE;
1749         }
1750
1751         if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
1752                 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
1753                 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) <<
1754                         GEN8_RPCS_SS_CNT_SHIFT;
1755                 rpcs |= GEN8_RPCS_ENABLE;
1756         }
1757
1758         if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) {
1759                 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
1760                         GEN8_RPCS_EU_MIN_SHIFT;
1761                 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
1762                         GEN8_RPCS_EU_MAX_SHIFT;
1763                 rpcs |= GEN8_RPCS_ENABLE;
1764         }
1765
1766         return rpcs;
1767 }
1768
1769 static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
1770 {
1771         u32 indirect_ctx_offset;
1772
1773         switch (INTEL_GEN(engine->i915)) {
1774         default:
1775                 MISSING_CASE(INTEL_GEN(engine->i915));
1776                 /* fall through */
1777         case 9:
1778                 indirect_ctx_offset =
1779                         GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
1780                 break;
1781         case 8:
1782                 indirect_ctx_offset =
1783                         GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
1784                 break;
1785         }
1786
1787         return indirect_ctx_offset;
1788 }
1789
1790 static void execlists_init_reg_state(u32 *regs,
1791                                      struct i915_gem_context *ctx,
1792                                      struct intel_engine_cs *engine,
1793                                      struct intel_ring *ring)
1794 {
1795         struct drm_i915_private *dev_priv = engine->i915;
1796         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
1797         u32 base = engine->mmio_base;
1798         bool rcs = engine->id == RCS;
1799
1800         /* A context is actually a big batch buffer with several
1801          * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The
1802          * values we are setting here are only for the first context restore:
1803          * on a subsequent save, the GPU will recreate this batchbuffer with new
1804          * values (including all the missing MI_LOAD_REGISTER_IMM commands that
1805          * we are not initializing here).
1806          */
1807         regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) |
1808                                  MI_LRI_FORCE_POSTED;
1809
1810         CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
1811                 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
1812                                    CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
1813                                    (HAS_RESOURCE_STREAMER(dev_priv) ?
1814                                    CTX_CTRL_RS_CTX_ENABLE : 0)));
1815         CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
1816         CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0);
1817         CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0);
1818         CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base),
1819                 RING_CTL_SIZE(ring->size) | RING_VALID);
1820         CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0);
1821         CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0);
1822         CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT);
1823         CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0);
1824         CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0);
1825         CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0);
1826         if (rcs) {
1827                 CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
1828                 CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0);
1829                 CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET,
1830                         RING_INDIRECT_CTX_OFFSET(base), 0);
1831
1832                 if (engine->wa_ctx.vma) {
1833                         struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1834                         u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
1835
1836                         regs[CTX_RCS_INDIRECT_CTX + 1] =
1837                                 (ggtt_offset + wa_ctx->indirect_ctx.offset) |
1838                                 (wa_ctx->indirect_ctx.size / CACHELINE_BYTES);
1839
1840                         regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] =
1841                                 intel_lr_indirect_ctx_offset(engine) << 6;
1842
1843                         regs[CTX_BB_PER_CTX_PTR + 1] =
1844                                 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;
1845                 }
1846         }
1847
1848         regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;
1849
1850         CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0);
1851         /* PDP values well be assigned later if needed */
1852         CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0);
1853         CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0);
1854         CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0);
1855         CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0);
1856         CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0);
1857         CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0);
1858         CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0);
1859         CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0);
1860
1861         if (ppgtt && i915_vm_is_48bit(&ppgtt->base)) {
1862                 /* 64b PPGTT (48bit canonical)
1863                  * PDP0_DESCRIPTOR contains the base address to PML4 and
1864                  * other PDP Descriptors are ignored.
1865                  */
1866                 ASSIGN_CTX_PML4(ppgtt, regs);
1867         }
1868
1869         if (rcs) {
1870                 regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
1871                 CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
1872                         make_rpcs(dev_priv));
1873         }
1874 }
1875
1876 static int
1877 populate_lr_context(struct i915_gem_context *ctx,
1878                     struct drm_i915_gem_object *ctx_obj,
1879                     struct intel_engine_cs *engine,
1880                     struct intel_ring *ring)
1881 {
1882         void *vaddr;
1883         int ret;
1884
1885         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
1886         if (ret) {
1887                 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
1888                 return ret;
1889         }
1890
1891         vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
1892         if (IS_ERR(vaddr)) {
1893                 ret = PTR_ERR(vaddr);
1894                 DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
1895                 return ret;
1896         }
1897         ctx_obj->mm.dirty = true;
1898
1899         /* The second page of the context object contains some fields which must
1900          * be set up prior to the first execution. */
1901
1902         execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE,
1903                                  ctx, engine, ring);
1904
1905         i915_gem_object_unpin_map(ctx_obj);
1906
1907         return 0;
1908 }
1909
1910 /**
1911  * intel_lr_context_size() - return the size of the context for an engine
1912  * @engine: which engine to find the context size for
1913  *
1914  * Each engine may require a different amount of space for a context image,
1915  * so when allocating (or copying) an image, this function can be used to
1916  * find the right size for the specific engine.
1917  *
1918  * Return: size (in bytes) of an engine-specific context image
1919  *
1920  * Note: this size includes the HWSP, which is part of the context image
1921  * in LRC mode, but does not include the "shared data page" used with
1922  * GuC submission. The caller should account for this if using the GuC.
1923  */
1924 uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
1925 {
1926         int ret = 0;
1927
1928         WARN_ON(INTEL_GEN(engine->i915) < 8);
1929
1930         switch (engine->id) {
1931         case RCS:
1932                 if (INTEL_GEN(engine->i915) >= 9)
1933                         ret = GEN9_LR_CONTEXT_RENDER_SIZE;
1934                 else
1935                         ret = GEN8_LR_CONTEXT_RENDER_SIZE;
1936                 break;
1937         case VCS:
1938         case BCS:
1939         case VECS:
1940         case VCS2:
1941                 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
1942                 break;
1943         }
1944
1945         return ret;
1946 }
1947
1948 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
1949                                             struct intel_engine_cs *engine)
1950 {
1951         struct drm_i915_gem_object *ctx_obj;
1952         struct intel_context *ce = &ctx->engine[engine->id];
1953         struct i915_vma *vma;
1954         uint32_t context_size;
1955         struct intel_ring *ring;
1956         int ret;
1957
1958         WARN_ON(ce->state);
1959
1960         context_size = round_up(intel_lr_context_size(engine),
1961                                 I915_GTT_PAGE_SIZE);
1962
1963         /* One extra page as the sharing data between driver and GuC */
1964         context_size += PAGE_SIZE * LRC_PPHWSP_PN;
1965
1966         ctx_obj = i915_gem_object_create(ctx->i915, context_size);
1967         if (IS_ERR(ctx_obj)) {
1968                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
1969                 return PTR_ERR(ctx_obj);
1970         }
1971
1972         vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL);
1973         if (IS_ERR(vma)) {
1974                 ret = PTR_ERR(vma);
1975                 goto error_deref_obj;
1976         }
1977
1978         ring = intel_engine_create_ring(engine, ctx->ring_size);
1979         if (IS_ERR(ring)) {
1980                 ret = PTR_ERR(ring);
1981                 goto error_deref_obj;
1982         }
1983
1984         ret = populate_lr_context(ctx, ctx_obj, engine, ring);
1985         if (ret) {
1986                 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
1987                 goto error_ring_free;
1988         }
1989
1990         ce->ring = ring;
1991         ce->state = vma;
1992         ce->initialised |= engine->init_context == NULL;
1993
1994         return 0;
1995
1996 error_ring_free:
1997         intel_ring_free(ring);
1998 error_deref_obj:
1999         i915_gem_object_put(ctx_obj);
2000         return ret;
2001 }
2002
2003 void intel_lr_context_resume(struct drm_i915_private *dev_priv)
2004 {
2005         struct intel_engine_cs *engine;
2006         struct i915_gem_context *ctx;
2007         enum intel_engine_id id;
2008
2009         /* Because we emit WA_TAIL_DWORDS there may be a disparity
2010          * between our bookkeeping in ce->ring->head and ce->ring->tail and
2011          * that stored in context. As we only write new commands from
2012          * ce->ring->tail onwards, everything before that is junk. If the GPU
2013          * starts reading from its RING_HEAD from the context, it may try to
2014          * execute that junk and die.
2015          *
2016          * So to avoid that we reset the context images upon resume. For
2017          * simplicity, we just zero everything out.
2018          */
2019         list_for_each_entry(ctx, &dev_priv->context_list, link) {
2020                 for_each_engine(engine, dev_priv, id) {
2021                         struct intel_context *ce = &ctx->engine[engine->id];
2022                         u32 *reg;
2023
2024                         if (!ce->state)
2025                                 continue;
2026
2027                         reg = i915_gem_object_pin_map(ce->state->obj,
2028                                                       I915_MAP_WB);
2029                         if (WARN_ON(IS_ERR(reg)))
2030                                 continue;
2031
2032                         reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
2033                         reg[CTX_RING_HEAD+1] = 0;
2034                         reg[CTX_RING_TAIL+1] = 0;
2035
2036                         ce->state->obj->mm.dirty = true;
2037                         i915_gem_object_unpin_map(ce->state->obj);
2038
2039                         ce->ring->head = ce->ring->tail = 0;
2040                         intel_ring_update_space(ce->ring);
2041                 }
2042         }
2043 }