]> git.karo-electronics.de Git - karo-tx-linux.git/blob - drivers/gpu/drm/i915/intel_lrc.c
Merge branch 'upstream' of git://git.linux-mips.org/pub/scm/ralf/upstream-linus
[karo-tx-linux.git] / drivers / gpu / drm / i915 / intel_lrc.c
1 /*
2  * Copyright © 2014 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Ben Widawsky <ben@bwidawsk.net>
25  *    Michel Thierry <michel.thierry@intel.com>
26  *    Thomas Daniel <thomas.daniel@intel.com>
27  *    Oscar Mateo <oscar.mateo@intel.com>
28  *
29  */
30
31 /**
32  * DOC: Logical Rings, Logical Ring Contexts and Execlists
33  *
34  * Motivation:
35  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
36  * These expanded contexts enable a number of new abilities, especially
37  * "Execlists" (also implemented in this file).
38  *
39  * One of the main differences with the legacy HW contexts is that logical
40  * ring contexts incorporate many more things to the context's state, like
41  * PDPs or ringbuffer control registers:
42  *
43  * The reason why PDPs are included in the context is straightforward: as
44  * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
45  * contained there mean you don't need to do a ppgtt->switch_mm yourself,
46  * instead, the GPU will do it for you on the context switch.
47  *
48  * But, what about the ringbuffer control registers (head, tail, etc..)?
49  * shouldn't we just need a set of those per engine command streamer? This is
50  * where the name "Logical Rings" starts to make sense: by virtualizing the
51  * rings, the engine cs shifts to a new "ring buffer" with every context
52  * switch. When you want to submit a workload to the GPU you: A) choose your
53  * context, B) find its appropriate virtualized ring, C) write commands to it
54  * and then, finally, D) tell the GPU to switch to that context.
55  *
56  * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
57  * to a contexts is via a context execution list, ergo "Execlists".
58  *
59  * LRC implementation:
60  * Regarding the creation of contexts, we have:
61  *
62  * - One global default context.
63  * - One local default context for each opened fd.
64  * - One local extra context for each context create ioctl call.
65  *
66  * Now that ringbuffers belong per-context (and not per-engine, like before)
67  * and that contexts are uniquely tied to a given engine (and not reusable,
68  * like before) we need:
69  *
70  * - One ringbuffer per-engine inside each context.
71  * - One backing object per-engine inside each context.
72  *
73  * The global default context starts its life with these new objects fully
74  * allocated and populated. The local default context for each opened fd is
75  * more complex, because we don't know at creation time which engine is going
76  * to use them. To handle this, we have implemented a deferred creation of LR
77  * contexts:
78  *
79  * The local context starts its life as a hollow or blank holder, that only
80  * gets populated for a given engine once we receive an execbuffer. If later
81  * on we receive another execbuffer ioctl for the same context but a different
82  * engine, we allocate/populate a new ringbuffer and context backing object and
83  * so on.
84  *
85  * Finally, regarding local contexts created using the ioctl call: as they are
86  * only allowed with the render ring, we can allocate & populate them right
87  * away (no need to defer anything, at least for now).
88  *
89  * Execlists implementation:
90  * Execlists are the new method by which, on gen8+ hardware, workloads are
91  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
92  * This method works as follows:
93  *
94  * When a request is committed, its commands (the BB start and any leading or
95  * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
96  * for the appropriate context. The tail pointer in the hardware context is not
97  * updated at this time, but instead, kept by the driver in the ringbuffer
98  * structure. A structure representing this request is added to a request queue
99  * for the appropriate engine: this structure contains a copy of the context's
100  * tail after the request was written to the ring buffer and a pointer to the
101  * context itself.
102  *
103  * If the engine's request queue was empty before the request was added, the
104  * queue is processed immediately. Otherwise the queue will be processed during
105  * a context switch interrupt. In any case, elements on the queue will get sent
106  * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
107  * globally unique 20-bits submission ID.
108  *
109  * When execution of a request completes, the GPU updates the context status
110  * buffer with a context complete event and generates a context switch interrupt.
111  * During the interrupt handling, the driver examines the events in the buffer:
112  * for each context complete event, if the announced ID matches that on the head
113  * of the request queue, then that request is retired and removed from the queue.
114  *
115  * After processing, if any requests were retired and the queue is not empty
116  * then a new execution list can be submitted. The two requests at the front of
117  * the queue are next to be submitted but since a context may not occur twice in
118  * an execution list, if subsequent requests have the same ID as the first then
119  * the two requests must be combined. This is done simply by discarding requests
120  * at the head of the queue until either only one requests is left (in which case
121  * we use a NULL second context) or the first two requests have unique IDs.
122  *
123  * By always executing the first two requests in the queue the driver ensures
124  * that the GPU is kept as busy as possible. In the case where a single context
125  * completes but a second context is still executing, the request for this second
126  * context will be at the head of the queue when we remove the first one. This
127  * request will then be resubmitted along with a new request for a different context,
128  * which will cause the hardware to continue executing the second request and queue
129  * the new request (the GPU detects the condition of a context getting preempted
130  * with the same context and optimizes the context switch flow by not doing
131  * preemption, but just sampling the new tail pointer).
132  *
133  */
134 #include <linux/interrupt.h>
135
136 #include <drm/drmP.h>
137 #include <drm/i915_drm.h>
138 #include "i915_drv.h"
139 #include "intel_mocs.h"
140
141 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
142 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
143 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
144
145 #define RING_EXECLIST_QFULL             (1 << 0x2)
146 #define RING_EXECLIST1_VALID            (1 << 0x3)
147 #define RING_EXECLIST0_VALID            (1 << 0x4)
148 #define RING_EXECLIST_ACTIVE_STATUS     (3 << 0xE)
149 #define RING_EXECLIST1_ACTIVE           (1 << 0x11)
150 #define RING_EXECLIST0_ACTIVE           (1 << 0x12)
151
152 #define GEN8_CTX_STATUS_IDLE_ACTIVE     (1 << 0)
153 #define GEN8_CTX_STATUS_PREEMPTED       (1 << 1)
154 #define GEN8_CTX_STATUS_ELEMENT_SWITCH  (1 << 2)
155 #define GEN8_CTX_STATUS_ACTIVE_IDLE     (1 << 3)
156 #define GEN8_CTX_STATUS_COMPLETE        (1 << 4)
157 #define GEN8_CTX_STATUS_LITE_RESTORE    (1 << 15)
158
159 #define GEN8_CTX_STATUS_COMPLETED_MASK \
160          (GEN8_CTX_STATUS_ACTIVE_IDLE | \
161           GEN8_CTX_STATUS_PREEMPTED | \
162           GEN8_CTX_STATUS_ELEMENT_SWITCH)
163
164 #define CTX_LRI_HEADER_0                0x01
165 #define CTX_CONTEXT_CONTROL             0x02
166 #define CTX_RING_HEAD                   0x04
167 #define CTX_RING_TAIL                   0x06
168 #define CTX_RING_BUFFER_START           0x08
169 #define CTX_RING_BUFFER_CONTROL         0x0a
170 #define CTX_BB_HEAD_U                   0x0c
171 #define CTX_BB_HEAD_L                   0x0e
172 #define CTX_BB_STATE                    0x10
173 #define CTX_SECOND_BB_HEAD_U            0x12
174 #define CTX_SECOND_BB_HEAD_L            0x14
175 #define CTX_SECOND_BB_STATE             0x16
176 #define CTX_BB_PER_CTX_PTR              0x18
177 #define CTX_RCS_INDIRECT_CTX            0x1a
178 #define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
179 #define CTX_LRI_HEADER_1                0x21
180 #define CTX_CTX_TIMESTAMP               0x22
181 #define CTX_PDP3_UDW                    0x24
182 #define CTX_PDP3_LDW                    0x26
183 #define CTX_PDP2_UDW                    0x28
184 #define CTX_PDP2_LDW                    0x2a
185 #define CTX_PDP1_UDW                    0x2c
186 #define CTX_PDP1_LDW                    0x2e
187 #define CTX_PDP0_UDW                    0x30
188 #define CTX_PDP0_LDW                    0x32
189 #define CTX_LRI_HEADER_2                0x41
190 #define CTX_R_PWR_CLK_STATE             0x42
191 #define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
192
193 #define GEN8_CTX_VALID (1<<0)
194 #define GEN8_CTX_FORCE_PD_RESTORE (1<<1)
195 #define GEN8_CTX_FORCE_RESTORE (1<<2)
196 #define GEN8_CTX_L3LLC_COHERENT (1<<5)
197 #define GEN8_CTX_PRIVILEGE (1<<8)
198
199 #define ASSIGN_CTX_REG(reg_state, pos, reg, val) do { \
200         (reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \
201         (reg_state)[(pos)+1] = (val); \
202 } while (0)
203
204 #define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do {                \
205         const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n)); \
206         reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
207         reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
208 } while (0)
209
210 #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \
211         reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \
212         reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \
213 } while (0)
214
215 enum {
216         FAULT_AND_HANG = 0,
217         FAULT_AND_HALT, /* Debug only */
218         FAULT_AND_STREAM,
219         FAULT_AND_CONTINUE /* Unsupported */
220 };
221 #define GEN8_CTX_ID_SHIFT 32
222 #define GEN8_CTX_ID_WIDTH 21
223 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT        0x17
224 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT        0x26
225
226 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
227 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
228
229 #define WA_TAIL_DWORDS 2
230
231 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
232                                             struct intel_engine_cs *engine);
233 static void execlists_init_reg_state(u32 *reg_state,
234                                      struct i915_gem_context *ctx,
235                                      struct intel_engine_cs *engine,
236                                      struct intel_ring *ring);
237
238 /**
239  * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
240  * @dev_priv: i915 device private
241  * @enable_execlists: value of i915.enable_execlists module parameter.
242  *
243  * Only certain platforms support Execlists (the prerequisites being
244  * support for Logical Ring Contexts and Aliasing PPGTT or better).
245  *
246  * Return: 1 if Execlists is supported and has to be enabled.
247  */
248 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists)
249 {
250         /* On platforms with execlist available, vGPU will only
251          * support execlist mode, no ring buffer mode.
252          */
253         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv))
254                 return 1;
255
256         if (INTEL_GEN(dev_priv) >= 9)
257                 return 1;
258
259         if (enable_execlists == 0)
260                 return 0;
261
262         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) &&
263             USES_PPGTT(dev_priv) &&
264             i915.use_mmio_flip >= 0)
265                 return 1;
266
267         return 0;
268 }
269
270 static void
271 logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
272 {
273         struct drm_i915_private *dev_priv = engine->i915;
274
275         engine->disable_lite_restore_wa =
276                 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) &&
277                 (engine->id == VCS || engine->id == VCS2);
278
279         engine->ctx_desc_template = GEN8_CTX_VALID;
280         if (IS_GEN8(dev_priv))
281                 engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT;
282         engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE;
283
284         /* TODO: WaDisableLiteRestore when we start using semaphore
285          * signalling between Command Streamers */
286         /* ring->ctx_desc_template |= GEN8_CTX_FORCE_RESTORE; */
287
288         /* WaEnableForceRestoreInCtxtDescForVCS:skl */
289         /* WaEnableForceRestoreInCtxtDescForVCS:bxt */
290         if (engine->disable_lite_restore_wa)
291                 engine->ctx_desc_template |= GEN8_CTX_FORCE_RESTORE;
292 }
293
294 /**
295  * intel_lr_context_descriptor_update() - calculate & cache the descriptor
296  *                                        descriptor for a pinned context
297  * @ctx: Context to work on
298  * @engine: Engine the descriptor will be used with
299  *
300  * The context descriptor encodes various attributes of a context,
301  * including its GTT address and some flags. Because it's fairly
302  * expensive to calculate, we'll just do it once and cache the result,
303  * which remains valid until the context is unpinned.
304  *
305  * This is what a descriptor looks like, from LSB to MSB::
306  *
307  *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx_desc_template)
308  *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
309  *      bits 32-52:    ctx ID, a globally unique tag
310  *      bits 53-54:    mbz, reserved for use by hardware
311  *      bits 55-63:    group ID, currently unused and set to 0
312  */
313 static void
314 intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
315                                    struct intel_engine_cs *engine)
316 {
317         struct intel_context *ce = &ctx->engine[engine->id];
318         u64 desc;
319
320         BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH));
321
322         desc = ctx->desc_template;                              /* bits  3-4  */
323         desc |= engine->ctx_desc_template;                      /* bits  0-11 */
324         desc |= i915_ggtt_offset(ce->state) + LRC_PPHWSP_PN * PAGE_SIZE;
325                                                                 /* bits 12-31 */
326         desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;           /* bits 32-52 */
327
328         ce->lrc_desc = desc;
329 }
330
331 uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
332                                      struct intel_engine_cs *engine)
333 {
334         return ctx->engine[engine->id].lrc_desc;
335 }
336
337 static inline void
338 execlists_context_status_change(struct drm_i915_gem_request *rq,
339                                 unsigned long status)
340 {
341         /*
342          * Only used when GVT-g is enabled now. When GVT-g is disabled,
343          * The compiler should eliminate this function as dead-code.
344          */
345         if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
346                 return;
347
348         atomic_notifier_call_chain(&rq->engine->context_status_notifier,
349                                    status, rq);
350 }
351
352 static void
353 execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
354 {
355         ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
356         ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
357         ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
358         ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
359 }
360
361 static u64 execlists_update_context(struct drm_i915_gem_request *rq)
362 {
363         struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
364         struct i915_hw_ppgtt *ppgtt =
365                 rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
366         u32 *reg_state = ce->lrc_reg_state;
367
368         reg_state[CTX_RING_TAIL+1] = rq->tail;
369
370         /* True 32b PPGTT with dynamic page allocation: update PDP
371          * registers and point the unallocated PDPs to scratch page.
372          * PML4 is allocated during ppgtt init, so this is not needed
373          * in 48-bit mode.
374          */
375         if (ppgtt && !USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
376                 execlists_update_context_pdps(ppgtt, reg_state);
377
378         return ce->lrc_desc;
379 }
380
381 static void execlists_submit_ports(struct intel_engine_cs *engine)
382 {
383         struct drm_i915_private *dev_priv = engine->i915;
384         struct execlist_port *port = engine->execlist_port;
385         u32 __iomem *elsp =
386                 dev_priv->regs + i915_mmio_reg_offset(RING_ELSP(engine));
387         u64 desc[2];
388
389         if (!port[0].count)
390                 execlists_context_status_change(port[0].request,
391                                                 INTEL_CONTEXT_SCHEDULE_IN);
392         desc[0] = execlists_update_context(port[0].request);
393         engine->preempt_wa = port[0].count++; /* bdw only? fixed on skl? */
394
395         if (port[1].request) {
396                 GEM_BUG_ON(port[1].count);
397                 execlists_context_status_change(port[1].request,
398                                                 INTEL_CONTEXT_SCHEDULE_IN);
399                 desc[1] = execlists_update_context(port[1].request);
400                 port[1].count = 1;
401         } else {
402                 desc[1] = 0;
403         }
404         GEM_BUG_ON(desc[0] == desc[1]);
405
406         /* You must always write both descriptors in the order below. */
407         writel(upper_32_bits(desc[1]), elsp);
408         writel(lower_32_bits(desc[1]), elsp);
409
410         writel(upper_32_bits(desc[0]), elsp);
411         /* The context is automatically loaded after the following */
412         writel(lower_32_bits(desc[0]), elsp);
413 }
414
415 static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
416 {
417         return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
418                 i915_gem_context_force_single_submission(ctx));
419 }
420
421 static bool can_merge_ctx(const struct i915_gem_context *prev,
422                           const struct i915_gem_context *next)
423 {
424         if (prev != next)
425                 return false;
426
427         if (ctx_single_port_submission(prev))
428                 return false;
429
430         return true;
431 }
432
433 static void execlists_dequeue(struct intel_engine_cs *engine)
434 {
435         struct drm_i915_gem_request *last;
436         struct execlist_port *port = engine->execlist_port;
437         unsigned long flags;
438         struct rb_node *rb;
439         bool submit = false;
440
441         last = port->request;
442         if (last)
443                 /* WaIdleLiteRestore:bdw,skl
444                  * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL
445                  * as we resubmit the request. See gen8_emit_breadcrumb()
446                  * for where we prepare the padding after the end of the
447                  * request.
448                  */
449                 last->tail = last->wa_tail;
450
451         GEM_BUG_ON(port[1].request);
452
453         /* Hardware submission is through 2 ports. Conceptually each port
454          * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
455          * static for a context, and unique to each, so we only execute
456          * requests belonging to a single context from each ring. RING_HEAD
457          * is maintained by the CS in the context image, it marks the place
458          * where it got up to last time, and through RING_TAIL we tell the CS
459          * where we want to execute up to this time.
460          *
461          * In this list the requests are in order of execution. Consecutive
462          * requests from the same context are adjacent in the ringbuffer. We
463          * can combine these requests into a single RING_TAIL update:
464          *
465          *              RING_HEAD...req1...req2
466          *                                    ^- RING_TAIL
467          * since to execute req2 the CS must first execute req1.
468          *
469          * Our goal then is to point each port to the end of a consecutive
470          * sequence of requests as being the most optimal (fewest wake ups
471          * and context switches) submission.
472          */
473
474         spin_lock_irqsave(&engine->timeline->lock, flags);
475         rb = engine->execlist_first;
476         while (rb) {
477                 struct drm_i915_gem_request *cursor =
478                         rb_entry(rb, typeof(*cursor), priotree.node);
479
480                 /* Can we combine this request with the current port? It has to
481                  * be the same context/ringbuffer and not have any exceptions
482                  * (e.g. GVT saying never to combine contexts).
483                  *
484                  * If we can combine the requests, we can execute both by
485                  * updating the RING_TAIL to point to the end of the second
486                  * request, and so we never need to tell the hardware about
487                  * the first.
488                  */
489                 if (last && !can_merge_ctx(cursor->ctx, last->ctx)) {
490                         /* If we are on the second port and cannot combine
491                          * this request with the last, then we are done.
492                          */
493                         if (port != engine->execlist_port)
494                                 break;
495
496                         /* If GVT overrides us we only ever submit port[0],
497                          * leaving port[1] empty. Note that we also have
498                          * to be careful that we don't queue the same
499                          * context (even though a different request) to
500                          * the second port.
501                          */
502                         if (ctx_single_port_submission(last->ctx) ||
503                             ctx_single_port_submission(cursor->ctx))
504                                 break;
505
506                         GEM_BUG_ON(last->ctx == cursor->ctx);
507
508                         i915_gem_request_assign(&port->request, last);
509                         port++;
510                 }
511
512                 rb = rb_next(rb);
513                 rb_erase(&cursor->priotree.node, &engine->execlist_queue);
514                 RB_CLEAR_NODE(&cursor->priotree.node);
515                 cursor->priotree.priority = INT_MAX;
516
517                 __i915_gem_request_submit(cursor);
518                 last = cursor;
519                 submit = true;
520         }
521         if (submit) {
522                 i915_gem_request_assign(&port->request, last);
523                 engine->execlist_first = rb;
524         }
525         spin_unlock_irqrestore(&engine->timeline->lock, flags);
526
527         if (submit)
528                 execlists_submit_ports(engine);
529 }
530
531 static bool execlists_elsp_idle(struct intel_engine_cs *engine)
532 {
533         return !engine->execlist_port[0].request;
534 }
535
536 /**
537  * intel_execlists_idle() - Determine if all engine submission ports are idle
538  * @dev_priv: i915 device private
539  *
540  * Return true if there are no requests pending on any of the submission ports
541  * of any engines.
542  */
543 bool intel_execlists_idle(struct drm_i915_private *dev_priv)
544 {
545         struct intel_engine_cs *engine;
546         enum intel_engine_id id;
547
548         if (!i915.enable_execlists)
549                 return true;
550
551         for_each_engine(engine, dev_priv, id)
552                 if (!execlists_elsp_idle(engine))
553                         return false;
554
555         return true;
556 }
557
558 static bool execlists_elsp_ready(struct intel_engine_cs *engine)
559 {
560         int port;
561
562         port = 1; /* wait for a free slot */
563         if (engine->disable_lite_restore_wa || engine->preempt_wa)
564                 port = 0; /* wait for GPU to be idle before continuing */
565
566         return !engine->execlist_port[port].request;
567 }
568
569 /*
570  * Check the unread Context Status Buffers and manage the submission of new
571  * contexts to the ELSP accordingly.
572  */
573 static void intel_lrc_irq_handler(unsigned long data)
574 {
575         struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
576         struct execlist_port *port = engine->execlist_port;
577         struct drm_i915_private *dev_priv = engine->i915;
578
579         intel_uncore_forcewake_get(dev_priv, engine->fw_domains);
580
581         if (!execlists_elsp_idle(engine)) {
582                 u32 __iomem *csb_mmio =
583                         dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
584                 u32 __iomem *buf =
585                         dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0));
586                 unsigned int csb, head, tail;
587
588                 csb = readl(csb_mmio);
589                 head = GEN8_CSB_READ_PTR(csb);
590                 tail = GEN8_CSB_WRITE_PTR(csb);
591                 if (tail < head)
592                         tail += GEN8_CSB_ENTRIES;
593                 while (head < tail) {
594                         unsigned int idx = ++head % GEN8_CSB_ENTRIES;
595                         unsigned int status = readl(buf + 2 * idx);
596
597                         if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
598                                 continue;
599
600                         GEM_BUG_ON(port[0].count == 0);
601                         if (--port[0].count == 0) {
602                                 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
603                                 execlists_context_status_change(port[0].request,
604                                                                 INTEL_CONTEXT_SCHEDULE_OUT);
605
606                                 i915_gem_request_put(port[0].request);
607                                 port[0] = port[1];
608                                 memset(&port[1], 0, sizeof(port[1]));
609
610                                 engine->preempt_wa = false;
611                         }
612
613                         GEM_BUG_ON(port[0].count == 0 &&
614                                    !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
615                 }
616
617                 writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK,
618                                      GEN8_CSB_WRITE_PTR(csb) << 8),
619                        csb_mmio);
620         }
621
622         if (execlists_elsp_ready(engine))
623                 execlists_dequeue(engine);
624
625         intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
626 }
627
628 static bool insert_request(struct i915_priotree *pt, struct rb_root *root)
629 {
630         struct rb_node **p, *rb;
631         bool first = true;
632
633         /* most positive priority is scheduled first, equal priorities fifo */
634         rb = NULL;
635         p = &root->rb_node;
636         while (*p) {
637                 struct i915_priotree *pos;
638
639                 rb = *p;
640                 pos = rb_entry(rb, typeof(*pos), node);
641                 if (pt->priority > pos->priority) {
642                         p = &rb->rb_left;
643                 } else {
644                         p = &rb->rb_right;
645                         first = false;
646                 }
647         }
648         rb_link_node(&pt->node, rb, p);
649         rb_insert_color(&pt->node, root);
650
651         return first;
652 }
653
654 static void execlists_submit_request(struct drm_i915_gem_request *request)
655 {
656         struct intel_engine_cs *engine = request->engine;
657         unsigned long flags;
658
659         /* Will be called from irq-context when using foreign fences. */
660         spin_lock_irqsave(&engine->timeline->lock, flags);
661
662         if (insert_request(&request->priotree, &engine->execlist_queue))
663                 engine->execlist_first = &request->priotree.node;
664         if (execlists_elsp_idle(engine))
665                 tasklet_hi_schedule(&engine->irq_tasklet);
666
667         spin_unlock_irqrestore(&engine->timeline->lock, flags);
668 }
669
670 static struct intel_engine_cs *
671 pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
672 {
673         struct intel_engine_cs *engine =
674                 container_of(pt, struct drm_i915_gem_request, priotree)->engine;
675
676         GEM_BUG_ON(!locked);
677
678         if (engine != locked) {
679                 spin_unlock(&locked->timeline->lock);
680                 spin_lock(&engine->timeline->lock);
681         }
682
683         return engine;
684 }
685
686 static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
687 {
688         struct intel_engine_cs *engine;
689         struct i915_dependency *dep, *p;
690         struct i915_dependency stack;
691         LIST_HEAD(dfs);
692
693         if (prio <= READ_ONCE(request->priotree.priority))
694                 return;
695
696         /* Need BKL in order to use the temporary link inside i915_dependency */
697         lockdep_assert_held(&request->i915->drm.struct_mutex);
698
699         stack.signaler = &request->priotree;
700         list_add(&stack.dfs_link, &dfs);
701
702         /* Recursively bump all dependent priorities to match the new request.
703          *
704          * A naive approach would be to use recursion:
705          * static void update_priorities(struct i915_priotree *pt, prio) {
706          *      list_for_each_entry(dep, &pt->signalers_list, signal_link)
707          *              update_priorities(dep->signal, prio)
708          *      insert_request(pt);
709          * }
710          * but that may have unlimited recursion depth and so runs a very
711          * real risk of overunning the kernel stack. Instead, we build
712          * a flat list of all dependencies starting with the current request.
713          * As we walk the list of dependencies, we add all of its dependencies
714          * to the end of the list (this may include an already visited
715          * request) and continue to walk onwards onto the new dependencies. The
716          * end result is a topological list of requests in reverse order, the
717          * last element in the list is the request we must execute first.
718          */
719         list_for_each_entry_safe(dep, p, &dfs, dfs_link) {
720                 struct i915_priotree *pt = dep->signaler;
721
722                 /* Within an engine, there can be no cycle, but we may
723                  * refer to the same dependency chain multiple times
724                  * (redundant dependencies are not eliminated) and across
725                  * engines.
726                  */
727                 list_for_each_entry(p, &pt->signalers_list, signal_link) {
728                         GEM_BUG_ON(p->signaler->priority < pt->priority);
729                         if (prio > READ_ONCE(p->signaler->priority))
730                                 list_move_tail(&p->dfs_link, &dfs);
731                 }
732
733                 list_safe_reset_next(dep, p, dfs_link);
734         }
735
736         engine = request->engine;
737         spin_lock_irq(&engine->timeline->lock);
738
739         /* Fifo and depth-first replacement ensure our deps execute before us */
740         list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
741                 struct i915_priotree *pt = dep->signaler;
742
743                 INIT_LIST_HEAD(&dep->dfs_link);
744
745                 engine = pt_lock_engine(pt, engine);
746
747                 if (prio <= pt->priority)
748                         continue;
749
750                 pt->priority = prio;
751                 if (!RB_EMPTY_NODE(&pt->node)) {
752                         rb_erase(&pt->node, &engine->execlist_queue);
753                         if (insert_request(pt, &engine->execlist_queue))
754                                 engine->execlist_first = &pt->node;
755                 }
756         }
757
758         spin_unlock_irq(&engine->timeline->lock);
759
760         /* XXX Do we need to preempt to make room for us and our deps? */
761 }
762
763 static int execlists_context_pin(struct intel_engine_cs *engine,
764                                  struct i915_gem_context *ctx)
765 {
766         struct intel_context *ce = &ctx->engine[engine->id];
767         unsigned int flags;
768         void *vaddr;
769         int ret;
770
771         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
772
773         if (ce->pin_count++)
774                 return 0;
775
776         if (!ce->state) {
777                 ret = execlists_context_deferred_alloc(ctx, engine);
778                 if (ret)
779                         goto err;
780         }
781         GEM_BUG_ON(!ce->state);
782
783         flags = PIN_GLOBAL;
784         if (ctx->ggtt_offset_bias)
785                 flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
786         if (i915_gem_context_is_kernel(ctx))
787                 flags |= PIN_HIGH;
788
789         ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN, flags);
790         if (ret)
791                 goto err;
792
793         vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
794         if (IS_ERR(vaddr)) {
795                 ret = PTR_ERR(vaddr);
796                 goto unpin_vma;
797         }
798
799         ret = intel_ring_pin(ce->ring, ctx->ggtt_offset_bias);
800         if (ret)
801                 goto unpin_map;
802
803         intel_lr_context_descriptor_update(ctx, engine);
804
805         ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
806         ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
807                 i915_ggtt_offset(ce->ring->vma);
808
809         ce->state->obj->mm.dirty = true;
810
811         i915_gem_context_get(ctx);
812         return 0;
813
814 unpin_map:
815         i915_gem_object_unpin_map(ce->state->obj);
816 unpin_vma:
817         __i915_vma_unpin(ce->state);
818 err:
819         ce->pin_count = 0;
820         return ret;
821 }
822
823 static void execlists_context_unpin(struct intel_engine_cs *engine,
824                                     struct i915_gem_context *ctx)
825 {
826         struct intel_context *ce = &ctx->engine[engine->id];
827
828         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
829         GEM_BUG_ON(ce->pin_count == 0);
830
831         if (--ce->pin_count)
832                 return;
833
834         intel_ring_unpin(ce->ring);
835
836         i915_gem_object_unpin_map(ce->state->obj);
837         i915_vma_unpin(ce->state);
838
839         i915_gem_context_put(ctx);
840 }
841
842 static int execlists_request_alloc(struct drm_i915_gem_request *request)
843 {
844         struct intel_engine_cs *engine = request->engine;
845         struct intel_context *ce = &request->ctx->engine[engine->id];
846         int ret;
847
848         GEM_BUG_ON(!ce->pin_count);
849
850         /* Flush enough space to reduce the likelihood of waiting after
851          * we start building the request - in which case we will just
852          * have to repeat work.
853          */
854         request->reserved_space += EXECLISTS_REQUEST_SIZE;
855
856         GEM_BUG_ON(!ce->ring);
857         request->ring = ce->ring;
858
859         if (i915.enable_guc_submission) {
860                 /*
861                  * Check that the GuC has space for the request before
862                  * going any further, as the i915_add_request() call
863                  * later on mustn't fail ...
864                  */
865                 ret = i915_guc_wq_reserve(request);
866                 if (ret)
867                         goto err;
868         }
869
870         ret = intel_ring_begin(request, 0);
871         if (ret)
872                 goto err_unreserve;
873
874         if (!ce->initialised) {
875                 ret = engine->init_context(request);
876                 if (ret)
877                         goto err_unreserve;
878
879                 ce->initialised = true;
880         }
881
882         /* Note that after this point, we have committed to using
883          * this request as it is being used to both track the
884          * state of engine initialisation and liveness of the
885          * golden renderstate above. Think twice before you try
886          * to cancel/unwind this request now.
887          */
888
889         request->reserved_space -= EXECLISTS_REQUEST_SIZE;
890         return 0;
891
892 err_unreserve:
893         if (i915.enable_guc_submission)
894                 i915_guc_wq_unreserve(request);
895 err:
896         return ret;
897 }
898
899 static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
900 {
901         int ret, i;
902         struct intel_ring *ring = req->ring;
903         struct i915_workarounds *w = &req->i915->workarounds;
904
905         if (w->count == 0)
906                 return 0;
907
908         ret = req->engine->emit_flush(req, EMIT_BARRIER);
909         if (ret)
910                 return ret;
911
912         ret = intel_ring_begin(req, w->count * 2 + 2);
913         if (ret)
914                 return ret;
915
916         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
917         for (i = 0; i < w->count; i++) {
918                 intel_ring_emit_reg(ring, w->reg[i].addr);
919                 intel_ring_emit(ring, w->reg[i].value);
920         }
921         intel_ring_emit(ring, MI_NOOP);
922
923         intel_ring_advance(ring);
924
925         ret = req->engine->emit_flush(req, EMIT_BARRIER);
926         if (ret)
927                 return ret;
928
929         return 0;
930 }
931
932 #define wa_ctx_emit(batch, index, cmd)                                  \
933         do {                                                            \
934                 int __index = (index)++;                                \
935                 if (WARN_ON(__index >= (PAGE_SIZE / sizeof(uint32_t)))) { \
936                         return -ENOSPC;                                 \
937                 }                                                       \
938                 batch[__index] = (cmd);                                 \
939         } while (0)
940
941 #define wa_ctx_emit_reg(batch, index, reg) \
942         wa_ctx_emit((batch), (index), i915_mmio_reg_offset(reg))
943
944 /*
945  * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
946  * PIPE_CONTROL instruction. This is required for the flush to happen correctly
947  * but there is a slight complication as this is applied in WA batch where the
948  * values are only initialized once so we cannot take register value at the
949  * beginning and reuse it further; hence we save its value to memory, upload a
950  * constant value with bit21 set and then we restore it back with the saved value.
951  * To simplify the WA, a constant value is formed by using the default value
952  * of this register. This shouldn't be a problem because we are only modifying
953  * it for a short period and this batch in non-premptible. We can ofcourse
954  * use additional instructions that read the actual value of the register
955  * at that time and set our bit of interest but it makes the WA complicated.
956  *
957  * This WA is also required for Gen9 so extracting as a function avoids
958  * code duplication.
959  */
960 static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
961                                                 uint32_t *batch,
962                                                 uint32_t index)
963 {
964         uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES);
965
966         wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
967                                    MI_SRM_LRM_GLOBAL_GTT));
968         wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
969         wa_ctx_emit(batch, index, i915_ggtt_offset(engine->scratch) + 256);
970         wa_ctx_emit(batch, index, 0);
971
972         wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
973         wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
974         wa_ctx_emit(batch, index, l3sqc4_flush);
975
976         wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
977         wa_ctx_emit(batch, index, (PIPE_CONTROL_CS_STALL |
978                                    PIPE_CONTROL_DC_FLUSH_ENABLE));
979         wa_ctx_emit(batch, index, 0);
980         wa_ctx_emit(batch, index, 0);
981         wa_ctx_emit(batch, index, 0);
982         wa_ctx_emit(batch, index, 0);
983
984         wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8 |
985                                    MI_SRM_LRM_GLOBAL_GTT));
986         wa_ctx_emit_reg(batch, index, GEN8_L3SQCREG4);
987         wa_ctx_emit(batch, index, i915_ggtt_offset(engine->scratch) + 256);
988         wa_ctx_emit(batch, index, 0);
989
990         return index;
991 }
992
993 static inline uint32_t wa_ctx_start(struct i915_wa_ctx_bb *wa_ctx,
994                                     uint32_t offset,
995                                     uint32_t start_alignment)
996 {
997         return wa_ctx->offset = ALIGN(offset, start_alignment);
998 }
999
1000 static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx,
1001                              uint32_t offset,
1002                              uint32_t size_alignment)
1003 {
1004         wa_ctx->size = offset - wa_ctx->offset;
1005
1006         WARN(wa_ctx->size % size_alignment,
1007              "wa_ctx_bb failed sanity checks: size %d is not aligned to %d\n",
1008              wa_ctx->size, size_alignment);
1009         return 0;
1010 }
1011
1012 /*
1013  * Typically we only have one indirect_ctx and per_ctx batch buffer which are
1014  * initialized at the beginning and shared across all contexts but this field
1015  * helps us to have multiple batches at different offsets and select them based
1016  * on a criteria. At the moment this batch always start at the beginning of the page
1017  * and at this point we don't have multiple wa_ctx batch buffers.
1018  *
1019  * The number of WA applied are not known at the beginning; we use this field
1020  * to return the no of DWORDS written.
1021  *
1022  * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
1023  * so it adds NOOPs as padding to make it cacheline aligned.
1024  * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
1025  * makes a complete batch buffer.
1026  */
1027 static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
1028                                     struct i915_wa_ctx_bb *wa_ctx,
1029                                     uint32_t *batch,
1030                                     uint32_t *offset)
1031 {
1032         uint32_t scratch_addr;
1033         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1034
1035         /* WaDisableCtxRestoreArbitration:bdw,chv */
1036         wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
1037
1038         /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
1039         if (IS_BROADWELL(engine->i915)) {
1040                 int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index);
1041                 if (rc < 0)
1042                         return rc;
1043                 index = rc;
1044         }
1045
1046         /* WaClearSlmSpaceAtContextSwitch:bdw,chv */
1047         /* Actual scratch location is at 128 bytes offset */
1048         scratch_addr = i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
1049
1050         wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
1051         wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 |
1052                                    PIPE_CONTROL_GLOBAL_GTT_IVB |
1053                                    PIPE_CONTROL_CS_STALL |
1054                                    PIPE_CONTROL_QW_WRITE));
1055         wa_ctx_emit(batch, index, scratch_addr);
1056         wa_ctx_emit(batch, index, 0);
1057         wa_ctx_emit(batch, index, 0);
1058         wa_ctx_emit(batch, index, 0);
1059
1060         /* Pad to end of cacheline */
1061         while (index % CACHELINE_DWORDS)
1062                 wa_ctx_emit(batch, index, MI_NOOP);
1063
1064         /*
1065          * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because
1066          * execution depends on the length specified in terms of cache lines
1067          * in the register CTX_RCS_INDIRECT_CTX
1068          */
1069
1070         return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS);
1071 }
1072
1073 /*
1074  *  This batch is started immediately after indirect_ctx batch. Since we ensure
1075  *  that indirect_ctx ends on a cacheline this batch is aligned automatically.
1076  *
1077  *  The number of DWORDS written are returned using this field.
1078  *
1079  *  This batch is terminated with MI_BATCH_BUFFER_END and so we need not add padding
1080  *  to align it with cacheline as padding after MI_BATCH_BUFFER_END is redundant.
1081  */
1082 static int gen8_init_perctx_bb(struct intel_engine_cs *engine,
1083                                struct i915_wa_ctx_bb *wa_ctx,
1084                                uint32_t *batch,
1085                                uint32_t *offset)
1086 {
1087         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1088
1089         /* WaDisableCtxRestoreArbitration:bdw,chv */
1090         wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
1091
1092         wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
1093
1094         return wa_ctx_end(wa_ctx, *offset = index, 1);
1095 }
1096
1097 static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
1098                                     struct i915_wa_ctx_bb *wa_ctx,
1099                                     uint32_t *batch,
1100                                     uint32_t *offset)
1101 {
1102         int ret;
1103         struct drm_i915_private *dev_priv = engine->i915;
1104         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1105
1106         /* WaDisableCtxRestoreArbitration:bxt */
1107         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1108                 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
1109
1110         /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
1111         ret = gen8_emit_flush_coherentl3_wa(engine, batch, index);
1112         if (ret < 0)
1113                 return ret;
1114         index = ret;
1115
1116         /* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl */
1117         wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
1118         wa_ctx_emit_reg(batch, index, COMMON_SLICE_CHICKEN2);
1119         wa_ctx_emit(batch, index, _MASKED_BIT_DISABLE(
1120                             GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE));
1121         wa_ctx_emit(batch, index, MI_NOOP);
1122
1123         /* WaClearSlmSpaceAtContextSwitch:kbl */
1124         /* Actual scratch location is at 128 bytes offset */
1125         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0)) {
1126                 u32 scratch_addr =
1127                         i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
1128
1129                 wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
1130                 wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 |
1131                                            PIPE_CONTROL_GLOBAL_GTT_IVB |
1132                                            PIPE_CONTROL_CS_STALL |
1133                                            PIPE_CONTROL_QW_WRITE));
1134                 wa_ctx_emit(batch, index, scratch_addr);
1135                 wa_ctx_emit(batch, index, 0);
1136                 wa_ctx_emit(batch, index, 0);
1137                 wa_ctx_emit(batch, index, 0);
1138         }
1139
1140         /* WaMediaPoolStateCmdInWABB:bxt */
1141         if (HAS_POOLED_EU(engine->i915)) {
1142                 /*
1143                  * EU pool configuration is setup along with golden context
1144                  * during context initialization. This value depends on
1145                  * device type (2x6 or 3x6) and needs to be updated based
1146                  * on which subslice is disabled especially for 2x6
1147                  * devices, however it is safe to load default
1148                  * configuration of 3x6 device instead of masking off
1149                  * corresponding bits because HW ignores bits of a disabled
1150                  * subslice and drops down to appropriate config. Please
1151                  * see render_state_setup() in i915_gem_render_state.c for
1152                  * possible configurations, to avoid duplication they are
1153                  * not shown here again.
1154                  */
1155                 u32 eu_pool_config = 0x00777000;
1156                 wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_STATE);
1157                 wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_ENABLE);
1158                 wa_ctx_emit(batch, index, eu_pool_config);
1159                 wa_ctx_emit(batch, index, 0);
1160                 wa_ctx_emit(batch, index, 0);
1161                 wa_ctx_emit(batch, index, 0);
1162         }
1163
1164         /* Pad to end of cacheline */
1165         while (index % CACHELINE_DWORDS)
1166                 wa_ctx_emit(batch, index, MI_NOOP);
1167
1168         return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS);
1169 }
1170
1171 static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
1172                                struct i915_wa_ctx_bb *wa_ctx,
1173                                uint32_t *batch,
1174                                uint32_t *offset)
1175 {
1176         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1177
1178         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
1179         if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
1180                 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
1181                 wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
1182                 wa_ctx_emit(batch, index,
1183                             _MASKED_BIT_ENABLE(DISABLE_PIXEL_MASK_CAMMING));
1184                 wa_ctx_emit(batch, index, MI_NOOP);
1185         }
1186
1187         /* WaClearTdlStateAckDirtyBits:bxt */
1188         if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_B0)) {
1189                 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4));
1190
1191                 wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK);
1192                 wa_ctx_emit(batch, index, _MASKED_BIT_DISABLE(GEN9_SUBSLICE_TDL_ACK_BITS));
1193
1194                 wa_ctx_emit_reg(batch, index, GEN9_STATE_ACK_SLICE1);
1195                 wa_ctx_emit(batch, index, _MASKED_BIT_DISABLE(GEN9_SUBSLICE_TDL_ACK_BITS));
1196
1197                 wa_ctx_emit_reg(batch, index, GEN9_STATE_ACK_SLICE2);
1198                 wa_ctx_emit(batch, index, _MASKED_BIT_DISABLE(GEN9_SUBSLICE_TDL_ACK_BITS));
1199
1200                 wa_ctx_emit_reg(batch, index, GEN7_ROW_CHICKEN2);
1201                 /* dummy write to CS, mask bits are 0 to ensure the register is not modified */
1202                 wa_ctx_emit(batch, index, 0x0);
1203                 wa_ctx_emit(batch, index, MI_NOOP);
1204         }
1205
1206         /* WaDisableCtxRestoreArbitration:bxt */
1207         if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
1208                 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
1209
1210         wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
1211
1212         return wa_ctx_end(wa_ctx, *offset = index, 1);
1213 }
1214
1215 static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *engine, u32 size)
1216 {
1217         struct drm_i915_gem_object *obj;
1218         struct i915_vma *vma;
1219         int err;
1220
1221         obj = i915_gem_object_create(engine->i915, PAGE_ALIGN(size));
1222         if (IS_ERR(obj))
1223                 return PTR_ERR(obj);
1224
1225         vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
1226         if (IS_ERR(vma)) {
1227                 err = PTR_ERR(vma);
1228                 goto err;
1229         }
1230
1231         err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH);
1232         if (err)
1233                 goto err;
1234
1235         engine->wa_ctx.vma = vma;
1236         return 0;
1237
1238 err:
1239         i915_gem_object_put(obj);
1240         return err;
1241 }
1242
1243 static void lrc_destroy_wa_ctx_obj(struct intel_engine_cs *engine)
1244 {
1245         i915_vma_unpin_and_release(&engine->wa_ctx.vma);
1246 }
1247
1248 static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1249 {
1250         struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1251         uint32_t *batch;
1252         uint32_t offset;
1253         struct page *page;
1254         int ret;
1255
1256         WARN_ON(engine->id != RCS);
1257
1258         /* update this when WA for higher Gen are added */
1259         if (INTEL_GEN(engine->i915) > 9) {
1260                 DRM_ERROR("WA batch buffer is not initialized for Gen%d\n",
1261                           INTEL_GEN(engine->i915));
1262                 return 0;
1263         }
1264
1265         /* some WA perform writes to scratch page, ensure it is valid */
1266         if (!engine->scratch) {
1267                 DRM_ERROR("scratch page not allocated for %s\n", engine->name);
1268                 return -EINVAL;
1269         }
1270
1271         ret = lrc_setup_wa_ctx_obj(engine, PAGE_SIZE);
1272         if (ret) {
1273                 DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
1274                 return ret;
1275         }
1276
1277         page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
1278         batch = kmap_atomic(page);
1279         offset = 0;
1280
1281         if (IS_GEN8(engine->i915)) {
1282                 ret = gen8_init_indirectctx_bb(engine,
1283                                                &wa_ctx->indirect_ctx,
1284                                                batch,
1285                                                &offset);
1286                 if (ret)
1287                         goto out;
1288
1289                 ret = gen8_init_perctx_bb(engine,
1290                                           &wa_ctx->per_ctx,
1291                                           batch,
1292                                           &offset);
1293                 if (ret)
1294                         goto out;
1295         } else if (IS_GEN9(engine->i915)) {
1296                 ret = gen9_init_indirectctx_bb(engine,
1297                                                &wa_ctx->indirect_ctx,
1298                                                batch,
1299                                                &offset);
1300                 if (ret)
1301                         goto out;
1302
1303                 ret = gen9_init_perctx_bb(engine,
1304                                           &wa_ctx->per_ctx,
1305                                           batch,
1306                                           &offset);
1307                 if (ret)
1308                         goto out;
1309         }
1310
1311 out:
1312         kunmap_atomic(batch);
1313         if (ret)
1314                 lrc_destroy_wa_ctx_obj(engine);
1315
1316         return ret;
1317 }
1318
1319 static int gen8_init_common_ring(struct intel_engine_cs *engine)
1320 {
1321         struct drm_i915_private *dev_priv = engine->i915;
1322         int ret;
1323
1324         ret = intel_mocs_init_engine(engine);
1325         if (ret)
1326                 return ret;
1327
1328         intel_engine_reset_breadcrumbs(engine);
1329         intel_engine_init_hangcheck(engine);
1330
1331         I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
1332         I915_WRITE(RING_MODE_GEN7(engine),
1333                    _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
1334                    _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
1335         I915_WRITE(RING_HWS_PGA(engine->mmio_base),
1336                    engine->status_page.ggtt_offset);
1337         POSTING_READ(RING_HWS_PGA(engine->mmio_base));
1338
1339         DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name);
1340
1341         /* After a GPU reset, we may have requests to replay */
1342         if (!execlists_elsp_idle(engine)) {
1343                 engine->execlist_port[0].count = 0;
1344                 engine->execlist_port[1].count = 0;
1345                 execlists_submit_ports(engine);
1346         }
1347
1348         return 0;
1349 }
1350
1351 static int gen8_init_render_ring(struct intel_engine_cs *engine)
1352 {
1353         struct drm_i915_private *dev_priv = engine->i915;
1354         int ret;
1355
1356         ret = gen8_init_common_ring(engine);
1357         if (ret)
1358                 return ret;
1359
1360         /* We need to disable the AsyncFlip performance optimisations in order
1361          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1362          * programmed to '1' on all products.
1363          *
1364          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
1365          */
1366         I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1367
1368         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1369
1370         return init_workarounds_ring(engine);
1371 }
1372
1373 static int gen9_init_render_ring(struct intel_engine_cs *engine)
1374 {
1375         int ret;
1376
1377         ret = gen8_init_common_ring(engine);
1378         if (ret)
1379                 return ret;
1380
1381         return init_workarounds_ring(engine);
1382 }
1383
1384 static void reset_common_ring(struct intel_engine_cs *engine,
1385                               struct drm_i915_gem_request *request)
1386 {
1387         struct drm_i915_private *dev_priv = engine->i915;
1388         struct execlist_port *port = engine->execlist_port;
1389         struct intel_context *ce;
1390
1391         /* If the request was innocent, we leave the request in the ELSP
1392          * and will try to replay it on restarting. The context image may
1393          * have been corrupted by the reset, in which case we may have
1394          * to service a new GPU hang, but more likely we can continue on
1395          * without impact.
1396          *
1397          * If the request was guilty, we presume the context is corrupt
1398          * and have to at least restore the RING register in the context
1399          * image back to the expected values to skip over the guilty request.
1400          */
1401         if (!request || request->fence.error != -EIO)
1402                 return;
1403
1404         /* We want a simple context + ring to execute the breadcrumb update.
1405          * We cannot rely on the context being intact across the GPU hang,
1406          * so clear it and rebuild just what we need for the breadcrumb.
1407          * All pending requests for this context will be zapped, and any
1408          * future request will be after userspace has had the opportunity
1409          * to recreate its own state.
1410          */
1411         ce = &request->ctx->engine[engine->id];
1412         execlists_init_reg_state(ce->lrc_reg_state,
1413                                  request->ctx, engine, ce->ring);
1414
1415         /* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
1416         ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
1417                 i915_ggtt_offset(ce->ring->vma);
1418         ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
1419
1420         request->ring->head = request->postfix;
1421         request->ring->last_retired_head = -1;
1422         intel_ring_update_space(request->ring);
1423
1424         if (i915.enable_guc_submission)
1425                 return;
1426
1427         /* Catch up with any missed context-switch interrupts */
1428         I915_WRITE(RING_CONTEXT_STATUS_PTR(engine), _MASKED_FIELD(0xffff, 0));
1429         if (request->ctx != port[0].request->ctx) {
1430                 i915_gem_request_put(port[0].request);
1431                 port[0] = port[1];
1432                 memset(&port[1], 0, sizeof(port[1]));
1433         }
1434
1435         GEM_BUG_ON(request->ctx != port[0].request->ctx);
1436
1437         /* Reset WaIdleLiteRestore:bdw,skl as well */
1438         request->tail =
1439                 intel_ring_wrap(request->ring,
1440                                 request->wa_tail - WA_TAIL_DWORDS*sizeof(u32));
1441 }
1442
1443 static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
1444 {
1445         struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt;
1446         struct intel_ring *ring = req->ring;
1447         struct intel_engine_cs *engine = req->engine;
1448         const int num_lri_cmds = GEN8_LEGACY_PDPES * 2;
1449         int i, ret;
1450
1451         ret = intel_ring_begin(req, num_lri_cmds * 2 + 2);
1452         if (ret)
1453                 return ret;
1454
1455         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(num_lri_cmds));
1456         for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
1457                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
1458
1459                 intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(engine, i));
1460                 intel_ring_emit(ring, upper_32_bits(pd_daddr));
1461                 intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(engine, i));
1462                 intel_ring_emit(ring, lower_32_bits(pd_daddr));
1463         }
1464
1465         intel_ring_emit(ring, MI_NOOP);
1466         intel_ring_advance(ring);
1467
1468         return 0;
1469 }
1470
1471 static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1472                               u64 offset, u32 len,
1473                               unsigned int dispatch_flags)
1474 {
1475         struct intel_ring *ring = req->ring;
1476         bool ppgtt = !(dispatch_flags & I915_DISPATCH_SECURE);
1477         int ret;
1478
1479         /* Don't rely in hw updating PDPs, specially in lite-restore.
1480          * Ideally, we should set Force PD Restore in ctx descriptor,
1481          * but we can't. Force Restore would be a second option, but
1482          * it is unsafe in case of lite-restore (because the ctx is
1483          * not idle). PML4 is allocated during ppgtt init so this is
1484          * not needed in 48-bit.*/
1485         if (req->ctx->ppgtt &&
1486             (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) {
1487                 if (!USES_FULL_48BIT_PPGTT(req->i915) &&
1488                     !intel_vgpu_active(req->i915)) {
1489                         ret = intel_logical_ring_emit_pdps(req);
1490                         if (ret)
1491                                 return ret;
1492                 }
1493
1494                 req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine);
1495         }
1496
1497         ret = intel_ring_begin(req, 4);
1498         if (ret)
1499                 return ret;
1500
1501         /* FIXME(BDW): Address space and security selectors. */
1502         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 |
1503                         (ppgtt<<8) |
1504                         (dispatch_flags & I915_DISPATCH_RS ?
1505                          MI_BATCH_RESOURCE_STREAMER : 0));
1506         intel_ring_emit(ring, lower_32_bits(offset));
1507         intel_ring_emit(ring, upper_32_bits(offset));
1508         intel_ring_emit(ring, MI_NOOP);
1509         intel_ring_advance(ring);
1510
1511         return 0;
1512 }
1513
1514 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
1515 {
1516         struct drm_i915_private *dev_priv = engine->i915;
1517         I915_WRITE_IMR(engine,
1518                        ~(engine->irq_enable_mask | engine->irq_keep_mask));
1519         POSTING_READ_FW(RING_IMR(engine->mmio_base));
1520 }
1521
1522 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
1523 {
1524         struct drm_i915_private *dev_priv = engine->i915;
1525         I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1526 }
1527
1528 static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode)
1529 {
1530         struct intel_ring *ring = request->ring;
1531         u32 cmd;
1532         int ret;
1533
1534         ret = intel_ring_begin(request, 4);
1535         if (ret)
1536                 return ret;
1537
1538         cmd = MI_FLUSH_DW + 1;
1539
1540         /* We always require a command barrier so that subsequent
1541          * commands, such as breadcrumb interrupts, are strictly ordered
1542          * wrt the contents of the write cache being flushed to memory
1543          * (and thus being coherent from the CPU).
1544          */
1545         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1546
1547         if (mode & EMIT_INVALIDATE) {
1548                 cmd |= MI_INVALIDATE_TLB;
1549                 if (request->engine->id == VCS)
1550                         cmd |= MI_INVALIDATE_BSD;
1551         }
1552
1553         intel_ring_emit(ring, cmd);
1554         intel_ring_emit(ring,
1555                         I915_GEM_HWS_SCRATCH_ADDR |
1556                         MI_FLUSH_DW_USE_GTT);
1557         intel_ring_emit(ring, 0); /* upper addr */
1558         intel_ring_emit(ring, 0); /* value */
1559         intel_ring_advance(ring);
1560
1561         return 0;
1562 }
1563
1564 static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
1565                                   u32 mode)
1566 {
1567         struct intel_ring *ring = request->ring;
1568         struct intel_engine_cs *engine = request->engine;
1569         u32 scratch_addr =
1570                 i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
1571         bool vf_flush_wa = false, dc_flush_wa = false;
1572         u32 flags = 0;
1573         int ret;
1574         int len;
1575
1576         flags |= PIPE_CONTROL_CS_STALL;
1577
1578         if (mode & EMIT_FLUSH) {
1579                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
1580                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1581                 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
1582                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
1583         }
1584
1585         if (mode & EMIT_INVALIDATE) {
1586                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
1587                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
1588                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1589                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1590                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
1591                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
1592                 flags |= PIPE_CONTROL_QW_WRITE;
1593                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
1594
1595                 /*
1596                  * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
1597                  * pipe control.
1598                  */
1599                 if (IS_GEN9(request->i915))
1600                         vf_flush_wa = true;
1601
1602                 /* WaForGAMHang:kbl */
1603                 if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0))
1604                         dc_flush_wa = true;
1605         }
1606
1607         len = 6;
1608
1609         if (vf_flush_wa)
1610                 len += 6;
1611
1612         if (dc_flush_wa)
1613                 len += 12;
1614
1615         ret = intel_ring_begin(request, len);
1616         if (ret)
1617                 return ret;
1618
1619         if (vf_flush_wa) {
1620                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
1621                 intel_ring_emit(ring, 0);
1622                 intel_ring_emit(ring, 0);
1623                 intel_ring_emit(ring, 0);
1624                 intel_ring_emit(ring, 0);
1625                 intel_ring_emit(ring, 0);
1626         }
1627
1628         if (dc_flush_wa) {
1629                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
1630                 intel_ring_emit(ring, PIPE_CONTROL_DC_FLUSH_ENABLE);
1631                 intel_ring_emit(ring, 0);
1632                 intel_ring_emit(ring, 0);
1633                 intel_ring_emit(ring, 0);
1634                 intel_ring_emit(ring, 0);
1635         }
1636
1637         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
1638         intel_ring_emit(ring, flags);
1639         intel_ring_emit(ring, scratch_addr);
1640         intel_ring_emit(ring, 0);
1641         intel_ring_emit(ring, 0);
1642         intel_ring_emit(ring, 0);
1643
1644         if (dc_flush_wa) {
1645                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
1646                 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL);
1647                 intel_ring_emit(ring, 0);
1648                 intel_ring_emit(ring, 0);
1649                 intel_ring_emit(ring, 0);
1650                 intel_ring_emit(ring, 0);
1651         }
1652
1653         intel_ring_advance(ring);
1654
1655         return 0;
1656 }
1657
1658 static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
1659 {
1660         /*
1661          * On BXT A steppings there is a HW coherency issue whereby the
1662          * MI_STORE_DATA_IMM storing the completed request's seqno
1663          * occasionally doesn't invalidate the CPU cache. Work around this by
1664          * clflushing the corresponding cacheline whenever the caller wants
1665          * the coherency to be guaranteed. Note that this cacheline is known
1666          * to be clean at this point, since we only write it in
1667          * bxt_a_set_seqno(), where we also do a clflush after the write. So
1668          * this clflush in practice becomes an invalidate operation.
1669          */
1670         intel_flush_status_page(engine, I915_GEM_HWS_INDEX);
1671 }
1672
1673 /*
1674  * Reserve space for 2 NOOPs at the end of each request to be
1675  * used as a workaround for not being allowed to do lite
1676  * restore with HEAD==TAIL (WaIdleLiteRestore).
1677  */
1678 static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *out)
1679 {
1680         *out++ = MI_NOOP;
1681         *out++ = MI_NOOP;
1682         request->wa_tail = intel_ring_offset(request->ring, out);
1683 }
1684
1685 static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request,
1686                                  u32 *out)
1687 {
1688         /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
1689         BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
1690
1691         *out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
1692         *out++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT;
1693         *out++ = 0;
1694         *out++ = request->global_seqno;
1695         *out++ = MI_USER_INTERRUPT;
1696         *out++ = MI_NOOP;
1697         request->tail = intel_ring_offset(request->ring, out);
1698
1699         gen8_emit_wa_tail(request, out);
1700 }
1701
1702 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
1703
1704 static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
1705                                         u32 *out)
1706 {
1707         /* We're using qword write, seqno should be aligned to 8 bytes. */
1708         BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
1709
1710         /* w/a for post sync ops following a GPGPU operation we
1711          * need a prior CS_STALL, which is emitted by the flush
1712          * following the batch.
1713          */
1714         *out++ = GFX_OP_PIPE_CONTROL(6);
1715         *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
1716                   PIPE_CONTROL_CS_STALL |
1717                   PIPE_CONTROL_QW_WRITE);
1718         *out++ = intel_hws_seqno_address(request->engine);
1719         *out++ = 0;
1720         *out++ = request->global_seqno;
1721         /* We're thrashing one dword of HWS. */
1722         *out++ = 0;
1723         *out++ = MI_USER_INTERRUPT;
1724         *out++ = MI_NOOP;
1725         request->tail = intel_ring_offset(request->ring, out);
1726
1727         gen8_emit_wa_tail(request, out);
1728 }
1729
1730 static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS;
1731
1732 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
1733 {
1734         int ret;
1735
1736         ret = intel_logical_ring_workarounds_emit(req);
1737         if (ret)
1738                 return ret;
1739
1740         ret = intel_rcs_context_init_mocs(req);
1741         /*
1742          * Failing to program the MOCS is non-fatal.The system will not
1743          * run at peak performance. So generate an error and carry on.
1744          */
1745         if (ret)
1746                 DRM_ERROR("MOCS failed to program: expect performance issues.\n");
1747
1748         return i915_gem_render_state_emit(req);
1749 }
1750
1751 /**
1752  * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
1753  * @engine: Engine Command Streamer.
1754  */
1755 void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
1756 {
1757         struct drm_i915_private *dev_priv;
1758
1759         /*
1760          * Tasklet cannot be active at this point due intel_mark_active/idle
1761          * so this is just for documentation.
1762          */
1763         if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state)))
1764                 tasklet_kill(&engine->irq_tasklet);
1765
1766         dev_priv = engine->i915;
1767
1768         if (engine->buffer) {
1769                 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
1770         }
1771
1772         if (engine->cleanup)
1773                 engine->cleanup(engine);
1774
1775         if (engine->status_page.vma) {
1776                 i915_gem_object_unpin_map(engine->status_page.vma->obj);
1777                 engine->status_page.vma = NULL;
1778         }
1779
1780         intel_engine_cleanup_common(engine);
1781
1782         lrc_destroy_wa_ctx_obj(engine);
1783         engine->i915 = NULL;
1784         dev_priv->engine[engine->id] = NULL;
1785         kfree(engine);
1786 }
1787
1788 void intel_execlists_enable_submission(struct drm_i915_private *dev_priv)
1789 {
1790         struct intel_engine_cs *engine;
1791         enum intel_engine_id id;
1792
1793         for_each_engine(engine, dev_priv, id) {
1794                 engine->submit_request = execlists_submit_request;
1795                 engine->schedule = execlists_schedule;
1796         }
1797 }
1798
1799 static void
1800 logical_ring_default_vfuncs(struct intel_engine_cs *engine)
1801 {
1802         /* Default vfuncs which can be overriden by each engine. */
1803         engine->init_hw = gen8_init_common_ring;
1804         engine->reset_hw = reset_common_ring;
1805
1806         engine->context_pin = execlists_context_pin;
1807         engine->context_unpin = execlists_context_unpin;
1808
1809         engine->request_alloc = execlists_request_alloc;
1810
1811         engine->emit_flush = gen8_emit_flush;
1812         engine->emit_breadcrumb = gen8_emit_breadcrumb;
1813         engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
1814         engine->submit_request = execlists_submit_request;
1815         engine->schedule = execlists_schedule;
1816
1817         engine->irq_enable = gen8_logical_ring_enable_irq;
1818         engine->irq_disable = gen8_logical_ring_disable_irq;
1819         engine->emit_bb_start = gen8_emit_bb_start;
1820         if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
1821                 engine->irq_seqno_barrier = bxt_a_seqno_barrier;
1822 }
1823
1824 static inline void
1825 logical_ring_default_irqs(struct intel_engine_cs *engine)
1826 {
1827         unsigned shift = engine->irq_shift;
1828         engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
1829         engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
1830 }
1831
1832 static int
1833 lrc_setup_hws(struct intel_engine_cs *engine, struct i915_vma *vma)
1834 {
1835         const int hws_offset = LRC_PPHWSP_PN * PAGE_SIZE;
1836         void *hws;
1837
1838         /* The HWSP is part of the default context object in LRC mode. */
1839         hws = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1840         if (IS_ERR(hws))
1841                 return PTR_ERR(hws);
1842
1843         engine->status_page.page_addr = hws + hws_offset;
1844         engine->status_page.ggtt_offset = i915_ggtt_offset(vma) + hws_offset;
1845         engine->status_page.vma = vma;
1846
1847         return 0;
1848 }
1849
1850 static void
1851 logical_ring_setup(struct intel_engine_cs *engine)
1852 {
1853         struct drm_i915_private *dev_priv = engine->i915;
1854         enum forcewake_domains fw_domains;
1855
1856         intel_engine_setup_common(engine);
1857
1858         /* Intentionally left blank. */
1859         engine->buffer = NULL;
1860
1861         fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
1862                                                     RING_ELSP(engine),
1863                                                     FW_REG_WRITE);
1864
1865         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
1866                                                      RING_CONTEXT_STATUS_PTR(engine),
1867                                                      FW_REG_READ | FW_REG_WRITE);
1868
1869         fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
1870                                                      RING_CONTEXT_STATUS_BUF_BASE(engine),
1871                                                      FW_REG_READ);
1872
1873         engine->fw_domains = fw_domains;
1874
1875         tasklet_init(&engine->irq_tasklet,
1876                      intel_lrc_irq_handler, (unsigned long)engine);
1877
1878         logical_ring_init_platform_invariants(engine);
1879         logical_ring_default_vfuncs(engine);
1880         logical_ring_default_irqs(engine);
1881 }
1882
1883 static int
1884 logical_ring_init(struct intel_engine_cs *engine)
1885 {
1886         struct i915_gem_context *dctx = engine->i915->kernel_context;
1887         int ret;
1888
1889         ret = intel_engine_init_common(engine);
1890         if (ret)
1891                 goto error;
1892
1893         /* And setup the hardware status page. */
1894         ret = lrc_setup_hws(engine, dctx->engine[engine->id].state);
1895         if (ret) {
1896                 DRM_ERROR("Failed to set up hws %s: %d\n", engine->name, ret);
1897                 goto error;
1898         }
1899
1900         return 0;
1901
1902 error:
1903         intel_logical_ring_cleanup(engine);
1904         return ret;
1905 }
1906
1907 int logical_render_ring_init(struct intel_engine_cs *engine)
1908 {
1909         struct drm_i915_private *dev_priv = engine->i915;
1910         int ret;
1911
1912         logical_ring_setup(engine);
1913
1914         if (HAS_L3_DPF(dev_priv))
1915                 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1916
1917         /* Override some for render ring. */
1918         if (INTEL_GEN(dev_priv) >= 9)
1919                 engine->init_hw = gen9_init_render_ring;
1920         else
1921                 engine->init_hw = gen8_init_render_ring;
1922         engine->init_context = gen8_init_rcs_context;
1923         engine->emit_flush = gen8_emit_flush_render;
1924         engine->emit_breadcrumb = gen8_emit_breadcrumb_render;
1925         engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz;
1926
1927         ret = intel_engine_create_scratch(engine, PAGE_SIZE);
1928         if (ret)
1929                 return ret;
1930
1931         ret = intel_init_workaround_bb(engine);
1932         if (ret) {
1933                 /*
1934                  * We continue even if we fail to initialize WA batch
1935                  * because we only expect rare glitches but nothing
1936                  * critical to prevent us from using GPU
1937                  */
1938                 DRM_ERROR("WA batch buffer initialization failed: %d\n",
1939                           ret);
1940         }
1941
1942         return logical_ring_init(engine);
1943 }
1944
1945 int logical_xcs_ring_init(struct intel_engine_cs *engine)
1946 {
1947         logical_ring_setup(engine);
1948
1949         return logical_ring_init(engine);
1950 }
1951
1952 static u32
1953 make_rpcs(struct drm_i915_private *dev_priv)
1954 {
1955         u32 rpcs = 0;
1956
1957         /*
1958          * No explicit RPCS request is needed to ensure full
1959          * slice/subslice/EU enablement prior to Gen9.
1960         */
1961         if (INTEL_GEN(dev_priv) < 9)
1962                 return 0;
1963
1964         /*
1965          * Starting in Gen9, render power gating can leave
1966          * slice/subslice/EU in a partially enabled state. We
1967          * must make an explicit request through RPCS for full
1968          * enablement.
1969         */
1970         if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) {
1971                 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
1972                 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) <<
1973                         GEN8_RPCS_S_CNT_SHIFT;
1974                 rpcs |= GEN8_RPCS_ENABLE;
1975         }
1976
1977         if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
1978                 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
1979                 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) <<
1980                         GEN8_RPCS_SS_CNT_SHIFT;
1981                 rpcs |= GEN8_RPCS_ENABLE;
1982         }
1983
1984         if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) {
1985                 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
1986                         GEN8_RPCS_EU_MIN_SHIFT;
1987                 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
1988                         GEN8_RPCS_EU_MAX_SHIFT;
1989                 rpcs |= GEN8_RPCS_ENABLE;
1990         }
1991
1992         return rpcs;
1993 }
1994
1995 static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
1996 {
1997         u32 indirect_ctx_offset;
1998
1999         switch (INTEL_GEN(engine->i915)) {
2000         default:
2001                 MISSING_CASE(INTEL_GEN(engine->i915));
2002                 /* fall through */
2003         case 9:
2004                 indirect_ctx_offset =
2005                         GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2006                 break;
2007         case 8:
2008                 indirect_ctx_offset =
2009                         GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2010                 break;
2011         }
2012
2013         return indirect_ctx_offset;
2014 }
2015
2016 static void execlists_init_reg_state(u32 *reg_state,
2017                                      struct i915_gem_context *ctx,
2018                                      struct intel_engine_cs *engine,
2019                                      struct intel_ring *ring)
2020 {
2021         struct drm_i915_private *dev_priv = engine->i915;
2022         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
2023
2024         /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
2025          * commands followed by (reg, value) pairs. The values we are setting here are
2026          * only for the first context restore: on a subsequent save, the GPU will
2027          * recreate this batchbuffer with new values (including all the missing
2028          * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
2029         reg_state[CTX_LRI_HEADER_0] =
2030                 MI_LOAD_REGISTER_IMM(engine->id == RCS ? 14 : 11) | MI_LRI_FORCE_POSTED;
2031         ASSIGN_CTX_REG(reg_state, CTX_CONTEXT_CONTROL,
2032                        RING_CONTEXT_CONTROL(engine),
2033                        _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
2034                                           CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
2035                                           (HAS_RESOURCE_STREAMER(dev_priv) ?
2036                                            CTX_CTRL_RS_CTX_ENABLE : 0)));
2037         ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base),
2038                        0);
2039         ASSIGN_CTX_REG(reg_state, CTX_RING_TAIL, RING_TAIL(engine->mmio_base),
2040                        0);
2041         ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_START,
2042                        RING_START(engine->mmio_base), 0);
2043         ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_CONTROL,
2044                        RING_CTL(engine->mmio_base),
2045                        RING_CTL_SIZE(ring->size) | RING_VALID);
2046         ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_U,
2047                        RING_BBADDR_UDW(engine->mmio_base), 0);
2048         ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_L,
2049                        RING_BBADDR(engine->mmio_base), 0);
2050         ASSIGN_CTX_REG(reg_state, CTX_BB_STATE,
2051                        RING_BBSTATE(engine->mmio_base),
2052                        RING_BB_PPGTT);
2053         ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_HEAD_U,
2054                        RING_SBBADDR_UDW(engine->mmio_base), 0);
2055         ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_HEAD_L,
2056                        RING_SBBADDR(engine->mmio_base), 0);
2057         ASSIGN_CTX_REG(reg_state, CTX_SECOND_BB_STATE,
2058                        RING_SBBSTATE(engine->mmio_base), 0);
2059         if (engine->id == RCS) {
2060                 ASSIGN_CTX_REG(reg_state, CTX_BB_PER_CTX_PTR,
2061                                RING_BB_PER_CTX_PTR(engine->mmio_base), 0);
2062                 ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX,
2063                                RING_INDIRECT_CTX(engine->mmio_base), 0);
2064                 ASSIGN_CTX_REG(reg_state, CTX_RCS_INDIRECT_CTX_OFFSET,
2065                                RING_INDIRECT_CTX_OFFSET(engine->mmio_base), 0);
2066                 if (engine->wa_ctx.vma) {
2067                         struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
2068                         u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2069
2070                         reg_state[CTX_RCS_INDIRECT_CTX+1] =
2071                                 (ggtt_offset + wa_ctx->indirect_ctx.offset * sizeof(uint32_t)) |
2072                                 (wa_ctx->indirect_ctx.size / CACHELINE_DWORDS);
2073
2074                         reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] =
2075                                 intel_lr_indirect_ctx_offset(engine) << 6;
2076
2077                         reg_state[CTX_BB_PER_CTX_PTR+1] =
2078                                 (ggtt_offset + wa_ctx->per_ctx.offset * sizeof(uint32_t)) |
2079                                 0x01;
2080                 }
2081         }
2082         reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;
2083         ASSIGN_CTX_REG(reg_state, CTX_CTX_TIMESTAMP,
2084                        RING_CTX_TIMESTAMP(engine->mmio_base), 0);
2085         /* PDP values well be assigned later if needed */
2086         ASSIGN_CTX_REG(reg_state, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3),
2087                        0);
2088         ASSIGN_CTX_REG(reg_state, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3),
2089                        0);
2090         ASSIGN_CTX_REG(reg_state, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2),
2091                        0);
2092         ASSIGN_CTX_REG(reg_state, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2),
2093                        0);
2094         ASSIGN_CTX_REG(reg_state, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1),
2095                        0);
2096         ASSIGN_CTX_REG(reg_state, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1),
2097                        0);
2098         ASSIGN_CTX_REG(reg_state, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0),
2099                        0);
2100         ASSIGN_CTX_REG(reg_state, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0),
2101                        0);
2102
2103         if (ppgtt && USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
2104                 /* 64b PPGTT (48bit canonical)
2105                  * PDP0_DESCRIPTOR contains the base address to PML4 and
2106                  * other PDP Descriptors are ignored.
2107                  */
2108                 ASSIGN_CTX_PML4(ppgtt, reg_state);
2109         }
2110
2111         if (engine->id == RCS) {
2112                 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
2113                 ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
2114                                make_rpcs(dev_priv));
2115         }
2116 }
2117
2118 static int
2119 populate_lr_context(struct i915_gem_context *ctx,
2120                     struct drm_i915_gem_object *ctx_obj,
2121                     struct intel_engine_cs *engine,
2122                     struct intel_ring *ring)
2123 {
2124         void *vaddr;
2125         int ret;
2126
2127         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
2128         if (ret) {
2129                 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
2130                 return ret;
2131         }
2132
2133         vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
2134         if (IS_ERR(vaddr)) {
2135                 ret = PTR_ERR(vaddr);
2136                 DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
2137                 return ret;
2138         }
2139         ctx_obj->mm.dirty = true;
2140
2141         /* The second page of the context object contains some fields which must
2142          * be set up prior to the first execution. */
2143
2144         execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE,
2145                                  ctx, engine, ring);
2146
2147         i915_gem_object_unpin_map(ctx_obj);
2148
2149         return 0;
2150 }
2151
2152 /**
2153  * intel_lr_context_size() - return the size of the context for an engine
2154  * @engine: which engine to find the context size for
2155  *
2156  * Each engine may require a different amount of space for a context image,
2157  * so when allocating (or copying) an image, this function can be used to
2158  * find the right size for the specific engine.
2159  *
2160  * Return: size (in bytes) of an engine-specific context image
2161  *
2162  * Note: this size includes the HWSP, which is part of the context image
2163  * in LRC mode, but does not include the "shared data page" used with
2164  * GuC submission. The caller should account for this if using the GuC.
2165  */
2166 uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
2167 {
2168         int ret = 0;
2169
2170         WARN_ON(INTEL_GEN(engine->i915) < 8);
2171
2172         switch (engine->id) {
2173         case RCS:
2174                 if (INTEL_GEN(engine->i915) >= 9)
2175                         ret = GEN9_LR_CONTEXT_RENDER_SIZE;
2176                 else
2177                         ret = GEN8_LR_CONTEXT_RENDER_SIZE;
2178                 break;
2179         case VCS:
2180         case BCS:
2181         case VECS:
2182         case VCS2:
2183                 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
2184                 break;
2185         }
2186
2187         return ret;
2188 }
2189
2190 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
2191                                             struct intel_engine_cs *engine)
2192 {
2193         struct drm_i915_gem_object *ctx_obj;
2194         struct intel_context *ce = &ctx->engine[engine->id];
2195         struct i915_vma *vma;
2196         uint32_t context_size;
2197         struct intel_ring *ring;
2198         int ret;
2199
2200         WARN_ON(ce->state);
2201
2202         context_size = round_up(intel_lr_context_size(engine),
2203                                 I915_GTT_PAGE_SIZE);
2204
2205         /* One extra page as the sharing data between driver and GuC */
2206         context_size += PAGE_SIZE * LRC_PPHWSP_PN;
2207
2208         ctx_obj = i915_gem_object_create(ctx->i915, context_size);
2209         if (IS_ERR(ctx_obj)) {
2210                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
2211                 return PTR_ERR(ctx_obj);
2212         }
2213
2214         vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL);
2215         if (IS_ERR(vma)) {
2216                 ret = PTR_ERR(vma);
2217                 goto error_deref_obj;
2218         }
2219
2220         ring = intel_engine_create_ring(engine, ctx->ring_size);
2221         if (IS_ERR(ring)) {
2222                 ret = PTR_ERR(ring);
2223                 goto error_deref_obj;
2224         }
2225
2226         ret = populate_lr_context(ctx, ctx_obj, engine, ring);
2227         if (ret) {
2228                 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
2229                 goto error_ring_free;
2230         }
2231
2232         ce->ring = ring;
2233         ce->state = vma;
2234         ce->initialised = engine->init_context == NULL;
2235
2236         return 0;
2237
2238 error_ring_free:
2239         intel_ring_free(ring);
2240 error_deref_obj:
2241         i915_gem_object_put(ctx_obj);
2242         return ret;
2243 }
2244
2245 void intel_lr_context_resume(struct drm_i915_private *dev_priv)
2246 {
2247         struct intel_engine_cs *engine;
2248         struct i915_gem_context *ctx;
2249         enum intel_engine_id id;
2250
2251         /* Because we emit WA_TAIL_DWORDS there may be a disparity
2252          * between our bookkeeping in ce->ring->head and ce->ring->tail and
2253          * that stored in context. As we only write new commands from
2254          * ce->ring->tail onwards, everything before that is junk. If the GPU
2255          * starts reading from its RING_HEAD from the context, it may try to
2256          * execute that junk and die.
2257          *
2258          * So to avoid that we reset the context images upon resume. For
2259          * simplicity, we just zero everything out.
2260          */
2261         list_for_each_entry(ctx, &dev_priv->context_list, link) {
2262                 for_each_engine(engine, dev_priv, id) {
2263                         struct intel_context *ce = &ctx->engine[engine->id];
2264                         u32 *reg;
2265
2266                         if (!ce->state)
2267                                 continue;
2268
2269                         reg = i915_gem_object_pin_map(ce->state->obj,
2270                                                       I915_MAP_WB);
2271                         if (WARN_ON(IS_ERR(reg)))
2272                                 continue;
2273
2274                         reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
2275                         reg[CTX_RING_HEAD+1] = 0;
2276                         reg[CTX_RING_TAIL+1] = 0;
2277
2278                         ce->state->obj->mm.dirty = true;
2279                         i915_gem_object_unpin_map(ce->state->obj);
2280
2281                         ce->ring->head = ce->ring->tail = 0;
2282                         ce->ring->last_retired_head = -1;
2283                         intel_ring_update_space(ce->ring);
2284                 }
2285         }
2286 }