drivers/gpu/drm/i915/intel_lrc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Ben Widawsky <ben@bwidawsk.net>
  25  *    Michel Thierry <michel.thierry@intel.com>
  26  *    Thomas Daniel <thomas.daniel@intel.com>
  27  *    Oscar Mateo <oscar.mateo@intel.com>
  28  *
  29  */
  30
  31 /**
  32  * DOC: Logical Rings, Logical Ring Contexts and Execlists
  33  *
  34  * Motivation:
  35  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
  36  * These expanded contexts enable a number of new abilities, especially
  37  * "Execlists" (also implemented in this file).
  38  *
  39  * One of the main differences with the legacy HW contexts is that logical
  40  * ring contexts incorporate many more things to the context's state, like
  41  * PDPs or ringbuffer control registers:
  42  *
  43  * The reason why PDPs are included in the context is straightforward: as
  44  * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
  45  * contained there mean you don't need to do a ppgtt->switch_mm yourself,
  46  * instead, the GPU will do it for you on the context switch.
  47  *
  48  * But, what about the ringbuffer control registers (head, tail, etc..)?
  49  * shouldn't we just need a set of those per engine command streamer? This is
  50  * where the name "Logical Rings" starts to make sense: by virtualizing the
  51  * rings, the engine cs shifts to a new "ring buffer" with every context
  52  * switch. When you want to submit a workload to the GPU you: A) choose your
  53  * context, B) find its appropriate virtualized ring, C) write commands to it
  54  * and then, finally, D) tell the GPU to switch to that context.
  55  *
  56  * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
  57  * to a contexts is via a context execution list, ergo "Execlists".
  58  *
  59  * LRC implementation:
  60  * Regarding the creation of contexts, we have:
  61  *
  62  * - One global default context.
  63  * - One local default context for each opened fd.
  64  * - One local extra context for each context create ioctl call.
  65  *
  66  * Now that ringbuffers belong per-context (and not per-engine, like before)
  67  * and that contexts are uniquely tied to a given engine (and not reusable,
  68  * like before) we need:
  69  *
  70  * - One ringbuffer per-engine inside each context.
  71  * - One backing object per-engine inside each context.
  72  *
  73  * The global default context starts its life with these new objects fully
  74  * allocated and populated. The local default context for each opened fd is
  75  * more complex, because we don't know at creation time which engine is going
  76  * to use them. To handle this, we have implemented a deferred creation of LR
  77  * contexts:
  78  *
  79  * The local context starts its life as a hollow or blank holder, that only
  80  * gets populated for a given engine once we receive an execbuffer. If later
  81  * on we receive another execbuffer ioctl for the same context but a different
  82  * engine, we allocate/populate a new ringbuffer and context backing object and
  83  * so on.
  84  *
  85  * Finally, regarding local contexts created using the ioctl call: as they are
  86  * only allowed with the render ring, we can allocate & populate them right
  87  * away (no need to defer anything, at least for now).
  88  *
  89  * Execlists implementation:
  90  * Execlists are the new method by which, on gen8+ hardware, workloads are
  91  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
  92  * This method works as follows:
  93  *
  94  * When a request is committed, its commands (the BB start and any leading or
  95  * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
  96  * for the appropriate context. The tail pointer in the hardware context is not
  97  * updated at this time, but instead, kept by the driver in the ringbuffer
  98  * structure. A structure representing this request is added to a request queue
  99  * for the appropriate engine: this structure contains a copy of the context's
 100  * tail after the request was written to the ring buffer and a pointer to the
 101  * context itself.
 102  *
 103  * If the engine's request queue was empty before the request was added, the
 104  * queue is processed immediately. Otherwise the queue will be processed during
 105  * a context switch interrupt. In any case, elements on the queue will get sent
 106  * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
 107  * globally unique 20-bits submission ID.
 108  *
 109  * When execution of a request completes, the GPU updates the context status
 110  * buffer with a context complete event and generates a context switch interrupt.
 111  * During the interrupt handling, the driver examines the events in the buffer:
 112  * for each context complete event, if the announced ID matches that on the head
 113  * of the request queue, then that request is retired and removed from the queue.
 114  *
 115  * After processing, if any requests were retired and the queue is not empty
 116  * then a new execution list can be submitted. The two requests at the front of
 117  * the queue are next to be submitted but since a context may not occur twice in
 118  * an execution list, if subsequent requests have the same ID as the first then
 119  * the two requests must be combined. This is done simply by discarding requests
 120  * at the head of the queue until either only one requests is left (in which case
 121  * we use a NULL second context) or the first two requests have unique IDs.
 122  *
 123  * By always executing the first two requests in the queue the driver ensures
 124  * that the GPU is kept as busy as possible. In the case where a single context
 125  * completes but a second context is still executing, the request for this second
 126  * context will be at the head of the queue when we remove the first one. This
 127  * request will then be resubmitted along with a new request for a different context,
 128  * which will cause the hardware to continue executing the second request and queue
 129  * the new request (the GPU detects the condition of a context getting preempted
 130  * with the same context and optimizes the context switch flow by not doing
 131  * preemption, but just sampling the new tail pointer).
 132  *
 133  */
 134
 135 #include <drm/drmP.h>
 136 #include <drm/i915_drm.h>
 137 #include "i915_drv.h"
 138
 139 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
 140 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
 141 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
 142
 143 #define RING_EXECLIST_QFULL             (1 << 0x2)
 144 #define RING_EXECLIST1_VALID            (1 << 0x3)
 145 #define RING_EXECLIST0_VALID            (1 << 0x4)
 146 #define RING_EXECLIST_ACTIVE_STATUS     (3 << 0xE)
 147 #define RING_EXECLIST1_ACTIVE           (1 << 0x11)
 148 #define RING_EXECLIST0_ACTIVE           (1 << 0x12)
 149
 150 #define GEN8_CTX_STATUS_IDLE_ACTIVE     (1 << 0)
 151 #define GEN8_CTX_STATUS_PREEMPTED       (1 << 1)
 152 #define GEN8_CTX_STATUS_ELEMENT_SWITCH  (1 << 2)
 153 #define GEN8_CTX_STATUS_ACTIVE_IDLE     (1 << 3)
 154 #define GEN8_CTX_STATUS_COMPLETE        (1 << 4)
 155 #define GEN8_CTX_STATUS_LITE_RESTORE    (1 << 15)
 156
 157 #define CTX_LRI_HEADER_0                0x01
 158 #define CTX_CONTEXT_CONTROL             0x02
 159 #define CTX_RING_HEAD                   0x04
 160 #define CTX_RING_TAIL                   0x06
 161 #define CTX_RING_BUFFER_START           0x08
 162 #define CTX_RING_BUFFER_CONTROL         0x0a
 163 #define CTX_BB_HEAD_U                   0x0c
 164 #define CTX_BB_HEAD_L                   0x0e
 165 #define CTX_BB_STATE                    0x10
 166 #define CTX_SECOND_BB_HEAD_U            0x12
 167 #define CTX_SECOND_BB_HEAD_L            0x14
 168 #define CTX_SECOND_BB_STATE             0x16
 169 #define CTX_BB_PER_CTX_PTR              0x18
 170 #define CTX_RCS_INDIRECT_CTX            0x1a
 171 #define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
 172 #define CTX_LRI_HEADER_1                0x21
 173 #define CTX_CTX_TIMESTAMP               0x22
 174 #define CTX_PDP3_UDW                    0x24
 175 #define CTX_PDP3_LDW                    0x26
 176 #define CTX_PDP2_UDW                    0x28
 177 #define CTX_PDP2_LDW                    0x2a
 178 #define CTX_PDP1_UDW                    0x2c
 179 #define CTX_PDP1_LDW                    0x2e
 180 #define CTX_PDP0_UDW                    0x30
 181 #define CTX_PDP0_LDW                    0x32
 182 #define CTX_LRI_HEADER_2                0x41
 183 #define CTX_R_PWR_CLK_STATE             0x42
 184 #define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
 185
 186 #define GEN8_CTX_VALID (1<<0)
 187 #define GEN8_CTX_FORCE_PD_RESTORE (1<<1)
 188 #define GEN8_CTX_FORCE_RESTORE (1<<2)
 189 #define GEN8_CTX_L3LLC_COHERENT (1<<5)
 190 #define GEN8_CTX_PRIVILEGE (1<<8)
 191
 192 #define ASSIGN_CTX_PDP(ppgtt, reg_state, n) { \
 193         const u64 _addr = test_bit(n, ppgtt->pdp.used_pdpes) ? \
 194                 ppgtt->pdp.page_directory[n]->daddr : \
 195                 ppgtt->scratch_pd->daddr; \
 196         reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
 197         reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
 198 }
 199
 200 enum {
 201         ADVANCED_CONTEXT = 0,
 202         LEGACY_CONTEXT,
 203         ADVANCED_AD_CONTEXT,
 204         LEGACY_64B_CONTEXT
 205 };
 206 #define GEN8_CTX_MODE_SHIFT 3
 207 enum {
 208         FAULT_AND_HANG = 0,
 209         FAULT_AND_HALT, /* Debug only */
 210         FAULT_AND_STREAM,
 211         FAULT_AND_CONTINUE /* Unsupported */
 212 };
 213 #define GEN8_CTX_ID_SHIFT 32
 214
 215 static int intel_lr_context_pin(struct intel_engine_cs *ring,
 216                 struct intel_context *ctx);
 217
 218 /**
 219  * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
 220  * @dev: DRM device.
 221  * @enable_execlists: value of i915.enable_execlists module parameter.
 222  *
 223  * Only certain platforms support Execlists (the prerequisites being
 224  * support for Logical Ring Contexts and Aliasing PPGTT or better).
 225  *
 226  * Return: 1 if Execlists is supported and has to be enabled.
 227  */
 228 int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
 229 {
 230         WARN_ON(i915.enable_ppgtt == -1);
 231
 232         if (INTEL_INFO(dev)->gen >= 9)
 233                 return 1;
 234
 235         if (enable_execlists == 0)
 236                 return 0;
 237
 238         if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) &&
 239             i915.use_mmio_flip >= 0)
 240                 return 1;
 241
 242         return 0;
 243 }
 244
 245 /**
 246  * intel_execlists_ctx_id() - get the Execlists Context ID
 247  * @ctx_obj: Logical Ring Context backing object.
 248  *
 249  * Do not confuse with ctx->id! Unfortunately we have a name overload
 250  * here: the old context ID we pass to userspace as a handler so that
 251  * they can refer to a context, and the new context ID we pass to the
 252  * ELSP so that the GPU can inform us of the context status via
 253  * interrupts.
 254  *
 255  * Return: 20-bits globally unique context ID.
 256  */
 257 u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj)
 258 {
 259         u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj);
 260
 261         /* LRCA is required to be 4K aligned so the more significant 20 bits
 262          * are globally unique */
 263         return lrca >> 12;
 264 }
 265
 266 static uint64_t execlists_ctx_descriptor(struct intel_engine_cs *ring,
 267                                          struct drm_i915_gem_object *ctx_obj)
 268 {
 269         struct drm_device *dev = ring->dev;
 270         uint64_t desc;
 271         uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj);
 272
 273         WARN_ON(lrca & 0xFFFFFFFF00000FFFULL);
 274
 275         desc = GEN8_CTX_VALID;
 276         desc |= LEGACY_CONTEXT << GEN8_CTX_MODE_SHIFT;
 277         if (IS_GEN8(ctx_obj->base.dev))
 278                 desc |= GEN8_CTX_L3LLC_COHERENT;
 279         desc |= GEN8_CTX_PRIVILEGE;
 280         desc |= lrca;
 281         desc |= (u64)intel_execlists_ctx_id(ctx_obj) << GEN8_CTX_ID_SHIFT;
 282
 283         /* TODO: WaDisableLiteRestore when we start using semaphore
 284          * signalling between Command Streamers */
 285         /* desc |= GEN8_CTX_FORCE_RESTORE; */
 286
 287         /* WaEnableForceRestoreInCtxtDescForVCS:skl */
 288         if (IS_GEN9(dev) &&
 289             INTEL_REVID(dev) <= SKL_REVID_B0 &&
 290             (ring->id == BCS || ring->id == VCS ||
 291             ring->id == VECS || ring->id == VCS2))
 292                 desc |= GEN8_CTX_FORCE_RESTORE;
 293
 294         return desc;
 295 }
 296
 297 static void execlists_elsp_write(struct intel_engine_cs *ring,
 298                                  struct drm_i915_gem_object *ctx_obj0,
 299                                  struct drm_i915_gem_object *ctx_obj1)
 300 {
 301         struct drm_device *dev = ring->dev;
 302         struct drm_i915_private *dev_priv = dev->dev_private;
 303         uint64_t temp = 0;
 304         uint32_t desc[4];
 305
 306         /* XXX: You must always write both descriptors in the order below. */
 307         if (ctx_obj1)
 308                 temp = execlists_ctx_descriptor(ring, ctx_obj1);
 309         else
 310                 temp = 0;
 311         desc[1] = (u32)(temp >> 32);
 312         desc[0] = (u32)temp;
 313
 314         temp = execlists_ctx_descriptor(ring, ctx_obj0);
 315         desc[3] = (u32)(temp >> 32);
 316         desc[2] = (u32)temp;
 317
 318         spin_lock(&dev_priv->uncore.lock);
 319         intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
 320         I915_WRITE_FW(RING_ELSP(ring), desc[1]);
 321         I915_WRITE_FW(RING_ELSP(ring), desc[0]);
 322         I915_WRITE_FW(RING_ELSP(ring), desc[3]);
 323
 324         /* The context is automatically loaded after the following */
 325         I915_WRITE_FW(RING_ELSP(ring), desc[2]);
 326
 327         /* ELSP is a wo register, so use another nearby reg for posting instead */
 328         POSTING_READ_FW(RING_EXECLIST_STATUS(ring));
 329         intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
 330         spin_unlock(&dev_priv->uncore.lock);
 331 }
 332
 333 static int execlists_update_context(struct drm_i915_gem_object *ctx_obj,
 334                                     struct drm_i915_gem_object *ring_obj,
 335                                     struct i915_hw_ppgtt *ppgtt,
 336                                     u32 tail)
 337 {
 338         struct page *page;
 339         uint32_t *reg_state;
 340
 341         page = i915_gem_object_get_page(ctx_obj, 1);
 342         reg_state = kmap_atomic(page);
 343
 344         reg_state[CTX_RING_TAIL+1] = tail;
 345         reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj);
 346
 347         /* True PPGTT with dynamic page allocation: update PDP registers and
 348          * point the unallocated PDPs to the scratch page
 349          */
 350         if (ppgtt) {
 351                 ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
 352                 ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
 353                 ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
 354                 ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
 355         }
 356
 357         kunmap_atomic(reg_state);
 358
 359         return 0;
 360 }
 361
 362 static void execlists_submit_contexts(struct intel_engine_cs *ring,
 363                                       struct intel_context *to0, u32 tail0,
 364                                       struct intel_context *to1, u32 tail1)
 365 {
 366         struct drm_i915_gem_object *ctx_obj0 = to0->engine[ring->id].state;
 367         struct intel_ringbuffer *ringbuf0 = to0->engine[ring->id].ringbuf;
 368         struct drm_i915_gem_object *ctx_obj1 = NULL;
 369         struct intel_ringbuffer *ringbuf1 = NULL;
 370
 371         BUG_ON(!ctx_obj0);
 372         WARN_ON(!i915_gem_obj_is_pinned(ctx_obj0));
 373         WARN_ON(!i915_gem_obj_is_pinned(ringbuf0->obj));
 374
 375         execlists_update_context(ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);
 376
 377         if (to1) {
 378                 ringbuf1 = to1->engine[ring->id].ringbuf;
 379                 ctx_obj1 = to1->engine[ring->id].state;
 380                 BUG_ON(!ctx_obj1);
 381                 WARN_ON(!i915_gem_obj_is_pinned(ctx_obj1));
 382                 WARN_ON(!i915_gem_obj_is_pinned(ringbuf1->obj));
 383
 384                 execlists_update_context(ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1);
 385         }
 386
 387         execlists_elsp_write(ring, ctx_obj0, ctx_obj1);
 388 }
 389
 390 static void execlists_context_unqueue(struct intel_engine_cs *ring)
 391 {
 392         struct drm_i915_gem_request *req0 = NULL, *req1 = NULL;
 393         struct drm_i915_gem_request *cursor = NULL, *tmp = NULL;
 394
 395         assert_spin_locked(&ring->execlist_lock);
 396
 397         if (list_empty(&ring->execlist_queue))
 398                 return;
 399
 400         /* Try to read in pairs */
 401         list_for_each_entry_safe(cursor, tmp, &ring->execlist_queue,
 402                                  execlist_link) {
 403                 if (!req0) {
 404                         req0 = cursor;
 405                 } else if (req0->ctx == cursor->ctx) {
 406                         /* Same ctx: ignore first request, as second request
 407                          * will update tail past first request's workload */
 408                         cursor->elsp_submitted = req0->elsp_submitted;
 409                         list_del(&req0->execlist_link);
 410                         list_add_tail(&req0->execlist_link,
 411                                 &ring->execlist_retired_req_list);
 412                         req0 = cursor;
 413                 } else {
 414                         req1 = cursor;
 415                         break;
 416                 }
 417         }
 418
 419         WARN_ON(req1 && req1->elsp_submitted);
 420
 421         execlists_submit_contexts(ring, req0->ctx, req0->tail,
 422                                   req1 ? req1->ctx : NULL,
 423                                   req1 ? req1->tail : 0);
 424
 425         req0->elsp_submitted++;
 426         if (req1)
 427                 req1->elsp_submitted++;
 428 }
 429
 430 static bool execlists_check_remove_request(struct intel_engine_cs *ring,
 431                                            u32 request_id)
 432 {
 433         struct drm_i915_gem_request *head_req;
 434
 435         assert_spin_locked(&ring->execlist_lock);
 436
 437         head_req = list_first_entry_or_null(&ring->execlist_queue,
 438                                             struct drm_i915_gem_request,
 439                                             execlist_link);
 440
 441         if (head_req != NULL) {
 442                 struct drm_i915_gem_object *ctx_obj =
 443                                 head_req->ctx->engine[ring->id].state;
 444                 if (intel_execlists_ctx_id(ctx_obj) == request_id) {
 445                         WARN(head_req->elsp_submitted == 0,
 446                              "Never submitted head request\n");
 447
 448                         if (--head_req->elsp_submitted <= 0) {
 449                                 list_del(&head_req->execlist_link);
 450                                 list_add_tail(&head_req->execlist_link,
 451                                         &ring->execlist_retired_req_list);
 452                                 return true;
 453                         }
 454                 }
 455         }
 456
 457         return false;
 458 }
 459
 460 /**
 461  * intel_lrc_irq_handler() - handle Context Switch interrupts
 462  * @ring: Engine Command Streamer to handle.
 463  *
 464  * Check the unread Context Status Buffers and manage the submission of new
 465  * contexts to the ELSP accordingly.
 466  */
 467 void intel_lrc_irq_handler(struct intel_engine_cs *ring)
 468 {
 469         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 470         u32 status_pointer;
 471         u8 read_pointer;
 472         u8 write_pointer;
 473         u32 status;
 474         u32 status_id;
 475         u32 submit_contexts = 0;
 476
 477         status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
 478
 479         read_pointer = ring->next_context_status_buffer;
 480         write_pointer = status_pointer & 0x07;
 481         if (read_pointer > write_pointer)
 482                 write_pointer += 6;
 483
 484         spin_lock(&ring->execlist_lock);
 485
 486         while (read_pointer < write_pointer) {
 487                 read_pointer++;
 488                 status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
 489                                 (read_pointer % 6) * 8);
 490                 status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
 491                                 (read_pointer % 6) * 8 + 4);
 492
 493                 if (status & GEN8_CTX_STATUS_PREEMPTED) {
 494                         if (status & GEN8_CTX_STATUS_LITE_RESTORE) {
 495                                 if (execlists_check_remove_request(ring, status_id))
 496                                         WARN(1, "Lite Restored request removed from queue\n");
 497                         } else
 498                                 WARN(1, "Preemption without Lite Restore\n");
 499                 }
 500
 501                  if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) ||
 502                      (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) {
 503                         if (execlists_check_remove_request(ring, status_id))
 504                                 submit_contexts++;
 505                 }
 506         }
 507
 508         if (submit_contexts != 0)
 509                 execlists_context_unqueue(ring);
 510
 511         spin_unlock(&ring->execlist_lock);
 512
 513         WARN(submit_contexts > 2, "More than two context complete events?\n");
 514         ring->next_context_status_buffer = write_pointer % 6;
 515
 516         I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
 517                    ((u32)ring->next_context_status_buffer & 0x07) << 8);
 518 }
 519
 520 static int execlists_context_queue(struct intel_engine_cs *ring,
 521                                    struct intel_context *to,
 522                                    u32 tail,
 523                                    struct drm_i915_gem_request *request)
 524 {
 525         struct drm_i915_gem_request *cursor;
 526         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 527         int num_elements = 0;
 528
 529         if (to != ring->default_context)
 530                 intel_lr_context_pin(ring, to);
 531
 532         if (!request) {
 533                 /*
 534                  * If there isn't a request associated with this submission,
 535                  * create one as a temporary holder.
 536                  */
 537                 request = kzalloc(sizeof(*request), GFP_KERNEL);
 538                 if (request == NULL)
 539                         return -ENOMEM;
 540                 request->ring = ring;
 541                 request->ctx = to;
 542                 kref_init(&request->ref);
 543                 i915_gem_context_reference(request->ctx);
 544         } else {
 545                 i915_gem_request_reference(request);
 546                 WARN_ON(to != request->ctx);
 547         }
 548         request->tail = tail;
 549
 550         spin_lock_irq(&ring->execlist_lock);
 551
 552         list_for_each_entry(cursor, &ring->execlist_queue, execlist_link)
 553                 if (++num_elements > 2)
 554                         break;
 555
 556         if (num_elements > 2) {
 557                 struct drm_i915_gem_request *tail_req;
 558
 559                 tail_req = list_last_entry(&ring->execlist_queue,
 560                                            struct drm_i915_gem_request,
 561                                            execlist_link);
 562
 563                 if (to == tail_req->ctx) {
 564                         WARN(tail_req->elsp_submitted != 0,
 565                                 "More than 2 already-submitted reqs queued\n");
 566                         list_del(&tail_req->execlist_link);
 567                         list_add_tail(&tail_req->execlist_link,
 568                                 &ring->execlist_retired_req_list);
 569                 }
 570         }
 571
 572         list_add_tail(&request->execlist_link, &ring->execlist_queue);
 573         if (num_elements == 0)
 574                 execlists_context_unqueue(ring);
 575
 576         spin_unlock_irq(&ring->execlist_lock);
 577
 578         return 0;
 579 }
 580
 581 static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf,
 582                                               struct intel_context *ctx)
 583 {
 584         struct intel_engine_cs *ring = ringbuf->ring;
 585         uint32_t flush_domains;
 586         int ret;
 587
 588         flush_domains = 0;
 589         if (ring->gpu_caches_dirty)
 590                 flush_domains = I915_GEM_GPU_DOMAINS;
 591
 592         ret = ring->emit_flush(ringbuf, ctx,
 593                                I915_GEM_GPU_DOMAINS, flush_domains);
 594         if (ret)
 595                 return ret;
 596
 597         ring->gpu_caches_dirty = false;
 598         return 0;
 599 }
 600
 601 static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf,
 602                                  struct intel_context *ctx,
 603                                  struct list_head *vmas)
 604 {
 605         struct intel_engine_cs *ring = ringbuf->ring;
 606         struct i915_vma *vma;
 607         uint32_t flush_domains = 0;
 608         bool flush_chipset = false;
 609         int ret;
 610
 611         list_for_each_entry(vma, vmas, exec_list) {
 612                 struct drm_i915_gem_object *obj = vma->obj;
 613
 614                 ret = i915_gem_object_sync(obj, ring);
 615                 if (ret)
 616                         return ret;
 617
 618                 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
 619                         flush_chipset |= i915_gem_clflush_object(obj, false);
 620
 621                 flush_domains |= obj->base.write_domain;
 622         }
 623
 624         if (flush_domains & I915_GEM_DOMAIN_GTT)
 625                 wmb();
 626
 627         /* Unconditionally invalidate gpu caches and ensure that we do flush
 628          * any residual writes from the previous batch.
 629          */
 630         return logical_ring_invalidate_all_caches(ringbuf, ctx);
 631 }
 632
 633 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request,
 634                                             struct intel_context *ctx)
 635 {
 636         int ret;
 637
 638         if (ctx != request->ring->default_context) {
 639                 ret = intel_lr_context_pin(request->ring, ctx);
 640                 if (ret)
 641                         return ret;
 642         }
 643
 644         request->ringbuf = ctx->engine[request->ring->id].ringbuf;
 645         request->ctx     = ctx;
 646         i915_gem_context_reference(request->ctx);
 647
 648         return 0;
 649 }
 650
 651 static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,
 652                                        struct intel_context *ctx,
 653                                        int bytes)
 654 {
 655         struct intel_engine_cs *ring = ringbuf->ring;
 656         struct drm_i915_gem_request *request;
 657         int ret, new_space;
 658
 659         if (intel_ring_space(ringbuf) >= bytes)
 660                 return 0;
 661
 662         list_for_each_entry(request, &ring->request_list, list) {
 663                 /*
 664                  * The request queue is per-engine, so can contain requests
 665                  * from multiple ringbuffers. Here, we must ignore any that
 666                  * aren't from the ringbuffer we're considering.
 667                  */
 668                 struct intel_context *ctx = request->ctx;
 669                 if (ctx->engine[ring->id].ringbuf != ringbuf)
 670                         continue;
 671
 672                 /* Would completion of this request free enough space? */
 673                 new_space = __intel_ring_space(request->postfix, ringbuf->tail,
 674                                        ringbuf->size);
 675                 if (new_space >= bytes)
 676                         break;
 677         }
 678
 679         if (WARN_ON(&request->list == &ring->request_list))
 680                 return -ENOSPC;
 681
 682         ret = i915_wait_request(request);
 683         if (ret)
 684                 return ret;
 685
 686         i915_gem_retire_requests_ring(ring);
 687
 688         WARN_ON(intel_ring_space(ringbuf) < new_space);
 689
 690         return intel_ring_space(ringbuf) >= bytes ? 0 : -ENOSPC;
 691 }
 692
 693 /*
 694  * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload
 695  * @ringbuf: Logical Ringbuffer to advance.
 696  *
 697  * The tail is updated in our logical ringbuffer struct, not in the actual context. What
 698  * really happens during submission is that the context and current tail will be placed
 699  * on a queue waiting for the ELSP to be ready to accept a new context submission. At that
 700  * point, the tail *inside* the context is updated and the ELSP written to.
 701  */
 702 static void
 703 intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf,
 704                                       struct intel_context *ctx,
 705                                       struct drm_i915_gem_request *request)
 706 {
 707         struct intel_engine_cs *ring = ringbuf->ring;
 708
 709         intel_logical_ring_advance(ringbuf);
 710
 711         if (intel_ring_stopped(ring))
 712                 return;
 713
 714         execlists_context_queue(ring, ctx, ringbuf->tail, request);
 715 }
 716
 717 static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf,
 718                                     struct intel_context *ctx)
 719 {
 720         uint32_t __iomem *virt;
 721         int rem = ringbuf->size - ringbuf->tail;
 722
 723         if (ringbuf->space < rem) {
 724                 int ret = logical_ring_wait_for_space(ringbuf, ctx, rem);
 725
 726                 if (ret)
 727                         return ret;
 728         }
 729
 730         virt = ringbuf->virtual_start + ringbuf->tail;
 731         rem /= 4;
 732         while (rem--)
 733                 iowrite32(MI_NOOP, virt++);
 734
 735         ringbuf->tail = 0;
 736         intel_ring_update_space(ringbuf);
 737
 738         return 0;
 739 }
 740
 741 static int logical_ring_prepare(struct intel_ringbuffer *ringbuf,
 742                                 struct intel_context *ctx, int bytes)
 743 {
 744         int ret;
 745
 746         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
 747                 ret = logical_ring_wrap_buffer(ringbuf, ctx);
 748                 if (unlikely(ret))
 749                         return ret;
 750         }
 751
 752         if (unlikely(ringbuf->space < bytes)) {
 753                 ret = logical_ring_wait_for_space(ringbuf, ctx, bytes);
 754                 if (unlikely(ret))
 755                         return ret;
 756         }
 757
 758         return 0;
 759 }
 760
 761 /**
 762  * intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands
 763  *
 764  * @ringbuf: Logical ringbuffer.
 765  * @num_dwords: number of DWORDs that we plan to write to the ringbuffer.
 766  *
 767  * The ringbuffer might not be ready to accept the commands right away (maybe it needs to
 768  * be wrapped, or wait a bit for the tail to be updated). This function takes care of that
 769  * and also preallocates a request (every workload submission is still mediated through
 770  * requests, same as it did with legacy ringbuffer submission).
 771  *
 772  * Return: non-zero if the ringbuffer is not ready to be written to.
 773  */
 774 static int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf,
 775                                     struct intel_context *ctx, int num_dwords)
 776 {
 777         struct intel_engine_cs *ring = ringbuf->ring;
 778         struct drm_device *dev = ring->dev;
 779         struct drm_i915_private *dev_priv = dev->dev_private;
 780         int ret;
 781
 782         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 783                                    dev_priv->mm.interruptible);
 784         if (ret)
 785                 return ret;
 786
 787         ret = logical_ring_prepare(ringbuf, ctx, num_dwords * sizeof(uint32_t));
 788         if (ret)
 789                 return ret;
 790
 791         /* Preallocate the olr before touching the ring */
 792         ret = i915_gem_request_alloc(ring, ctx);
 793         if (ret)
 794                 return ret;
 795
 796         ringbuf->space -= num_dwords * sizeof(uint32_t);
 797         return 0;
 798 }
 799
 800 /**
 801  * execlists_submission() - submit a batchbuffer for execution, Execlists style
 802  * @dev: DRM device.
 803  * @file: DRM file.
 804  * @ring: Engine Command Streamer to submit to.
 805  * @ctx: Context to employ for this submission.
 806  * @args: execbuffer call arguments.
 807  * @vmas: list of vmas.
 808  * @batch_obj: the batchbuffer to submit.
 809  * @exec_start: batchbuffer start virtual address pointer.
 810  * @dispatch_flags: translated execbuffer call flags.
 811  *
 812  * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts
 813  * away the submission details of the execbuffer ioctl call.
 814  *
 815  * Return: non-zero if the submission fails.
 816  */
 817 int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
 818                                struct intel_engine_cs *ring,
 819                                struct intel_context *ctx,
 820                                struct drm_i915_gem_execbuffer2 *args,
 821                                struct list_head *vmas,
 822                                struct drm_i915_gem_object *batch_obj,
 823                                u64 exec_start, u32 dispatch_flags)
 824 {
 825         struct drm_i915_private *dev_priv = dev->dev_private;
 826         struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
 827         int instp_mode;
 828         u32 instp_mask;
 829         int ret;
 830
 831         instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
 832         instp_mask = I915_EXEC_CONSTANTS_MASK;
 833         switch (instp_mode) {
 834         case I915_EXEC_CONSTANTS_REL_GENERAL:
 835         case I915_EXEC_CONSTANTS_ABSOLUTE:
 836         case I915_EXEC_CONSTANTS_REL_SURFACE:
 837                 if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
 838                         DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
 839                         return -EINVAL;
 840                 }
 841
 842                 if (instp_mode != dev_priv->relative_constants_mode) {
 843                         if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
 844                                 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
 845                                 return -EINVAL;
 846                         }
 847
 848                         /* The HW changed the meaning on this bit on gen6 */
 849                         instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
 850                 }
 851                 break;
 852         default:
 853                 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
 854                 return -EINVAL;
 855         }
 856
 857         if (args->num_cliprects != 0) {
 858                 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
 859                 return -EINVAL;
 860         } else {
 861                 if (args->DR4 == 0xffffffff) {
 862                         DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
 863                         args->DR4 = 0;
 864                 }
 865
 866                 if (args->DR1 || args->DR4 || args->cliprects_ptr) {
 867                         DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
 868                         return -EINVAL;
 869                 }
 870         }
 871
 872         if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
 873                 DRM_DEBUG("sol reset is gen7 only\n");
 874                 return -EINVAL;
 875         }
 876
 877         ret = execlists_move_to_gpu(ringbuf, ctx, vmas);
 878         if (ret)
 879                 return ret;
 880
 881         if (ring == &dev_priv->ring[RCS] &&
 882             instp_mode != dev_priv->relative_constants_mode) {
 883                 ret = intel_logical_ring_begin(ringbuf, ctx, 4);
 884                 if (ret)
 885                         return ret;
 886
 887                 intel_logical_ring_emit(ringbuf, MI_NOOP);
 888                 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
 889                 intel_logical_ring_emit(ringbuf, INSTPM);
 890                 intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
 891                 intel_logical_ring_advance(ringbuf);
 892
 893                 dev_priv->relative_constants_mode = instp_mode;
 894         }
 895
 896         ret = ring->emit_bb_start(ringbuf, ctx, exec_start, dispatch_flags);
 897         if (ret)
 898                 return ret;
 899
 900         trace_i915_gem_ring_dispatch(intel_ring_get_request(ring), dispatch_flags);
 901
 902         i915_gem_execbuffer_move_to_active(vmas, ring);
 903         i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
 904
 905         return 0;
 906 }
 907
 908 void intel_execlists_retire_requests(struct intel_engine_cs *ring)
 909 {
 910         struct drm_i915_gem_request *req, *tmp;
 911         struct list_head retired_list;
 912
 913         WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 914         if (list_empty(&ring->execlist_retired_req_list))
 915                 return;
 916
 917         INIT_LIST_HEAD(&retired_list);
 918         spin_lock_irq(&ring->execlist_lock);
 919         list_replace_init(&ring->execlist_retired_req_list, &retired_list);
 920         spin_unlock_irq(&ring->execlist_lock);
 921
 922         list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) {
 923                 struct intel_context *ctx = req->ctx;
 924                 struct drm_i915_gem_object *ctx_obj =
 925                                 ctx->engine[ring->id].state;
 926
 927                 if (ctx_obj && (ctx != ring->default_context))
 928                         intel_lr_context_unpin(ring, ctx);
 929                 list_del(&req->execlist_link);
 930                 i915_gem_request_unreference(req);
 931         }
 932 }
 933
 934 void intel_logical_ring_stop(struct intel_engine_cs *ring)
 935 {
 936         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 937         int ret;
 938
 939         if (!intel_ring_initialized(ring))
 940                 return;
 941
 942         ret = intel_ring_idle(ring);
 943         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
 944                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
 945                           ring->name, ret);
 946
 947         /* TODO: Is this correct with Execlists enabled? */
 948         I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
 949         if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
 950                 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
 951                 return;
 952         }
 953         I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
 954 }
 955
 956 int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf,
 957                                   struct intel_context *ctx)
 958 {
 959         struct intel_engine_cs *ring = ringbuf->ring;
 960         int ret;
 961
 962         if (!ring->gpu_caches_dirty)
 963                 return 0;
 964
 965         ret = ring->emit_flush(ringbuf, ctx, 0, I915_GEM_GPU_DOMAINS);
 966         if (ret)
 967                 return ret;
 968
 969         ring->gpu_caches_dirty = false;
 970         return 0;
 971 }
 972
 973 static int intel_lr_context_pin(struct intel_engine_cs *ring,
 974                 struct intel_context *ctx)
 975 {
 976         struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
 977         struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
 978         int ret = 0;
 979
 980         WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 981         if (ctx->engine[ring->id].pin_count++ == 0) {
 982                 ret = i915_gem_obj_ggtt_pin(ctx_obj,
 983                                 GEN8_LR_CONTEXT_ALIGN, 0);
 984                 if (ret)
 985                         goto reset_pin_count;
 986
 987                 ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
 988                 if (ret)
 989                         goto unpin_ctx_obj;
 990         }
 991
 992         return ret;
 993
 994 unpin_ctx_obj:
 995         i915_gem_object_ggtt_unpin(ctx_obj);
 996 reset_pin_count:
 997         ctx->engine[ring->id].pin_count = 0;
 998
 999         return ret;
1000 }
1001
1002 void intel_lr_context_unpin(struct intel_engine_cs *ring,
1003                 struct intel_context *ctx)
1004 {
1005         struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
1006         struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
1007
1008         if (ctx_obj) {
1009                 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1010                 if (--ctx->engine[ring->id].pin_count == 0) {
1011                         intel_unpin_ringbuffer_obj(ringbuf);
1012                         i915_gem_object_ggtt_unpin(ctx_obj);
1013                 }
1014         }
1015 }
1016
1017 static int intel_logical_ring_workarounds_emit(struct intel_engine_cs *ring,
1018                                                struct intel_context *ctx)
1019 {
1020         int ret, i;
1021         struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
1022         struct drm_device *dev = ring->dev;
1023         struct drm_i915_private *dev_priv = dev->dev_private;
1024         struct i915_workarounds *w = &dev_priv->workarounds;
1025
1026         if (WARN_ON_ONCE(w->count == 0))
1027                 return 0;
1028
1029         ring->gpu_caches_dirty = true;
1030         ret = logical_ring_flush_all_caches(ringbuf, ctx);
1031         if (ret)
1032                 return ret;
1033
1034         ret = intel_logical_ring_begin(ringbuf, ctx, w->count * 2 + 2);
1035         if (ret)
1036                 return ret;
1037
1038         intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count));
1039         for (i = 0; i < w->count; i++) {
1040                 intel_logical_ring_emit(ringbuf, w->reg[i].addr);
1041                 intel_logical_ring_emit(ringbuf, w->reg[i].value);
1042         }
1043         intel_logical_ring_emit(ringbuf, MI_NOOP);
1044
1045         intel_logical_ring_advance(ringbuf);
1046
1047         ring->gpu_caches_dirty = true;
1048         ret = logical_ring_flush_all_caches(ringbuf, ctx);
1049         if (ret)
1050                 return ret;
1051
1052         return 0;
1053 }
1054
1055 static int gen8_init_common_ring(struct intel_engine_cs *ring)
1056 {
1057         struct drm_device *dev = ring->dev;
1058         struct drm_i915_private *dev_priv = dev->dev_private;
1059
1060         I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
1061         I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
1062
1063         I915_WRITE(RING_MODE_GEN7(ring),
1064                    _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
1065                    _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
1066         POSTING_READ(RING_MODE_GEN7(ring));
1067         ring->next_context_status_buffer = 0;
1068         DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
1069
1070         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
1071
1072         return 0;
1073 }
1074
1075 static int gen8_init_render_ring(struct intel_engine_cs *ring)
1076 {
1077         struct drm_device *dev = ring->dev;
1078         struct drm_i915_private *dev_priv = dev->dev_private;
1079         int ret;
1080
1081         ret = gen8_init_common_ring(ring);
1082         if (ret)
1083                 return ret;
1084
1085         /* We need to disable the AsyncFlip performance optimisations in order
1086          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1087          * programmed to '1' on all products.
1088          *
1089          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
1090          */
1091         I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1092
1093         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1094
1095         return init_workarounds_ring(ring);
1096 }
1097
1098 static int gen9_init_render_ring(struct intel_engine_cs *ring)
1099 {
1100         int ret;
1101
1102         ret = gen8_init_common_ring(ring);
1103         if (ret)
1104                 return ret;
1105
1106         return init_workarounds_ring(ring);
1107 }
1108
1109 static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf,
1110                               struct intel_context *ctx,
1111                               u64 offset, unsigned dispatch_flags)
1112 {
1113         bool ppgtt = !(dispatch_flags & I915_DISPATCH_SECURE);
1114         int ret;
1115
1116         ret = intel_logical_ring_begin(ringbuf, ctx, 4);
1117         if (ret)
1118                 return ret;
1119
1120         /* FIXME(BDW): Address space and security selectors. */
1121         intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
1122         intel_logical_ring_emit(ringbuf, lower_32_bits(offset));
1123         intel_logical_ring_emit(ringbuf, upper_32_bits(offset));
1124         intel_logical_ring_emit(ringbuf, MI_NOOP);
1125         intel_logical_ring_advance(ringbuf);
1126
1127         return 0;
1128 }
1129
1130 static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring)
1131 {
1132         struct drm_device *dev = ring->dev;
1133         struct drm_i915_private *dev_priv = dev->dev_private;
1134         unsigned long flags;
1135
1136         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1137                 return false;
1138
1139         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1140         if (ring->irq_refcount++ == 0) {
1141                 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
1142                 POSTING_READ(RING_IMR(ring->mmio_base));
1143         }
1144         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1145
1146         return true;
1147 }
1148
1149 static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring)
1150 {
1151         struct drm_device *dev = ring->dev;
1152         struct drm_i915_private *dev_priv = dev->dev_private;
1153         unsigned long flags;
1154
1155         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1156         if (--ring->irq_refcount == 0) {
1157                 I915_WRITE_IMR(ring, ~ring->irq_keep_mask);
1158                 POSTING_READ(RING_IMR(ring->mmio_base));
1159         }
1160         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1161 }
1162
1163 static int gen8_emit_flush(struct intel_ringbuffer *ringbuf,
1164                            struct intel_context *ctx,
1165                            u32 invalidate_domains,
1166                            u32 unused)
1167 {
1168         struct intel_engine_cs *ring = ringbuf->ring;
1169         struct drm_device *dev = ring->dev;
1170         struct drm_i915_private *dev_priv = dev->dev_private;
1171         uint32_t cmd;
1172         int ret;
1173
1174         ret = intel_logical_ring_begin(ringbuf, ctx, 4);
1175         if (ret)
1176                 return ret;
1177
1178         cmd = MI_FLUSH_DW + 1;
1179
1180         /* We always require a command barrier so that subsequent
1181          * commands, such as breadcrumb interrupts, are strictly ordered
1182          * wrt the contents of the write cache being flushed to memory
1183          * (and thus being coherent from the CPU).
1184          */
1185         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1186
1187         if (invalidate_domains & I915_GEM_GPU_DOMAINS) {
1188                 cmd |= MI_INVALIDATE_TLB;
1189                 if (ring == &dev_priv->ring[VCS])
1190                         cmd |= MI_INVALIDATE_BSD;
1191         }
1192
1193         intel_logical_ring_emit(ringbuf, cmd);
1194         intel_logical_ring_emit(ringbuf,
1195                                 I915_GEM_HWS_SCRATCH_ADDR |
1196                                 MI_FLUSH_DW_USE_GTT);
1197         intel_logical_ring_emit(ringbuf, 0); /* upper addr */
1198         intel_logical_ring_emit(ringbuf, 0); /* value */
1199         intel_logical_ring_advance(ringbuf);
1200
1201         return 0;
1202 }
1203
1204 static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf,
1205                                   struct intel_context *ctx,
1206                                   u32 invalidate_domains,
1207                                   u32 flush_domains)
1208 {
1209         struct intel_engine_cs *ring = ringbuf->ring;
1210         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
1211         u32 flags = 0;
1212         int ret;
1213
1214         flags |= PIPE_CONTROL_CS_STALL;
1215
1216         if (flush_domains) {
1217                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
1218                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1219         }
1220
1221         if (invalidate_domains) {
1222                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
1223                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
1224                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1225                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1226                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
1227                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
1228                 flags |= PIPE_CONTROL_QW_WRITE;
1229                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
1230         }
1231
1232         ret = intel_logical_ring_begin(ringbuf, ctx, 6);
1233         if (ret)
1234                 return ret;
1235
1236         intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
1237         intel_logical_ring_emit(ringbuf, flags);
1238         intel_logical_ring_emit(ringbuf, scratch_addr);
1239         intel_logical_ring_emit(ringbuf, 0);
1240         intel_logical_ring_emit(ringbuf, 0);
1241         intel_logical_ring_emit(ringbuf, 0);
1242         intel_logical_ring_advance(ringbuf);
1243
1244         return 0;
1245 }
1246
1247 static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1248 {
1249         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1250 }
1251
1252 static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1253 {
1254         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1255 }
1256
1257 static int gen8_emit_request(struct intel_ringbuffer *ringbuf,
1258                              struct drm_i915_gem_request *request)
1259 {
1260         struct intel_engine_cs *ring = ringbuf->ring;
1261         u32 cmd;
1262         int ret;
1263
1264         ret = intel_logical_ring_begin(ringbuf, request->ctx, 6);
1265         if (ret)
1266                 return ret;
1267
1268         cmd = MI_STORE_DWORD_IMM_GEN4;
1269         cmd |= MI_GLOBAL_GTT;
1270
1271         intel_logical_ring_emit(ringbuf, cmd);
1272         intel_logical_ring_emit(ringbuf,
1273                                 (ring->status_page.gfx_addr +
1274                                 (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
1275         intel_logical_ring_emit(ringbuf, 0);
1276         intel_logical_ring_emit(ringbuf,
1277                 i915_gem_request_get_seqno(ring->outstanding_lazy_request));
1278         intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
1279         intel_logical_ring_emit(ringbuf, MI_NOOP);
1280         intel_logical_ring_advance_and_submit(ringbuf, request->ctx, request);
1281
1282         return 0;
1283 }
1284
1285 static int intel_lr_context_render_state_init(struct intel_engine_cs *ring,
1286                                               struct intel_context *ctx)
1287 {
1288         struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
1289         struct render_state so;
1290         struct drm_i915_file_private *file_priv = ctx->file_priv;
1291         struct drm_file *file = file_priv ? file_priv->file : NULL;
1292         int ret;
1293
1294         ret = i915_gem_render_state_prepare(ring, &so);
1295         if (ret)
1296                 return ret;
1297
1298         if (so.rodata == NULL)
1299                 return 0;
1300
1301         ret = ring->emit_bb_start(ringbuf,
1302                         ctx,
1303                         so.ggtt_offset,
1304                         I915_DISPATCH_SECURE);
1305         if (ret)
1306                 goto out;
1307
1308         i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring);
1309
1310         ret = __i915_add_request(ring, file, so.obj);
1311         /* intel_logical_ring_add_request moves object to inactive if it
1312          * fails */
1313 out:
1314         i915_gem_render_state_fini(&so);
1315         return ret;
1316 }
1317
1318 static int gen8_init_rcs_context(struct intel_engine_cs *ring,
1319                        struct intel_context *ctx)
1320 {
1321         int ret;
1322
1323         ret = intel_logical_ring_workarounds_emit(ring, ctx);
1324         if (ret)
1325                 return ret;
1326
1327         return intel_lr_context_render_state_init(ring, ctx);
1328 }
1329
1330 /**
1331  * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
1332  *
1333  * @ring: Engine Command Streamer.
1334  *
1335  */
1336 void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
1337 {
1338         struct drm_i915_private *dev_priv;
1339
1340         if (!intel_ring_initialized(ring))
1341                 return;
1342
1343         dev_priv = ring->dev->dev_private;
1344
1345         intel_logical_ring_stop(ring);
1346         WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
1347         i915_gem_request_assign(&ring->outstanding_lazy_request, NULL);
1348
1349         if (ring->cleanup)
1350                 ring->cleanup(ring);
1351
1352         i915_cmd_parser_fini_ring(ring);
1353         i915_gem_batch_pool_fini(&ring->batch_pool);
1354
1355         if (ring->status_page.obj) {
1356                 kunmap(sg_page(ring->status_page.obj->pages->sgl));
1357                 ring->status_page.obj = NULL;
1358         }
1359 }
1360
1361 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
1362 {
1363         int ret;
1364
1365         /* Intentionally left blank. */
1366         ring->buffer = NULL;
1367
1368         ring->dev = dev;
1369         INIT_LIST_HEAD(&ring->active_list);
1370         INIT_LIST_HEAD(&ring->request_list);
1371         i915_gem_batch_pool_init(dev, &ring->batch_pool);
1372         init_waitqueue_head(&ring->irq_queue);
1373
1374         INIT_LIST_HEAD(&ring->execlist_queue);
1375         INIT_LIST_HEAD(&ring->execlist_retired_req_list);
1376         spin_lock_init(&ring->execlist_lock);
1377
1378         ret = i915_cmd_parser_init_ring(ring);
1379         if (ret)
1380                 return ret;
1381
1382         ret = intel_lr_context_deferred_create(ring->default_context, ring);
1383
1384         return ret;
1385 }
1386
1387 static int logical_render_ring_init(struct drm_device *dev)
1388 {
1389         struct drm_i915_private *dev_priv = dev->dev_private;
1390         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
1391         int ret;
1392
1393         ring->name = "render ring";
1394         ring->id = RCS;
1395         ring->mmio_base = RENDER_RING_BASE;
1396         ring->irq_enable_mask =
1397                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
1398         ring->irq_keep_mask =
1399                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
1400         if (HAS_L3_DPF(dev))
1401                 ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1402
1403         if (INTEL_INFO(dev)->gen >= 9)
1404                 ring->init_hw = gen9_init_render_ring;
1405         else
1406                 ring->init_hw = gen8_init_render_ring;
1407         ring->init_context = gen8_init_rcs_context;
1408         ring->cleanup = intel_fini_pipe_control;
1409         ring->get_seqno = gen8_get_seqno;
1410         ring->set_seqno = gen8_set_seqno;
1411         ring->emit_request = gen8_emit_request;
1412         ring->emit_flush = gen8_emit_flush_render;
1413         ring->irq_get = gen8_logical_ring_get_irq;
1414         ring->irq_put = gen8_logical_ring_put_irq;
1415         ring->emit_bb_start = gen8_emit_bb_start;
1416
1417         ring->dev = dev;
1418         ret = logical_ring_init(dev, ring);
1419         if (ret)
1420                 return ret;
1421
1422         return intel_init_pipe_control(ring);
1423 }
1424
1425 static int logical_bsd_ring_init(struct drm_device *dev)
1426 {
1427         struct drm_i915_private *dev_priv = dev->dev_private;
1428         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
1429
1430         ring->name = "bsd ring";
1431         ring->id = VCS;
1432         ring->mmio_base = GEN6_BSD_RING_BASE;
1433         ring->irq_enable_mask =
1434                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
1435         ring->irq_keep_mask =
1436                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
1437
1438         ring->init_hw = gen8_init_common_ring;
1439         ring->get_seqno = gen8_get_seqno;
1440         ring->set_seqno = gen8_set_seqno;
1441         ring->emit_request = gen8_emit_request;
1442         ring->emit_flush = gen8_emit_flush;
1443         ring->irq_get = gen8_logical_ring_get_irq;
1444         ring->irq_put = gen8_logical_ring_put_irq;
1445         ring->emit_bb_start = gen8_emit_bb_start;
1446
1447         return logical_ring_init(dev, ring);
1448 }
1449
1450 static int logical_bsd2_ring_init(struct drm_device *dev)
1451 {
1452         struct drm_i915_private *dev_priv = dev->dev_private;
1453         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
1454
1455         ring->name = "bds2 ring";
1456         ring->id = VCS2;
1457         ring->mmio_base = GEN8_BSD2_RING_BASE;
1458         ring->irq_enable_mask =
1459                 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
1460         ring->irq_keep_mask =
1461                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
1462
1463         ring->init_hw = gen8_init_common_ring;
1464         ring->get_seqno = gen8_get_seqno;
1465         ring->set_seqno = gen8_set_seqno;
1466         ring->emit_request = gen8_emit_request;
1467         ring->emit_flush = gen8_emit_flush;
1468         ring->irq_get = gen8_logical_ring_get_irq;
1469         ring->irq_put = gen8_logical_ring_put_irq;
1470         ring->emit_bb_start = gen8_emit_bb_start;
1471
1472         return logical_ring_init(dev, ring);
1473 }
1474
1475 static int logical_blt_ring_init(struct drm_device *dev)
1476 {
1477         struct drm_i915_private *dev_priv = dev->dev_private;
1478         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
1479
1480         ring->name = "blitter ring";
1481         ring->id = BCS;
1482         ring->mmio_base = BLT_RING_BASE;
1483         ring->irq_enable_mask =
1484                 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
1485         ring->irq_keep_mask =
1486                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
1487
1488         ring->init_hw = gen8_init_common_ring;
1489         ring->get_seqno = gen8_get_seqno;
1490         ring->set_seqno = gen8_set_seqno;
1491         ring->emit_request = gen8_emit_request;
1492         ring->emit_flush = gen8_emit_flush;
1493         ring->irq_get = gen8_logical_ring_get_irq;
1494         ring->irq_put = gen8_logical_ring_put_irq;
1495         ring->emit_bb_start = gen8_emit_bb_start;
1496
1497         return logical_ring_init(dev, ring);
1498 }
1499
1500 static int logical_vebox_ring_init(struct drm_device *dev)
1501 {
1502         struct drm_i915_private *dev_priv = dev->dev_private;
1503         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
1504
1505         ring->name = "video enhancement ring";
1506         ring->id = VECS;
1507         ring->mmio_base = VEBOX_RING_BASE;
1508         ring->irq_enable_mask =
1509                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
1510         ring->irq_keep_mask =
1511                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
1512
1513         ring->init_hw = gen8_init_common_ring;
1514         ring->get_seqno = gen8_get_seqno;
1515         ring->set_seqno = gen8_set_seqno;
1516         ring->emit_request = gen8_emit_request;
1517         ring->emit_flush = gen8_emit_flush;
1518         ring->irq_get = gen8_logical_ring_get_irq;
1519         ring->irq_put = gen8_logical_ring_put_irq;
1520         ring->emit_bb_start = gen8_emit_bb_start;
1521
1522         return logical_ring_init(dev, ring);
1523 }
1524
1525 /**
1526  * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers
1527  * @dev: DRM device.
1528  *
1529  * This function inits the engines for an Execlists submission style (the equivalent in the
1530  * legacy ringbuffer submission world would be i915_gem_init_rings). It does it only for
1531  * those engines that are present in the hardware.
1532  *
1533  * Return: non-zero if the initialization failed.
1534  */
1535 int intel_logical_rings_init(struct drm_device *dev)
1536 {
1537         struct drm_i915_private *dev_priv = dev->dev_private;
1538         int ret;
1539
1540         ret = logical_render_ring_init(dev);
1541         if (ret)
1542                 return ret;
1543
1544         if (HAS_BSD(dev)) {
1545                 ret = logical_bsd_ring_init(dev);
1546                 if (ret)
1547                         goto cleanup_render_ring;
1548         }
1549
1550         if (HAS_BLT(dev)) {
1551                 ret = logical_blt_ring_init(dev);
1552                 if (ret)
1553                         goto cleanup_bsd_ring;
1554         }
1555
1556         if (HAS_VEBOX(dev)) {
1557                 ret = logical_vebox_ring_init(dev);
1558                 if (ret)
1559                         goto cleanup_blt_ring;
1560         }
1561
1562         if (HAS_BSD2(dev)) {
1563                 ret = logical_bsd2_ring_init(dev);
1564                 if (ret)
1565                         goto cleanup_vebox_ring;
1566         }
1567
1568         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
1569         if (ret)
1570                 goto cleanup_bsd2_ring;
1571
1572         return 0;
1573
1574 cleanup_bsd2_ring:
1575         intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
1576 cleanup_vebox_ring:
1577         intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
1578 cleanup_blt_ring:
1579         intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
1580 cleanup_bsd_ring:
1581         intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
1582 cleanup_render_ring:
1583         intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
1584
1585         return ret;
1586 }
1587
1588 static u32
1589 make_rpcs(struct drm_device *dev)
1590 {
1591         u32 rpcs = 0;
1592
1593         /*
1594          * No explicit RPCS request is needed to ensure full
1595          * slice/subslice/EU enablement prior to Gen9.
1596         */
1597         if (INTEL_INFO(dev)->gen < 9)
1598                 return 0;
1599
1600         /*
1601          * Starting in Gen9, render power gating can leave
1602          * slice/subslice/EU in a partially enabled state. We
1603          * must make an explicit request through RPCS for full
1604          * enablement.
1605         */
1606         if (INTEL_INFO(dev)->has_slice_pg) {
1607                 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
1608                 rpcs |= INTEL_INFO(dev)->slice_total <<
1609                         GEN8_RPCS_S_CNT_SHIFT;
1610                 rpcs |= GEN8_RPCS_ENABLE;
1611         }
1612
1613         if (INTEL_INFO(dev)->has_subslice_pg) {
1614                 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
1615                 rpcs |= INTEL_INFO(dev)->subslice_per_slice <<
1616                         GEN8_RPCS_SS_CNT_SHIFT;
1617                 rpcs |= GEN8_RPCS_ENABLE;
1618         }
1619
1620         if (INTEL_INFO(dev)->has_eu_pg) {
1621                 rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
1622                         GEN8_RPCS_EU_MIN_SHIFT;
1623                 rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
1624                         GEN8_RPCS_EU_MAX_SHIFT;
1625                 rpcs |= GEN8_RPCS_ENABLE;
1626         }
1627
1628         return rpcs;
1629 }
1630
1631 static int
1632 populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
1633                     struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
1634 {
1635         struct drm_device *dev = ring->dev;
1636         struct drm_i915_private *dev_priv = dev->dev_private;
1637         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
1638         struct page *page;
1639         uint32_t *reg_state;
1640         int ret;
1641
1642         if (!ppgtt)
1643                 ppgtt = dev_priv->mm.aliasing_ppgtt;
1644
1645         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
1646         if (ret) {
1647                 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
1648                 return ret;
1649         }
1650
1651         ret = i915_gem_object_get_pages(ctx_obj);
1652         if (ret) {
1653                 DRM_DEBUG_DRIVER("Could not get object pages\n");
1654                 return ret;
1655         }
1656
1657         i915_gem_object_pin_pages(ctx_obj);
1658
1659         /* The second page of the context object contains some fields which must
1660          * be set up prior to the first execution. */
1661         page = i915_gem_object_get_page(ctx_obj, 1);
1662         reg_state = kmap_atomic(page);
1663
1664         /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
1665          * commands followed by (reg, value) pairs. The values we are setting here are
1666          * only for the first context restore: on a subsequent save, the GPU will
1667          * recreate this batchbuffer with new values (including all the missing
1668          * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
1669         if (ring->id == RCS)
1670                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
1671         else
1672                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
1673         reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
1674         reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
1675         reg_state[CTX_CONTEXT_CONTROL+1] =
1676                 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
1677                                 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
1678         reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
1679         reg_state[CTX_RING_HEAD+1] = 0;
1680         reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
1681         reg_state[CTX_RING_TAIL+1] = 0;
1682         reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
1683         /* Ring buffer start address is not known until the buffer is pinned.
1684          * It is written to the context image in execlists_update_context()
1685          */
1686         reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
1687         reg_state[CTX_RING_BUFFER_CONTROL+1] =
1688                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
1689         reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
1690         reg_state[CTX_BB_HEAD_U+1] = 0;
1691         reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
1692         reg_state[CTX_BB_HEAD_L+1] = 0;
1693         reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
1694         reg_state[CTX_BB_STATE+1] = (1<<5);
1695         reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
1696         reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
1697         reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
1698         reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
1699         reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
1700         reg_state[CTX_SECOND_BB_STATE+1] = 0;
1701         if (ring->id == RCS) {
1702                 /* TODO: according to BSpec, the register state context
1703                  * for CHV does not have these. OTOH, these registers do
1704                  * exist in CHV. I'm waiting for a clarification */
1705                 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
1706                 reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
1707                 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
1708                 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
1709                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
1710                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
1711         }
1712         reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
1713         reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
1714         reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
1715         reg_state[CTX_CTX_TIMESTAMP+1] = 0;
1716         reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
1717         reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
1718         reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
1719         reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
1720         reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
1721         reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
1722         reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
1723         reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
1724
1725         /* With dynamic page allocation, PDPs may not be allocated at this point,
1726          * Point the unallocated PDPs to the scratch page
1727          */
1728         ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
1729         ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
1730         ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
1731         ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
1732         if (ring->id == RCS) {
1733                 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
1734                 reg_state[CTX_R_PWR_CLK_STATE] = GEN8_R_PWR_CLK_STATE;
1735                 reg_state[CTX_R_PWR_CLK_STATE+1] = make_rpcs(dev);
1736         }
1737
1738         kunmap_atomic(reg_state);
1739
1740         ctx_obj->dirty = 1;
1741         set_page_dirty(page);
1742         i915_gem_object_unpin_pages(ctx_obj);
1743
1744         return 0;
1745 }
1746
1747 /**
1748  * intel_lr_context_free() - free the LRC specific bits of a context
1749  * @ctx: the LR context to free.
1750  *
1751  * The real context freeing is done in i915_gem_context_free: this only
1752  * takes care of the bits that are LRC related: the per-engine backing
1753  * objects and the logical ringbuffer.
1754  */
1755 void intel_lr_context_free(struct intel_context *ctx)
1756 {
1757         int i;
1758
1759         for (i = 0; i < I915_NUM_RINGS; i++) {
1760                 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
1761
1762                 if (ctx_obj) {
1763                         struct intel_ringbuffer *ringbuf =
1764                                         ctx->engine[i].ringbuf;
1765                         struct intel_engine_cs *ring = ringbuf->ring;
1766
1767                         if (ctx == ring->default_context) {
1768                                 intel_unpin_ringbuffer_obj(ringbuf);
1769                                 i915_gem_object_ggtt_unpin(ctx_obj);
1770                         }
1771                         WARN_ON(ctx->engine[ring->id].pin_count);
1772                         intel_destroy_ringbuffer_obj(ringbuf);
1773                         kfree(ringbuf);
1774                         drm_gem_object_unreference(&ctx_obj->base);
1775                 }
1776         }
1777 }
1778
1779 static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
1780 {
1781         int ret = 0;
1782
1783         WARN_ON(INTEL_INFO(ring->dev)->gen < 8);
1784
1785         switch (ring->id) {
1786         case RCS:
1787                 if (INTEL_INFO(ring->dev)->gen >= 9)
1788                         ret = GEN9_LR_CONTEXT_RENDER_SIZE;
1789                 else
1790                         ret = GEN8_LR_CONTEXT_RENDER_SIZE;
1791                 break;
1792         case VCS:
1793         case BCS:
1794         case VECS:
1795         case VCS2:
1796                 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
1797                 break;
1798         }
1799
1800         return ret;
1801 }
1802
1803 static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring,
1804                 struct drm_i915_gem_object *default_ctx_obj)
1805 {
1806         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1807
1808         /* The status page is offset 0 from the default context object
1809          * in LRC mode. */
1810         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj);
1811         ring->status_page.page_addr =
1812                         kmap(sg_page(default_ctx_obj->pages->sgl));
1813         ring->status_page.obj = default_ctx_obj;
1814
1815         I915_WRITE(RING_HWS_PGA(ring->mmio_base),
1816                         (u32)ring->status_page.gfx_addr);
1817         POSTING_READ(RING_HWS_PGA(ring->mmio_base));
1818 }
1819
1820 /**
1821  * intel_lr_context_deferred_create() - create the LRC specific bits of a context
1822  * @ctx: LR context to create.
1823  * @ring: engine to be used with the context.
1824  *
1825  * This function can be called more than once, with different engines, if we plan
1826  * to use the context with them. The context backing objects and the ringbuffers
1827  * (specially the ringbuffer backing objects) suck a lot of memory up, and that's why
1828  * the creation is a deferred call: it's better to make sure first that we need to use
1829  * a given ring with the context.
1830  *
1831  * Return: non-zero on error.
1832  */
1833 int intel_lr_context_deferred_create(struct intel_context *ctx,
1834                                      struct intel_engine_cs *ring)
1835 {
1836         const bool is_global_default_ctx = (ctx == ring->default_context);
1837         struct drm_device *dev = ring->dev;
1838         struct drm_i915_gem_object *ctx_obj;
1839         uint32_t context_size;
1840         struct intel_ringbuffer *ringbuf;
1841         int ret;
1842
1843         WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
1844         WARN_ON(ctx->engine[ring->id].state);
1845
1846         context_size = round_up(get_lr_context_size(ring), 4096);
1847
1848         ctx_obj = i915_gem_alloc_object(dev, context_size);
1849         if (IS_ERR(ctx_obj)) {
1850                 ret = PTR_ERR(ctx_obj);
1851                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret);
1852                 return ret;
1853         }
1854
1855         if (is_global_default_ctx) {
1856                 ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
1857                 if (ret) {
1858                         DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n",
1859                                         ret);
1860                         drm_gem_object_unreference(&ctx_obj->base);
1861                         return ret;
1862                 }
1863         }
1864
1865         ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
1866         if (!ringbuf) {
1867                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
1868                                 ring->name);
1869                 ret = -ENOMEM;
1870                 goto error_unpin_ctx;
1871         }
1872
1873         ringbuf->ring = ring;
1874
1875         ringbuf->size = 32 * PAGE_SIZE;
1876         ringbuf->effective_size = ringbuf->size;
1877         ringbuf->head = 0;
1878         ringbuf->tail = 0;
1879         ringbuf->last_retired_head = -1;
1880         intel_ring_update_space(ringbuf);
1881
1882         if (ringbuf->obj == NULL) {
1883                 ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
1884                 if (ret) {
1885                         DRM_DEBUG_DRIVER(
1886                                 "Failed to allocate ringbuffer obj %s: %d\n",
1887                                 ring->name, ret);
1888                         goto error_free_rbuf;
1889                 }
1890
1891                 if (is_global_default_ctx) {
1892                         ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
1893                         if (ret) {
1894                                 DRM_ERROR(
1895                                         "Failed to pin and map ringbuffer %s: %d\n",
1896                                         ring->name, ret);
1897                                 goto error_destroy_rbuf;
1898                         }
1899                 }
1900
1901         }
1902
1903         ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
1904         if (ret) {
1905                 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
1906                 goto error;
1907         }
1908
1909         ctx->engine[ring->id].ringbuf = ringbuf;
1910         ctx->engine[ring->id].state = ctx_obj;
1911
1912         if (ctx == ring->default_context)
1913                 lrc_setup_hardware_status_page(ring, ctx_obj);
1914         else if (ring->id == RCS && !ctx->rcs_initialized) {
1915                 if (ring->init_context) {
1916                         ret = ring->init_context(ring, ctx);
1917                         if (ret) {
1918                                 DRM_ERROR("ring init context: %d\n", ret);
1919                                 ctx->engine[ring->id].ringbuf = NULL;
1920                                 ctx->engine[ring->id].state = NULL;
1921                                 goto error;
1922                         }
1923                 }
1924
1925                 ctx->rcs_initialized = true;
1926         }
1927
1928         return 0;
1929
1930 error:
1931         if (is_global_default_ctx)
1932                 intel_unpin_ringbuffer_obj(ringbuf);
1933 error_destroy_rbuf:
1934         intel_destroy_ringbuffer_obj(ringbuf);
1935 error_free_rbuf:
1936         kfree(ringbuf);
1937 error_unpin_ctx:
1938         if (is_global_default_ctx)
1939                 i915_gem_object_ggtt_unpin(ctx_obj);
1940         drm_gem_object_unreference(&ctx_obj->base);
1941         return ret;
1942 }
1943
1944 void intel_lr_context_reset(struct drm_device *dev,
1945                         struct intel_context *ctx)
1946 {
1947         struct drm_i915_private *dev_priv = dev->dev_private;
1948         struct intel_engine_cs *ring;
1949         int i;
1950
1951         for_each_ring(ring, dev_priv, i) {
1952                 struct drm_i915_gem_object *ctx_obj =
1953                                 ctx->engine[ring->id].state;
1954                 struct intel_ringbuffer *ringbuf =
1955                                 ctx->engine[ring->id].ringbuf;
1956                 uint32_t *reg_state;
1957                 struct page *page;
1958
1959                 if (!ctx_obj)
1960                         continue;
1961
1962                 if (i915_gem_object_get_pages(ctx_obj)) {
1963                         WARN(1, "Failed get_pages for context obj\n");
1964                         continue;
1965                 }
1966                 page = i915_gem_object_get_page(ctx_obj, 1);
1967                 reg_state = kmap_atomic(page);
1968
1969                 reg_state[CTX_RING_HEAD+1] = 0;
1970                 reg_state[CTX_RING_TAIL+1] = 0;
1971
1972                 kunmap_atomic(reg_state);
1973
1974                 ringbuf->head = 0;
1975                 ringbuf->tail = 0;
1976         }
1977 }