drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  * Copyright © 2011-2014 Intel Corporation
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/seq_file.h>
  27 #include <linux/stop_machine.h>
  28 #include <drm/drmP.h>
  29 #include <drm/i915_drm.h>
  30 #include "i915_drv.h"
  31 #include "i915_vgpu.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include "intel_frontbuffer.h"
  35
  36 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
  37
  38 /**
  39  * DOC: Global GTT views
  40  *
  41  * Background and previous state
  42  *
  43  * Historically objects could exists (be bound) in global GTT space only as
  44  * singular instances with a view representing all of the object's backing pages
  45  * in a linear fashion. This view will be called a normal view.
  46  *
  47  * To support multiple views of the same object, where the number of mapped
  48  * pages is not equal to the backing store, or where the layout of the pages
  49  * is not linear, concept of a GGTT view was added.
  50  *
  51  * One example of an alternative view is a stereo display driven by a single
  52  * image. In this case we would have a framebuffer looking like this
  53  * (2x2 pages):
  54  *
  55  *    12
  56  *    34
  57  *
  58  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
  59  * rendering. In contrast, fed to the display engine would be an alternative
  60  * view which could look something like this:
  61  *
  62  *   1212
  63  *   3434
  64  *
  65  * In this example both the size and layout of pages in the alternative view is
  66  * different from the normal view.
  67  *
  68  * Implementation and usage
  69  *
  70  * GGTT views are implemented using VMAs and are distinguished via enum
  71  * i915_ggtt_view_type and struct i915_ggtt_view.
  72  *
  73  * A new flavour of core GEM functions which work with GGTT bound objects were
  74  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
  75  * renaming  in large amounts of code. They take the struct i915_ggtt_view
  76  * parameter encapsulating all metadata required to implement a view.
  77  *
  78  * As a helper for callers which are only interested in the normal view,
  79  * globally const i915_ggtt_view_normal singleton instance exists. All old core
  80  * GEM API functions, the ones not taking the view parameter, are operating on,
  81  * or with the normal GGTT view.
  82  *
  83  * Code wanting to add or use a new GGTT view needs to:
  84  *
  85  * 1. Add a new enum with a suitable name.
  86  * 2. Extend the metadata in the i915_ggtt_view structure if required.
  87  * 3. Add support to i915_get_vma_pages().
  88  *
  89  * New views are required to build a scatter-gather table from within the
  90  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
  91  * exists for the lifetime of an VMA.
  92  *
  93  * Core API is designed to have copy semantics which means that passed in
  94  * struct i915_ggtt_view does not need to be persistent (left around after
  95  * calling the core API functions).
  96  *
  97  */
  98
  99 static int
 100 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 101
 102 const struct i915_ggtt_view i915_ggtt_view_normal = {
 103         .type = I915_GGTT_VIEW_NORMAL,
 104 };
 105 const struct i915_ggtt_view i915_ggtt_view_rotated = {
 106         .type = I915_GGTT_VIEW_ROTATED,
 107 };
 108
 109 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 110                                 int enable_ppgtt)
 111 {
 112         bool has_aliasing_ppgtt;
 113         bool has_full_ppgtt;
 114         bool has_full_48bit_ppgtt;
 115
 116         has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt;
 117         has_full_ppgtt = dev_priv->info.has_full_ppgtt;
 118         has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
 119
 120         if (intel_vgpu_active(dev_priv)) {
 121                 /* emulation is too hard */
 122                 has_full_ppgtt = false;
 123                 has_full_48bit_ppgtt = false;
 124         }
 125
 126         if (!has_aliasing_ppgtt)
 127                 return 0;
 128
 129         /*
 130          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
 131          * execlists, the sole mechanism available to submit work.
 132          */
 133         if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
 134                 return 0;
 135
 136         if (enable_ppgtt == 1)
 137                 return 1;
 138
 139         if (enable_ppgtt == 2 && has_full_ppgtt)
 140                 return 2;
 141
 142         if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
 143                 return 3;
 144
 145 #ifdef CONFIG_INTEL_IOMMU
 146         /* Disable ppgtt on SNB if VT-d is on. */
 147         if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
 148                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
 149                 return 0;
 150         }
 151 #endif
 152
 153         /* Early VLV doesn't have this */
 154         if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
 155                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
 156                 return 0;
 157         }
 158
 159         if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
 160                 return has_full_48bit_ppgtt ? 3 : 2;
 161         else
 162                 return has_aliasing_ppgtt ? 1 : 0;
 163 }
 164
 165 static int ppgtt_bind_vma(struct i915_vma *vma,
 166                           enum i915_cache_level cache_level,
 167                           u32 unused)
 168 {
 169         u32 pte_flags = 0;
 170
 171         vma->pages = vma->obj->mm.pages;
 172
 173         /* Currently applicable only to VLV */
 174         if (vma->obj->gt_ro)
 175                 pte_flags |= PTE_READ_ONLY;
 176
 177         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 178                                 cache_level, pte_flags);
 179
 180         return 0;
 181 }
 182
 183 static void ppgtt_unbind_vma(struct i915_vma *vma)
 184 {
 185         vma->vm->clear_range(vma->vm,
 186                              vma->node.start,
 187                              vma->size);
 188 }
 189
 190 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
 191                                   enum i915_cache_level level)
 192 {
 193         gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
 194         pte |= addr;
 195
 196         switch (level) {
 197         case I915_CACHE_NONE:
 198                 pte |= PPAT_UNCACHED_INDEX;
 199                 break;
 200         case I915_CACHE_WT:
 201                 pte |= PPAT_DISPLAY_ELLC_INDEX;
 202                 break;
 203         default:
 204                 pte |= PPAT_CACHED_INDEX;
 205                 break;
 206         }
 207
 208         return pte;
 209 }
 210
 211 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 212                                   const enum i915_cache_level level)
 213 {
 214         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
 215         pde |= addr;
 216         if (level != I915_CACHE_NONE)
 217                 pde |= PPAT_CACHED_PDE_INDEX;
 218         else
 219                 pde |= PPAT_UNCACHED_INDEX;
 220         return pde;
 221 }
 222
 223 #define gen8_pdpe_encode gen8_pde_encode
 224 #define gen8_pml4e_encode gen8_pde_encode
 225
 226 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 227                                  enum i915_cache_level level,
 228                                  u32 unused)
 229 {
 230         gen6_pte_t pte = GEN6_PTE_VALID;
 231         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 232
 233         switch (level) {
 234         case I915_CACHE_L3_LLC:
 235         case I915_CACHE_LLC:
 236                 pte |= GEN6_PTE_CACHE_LLC;
 237                 break;
 238         case I915_CACHE_NONE:
 239                 pte |= GEN6_PTE_UNCACHED;
 240                 break;
 241         default:
 242                 MISSING_CASE(level);
 243         }
 244
 245         return pte;
 246 }
 247
 248 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 249                                  enum i915_cache_level level,
 250                                  u32 unused)
 251 {
 252         gen6_pte_t pte = GEN6_PTE_VALID;
 253         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 254
 255         switch (level) {
 256         case I915_CACHE_L3_LLC:
 257                 pte |= GEN7_PTE_CACHE_L3_LLC;
 258                 break;
 259         case I915_CACHE_LLC:
 260                 pte |= GEN6_PTE_CACHE_LLC;
 261                 break;
 262         case I915_CACHE_NONE:
 263                 pte |= GEN6_PTE_UNCACHED;
 264                 break;
 265         default:
 266                 MISSING_CASE(level);
 267         }
 268
 269         return pte;
 270 }
 271
 272 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 273                                  enum i915_cache_level level,
 274                                  u32 flags)
 275 {
 276         gen6_pte_t pte = GEN6_PTE_VALID;
 277         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 278
 279         if (!(flags & PTE_READ_ONLY))
 280                 pte |= BYT_PTE_WRITEABLE;
 281
 282         if (level != I915_CACHE_NONE)
 283                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 284
 285         return pte;
 286 }
 287
 288 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 289                                  enum i915_cache_level level,
 290                                  u32 unused)
 291 {
 292         gen6_pte_t pte = GEN6_PTE_VALID;
 293         pte |= HSW_PTE_ADDR_ENCODE(addr);
 294
 295         if (level != I915_CACHE_NONE)
 296                 pte |= HSW_WB_LLC_AGE3;
 297
 298         return pte;
 299 }
 300
 301 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 302                                   enum i915_cache_level level,
 303                                   u32 unused)
 304 {
 305         gen6_pte_t pte = GEN6_PTE_VALID;
 306         pte |= HSW_PTE_ADDR_ENCODE(addr);
 307
 308         switch (level) {
 309         case I915_CACHE_NONE:
 310                 break;
 311         case I915_CACHE_WT:
 312                 pte |= HSW_WT_ELLC_LLC_AGE3;
 313                 break;
 314         default:
 315                 pte |= HSW_WB_ELLC_LLC_AGE3;
 316                 break;
 317         }
 318
 319         return pte;
 320 }
 321
 322 static int __setup_page_dma(struct drm_i915_private *dev_priv,
 323                             struct i915_page_dma *p, gfp_t flags)
 324 {
 325         struct device *kdev = &dev_priv->drm.pdev->dev;
 326
 327         p->page = alloc_page(flags);
 328         if (!p->page)
 329                 return -ENOMEM;
 330
 331         p->daddr = dma_map_page(kdev,
 332                                 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
 333
 334         if (dma_mapping_error(kdev, p->daddr)) {
 335                 __free_page(p->page);
 336                 return -EINVAL;
 337         }
 338
 339         return 0;
 340 }
 341
 342 static int setup_page_dma(struct drm_i915_private *dev_priv,
 343                           struct i915_page_dma *p)
 344 {
 345         return __setup_page_dma(dev_priv, p, I915_GFP_DMA);
 346 }
 347
 348 static void cleanup_page_dma(struct drm_i915_private *dev_priv,
 349                              struct i915_page_dma *p)
 350 {
 351         struct pci_dev *pdev = dev_priv->drm.pdev;
 352
 353         if (WARN_ON(!p->page))
 354                 return;
 355
 356         dma_unmap_page(&pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 357         __free_page(p->page);
 358         memset(p, 0, sizeof(*p));
 359 }
 360
 361 static void *kmap_page_dma(struct i915_page_dma *p)
 362 {
 363         return kmap_atomic(p->page);
 364 }
 365
 366 /* We use the flushing unmap only with ppgtt structures:
 367  * page directories, page tables and scratch pages.
 368  */
 369 static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr)
 370 {
 371         /* There are only few exceptions for gen >=6. chv and bxt.
 372          * And we are not sure about the latter so play safe for now.
 373          */
 374         if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
 375                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 376
 377         kunmap_atomic(vaddr);
 378 }
 379
 380 #define kmap_px(px) kmap_page_dma(px_base(px))
 381 #define kunmap_px(ppgtt, vaddr) \
 382                 kunmap_page_dma((ppgtt)->base.i915, (vaddr))
 383
 384 #define setup_px(dev_priv, px) setup_page_dma((dev_priv), px_base(px))
 385 #define cleanup_px(dev_priv, px) cleanup_page_dma((dev_priv), px_base(px))
 386 #define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v))
 387 #define fill32_px(dev_priv, px, v) \
 388                 fill_page_dma_32((dev_priv), px_base(px), (v))
 389
 390 static void fill_page_dma(struct drm_i915_private *dev_priv,
 391                           struct i915_page_dma *p, const uint64_t val)
 392 {
 393         int i;
 394         uint64_t * const vaddr = kmap_page_dma(p);
 395
 396         for (i = 0; i < 512; i++)
 397                 vaddr[i] = val;
 398
 399         kunmap_page_dma(dev_priv, vaddr);
 400 }
 401
 402 static void fill_page_dma_32(struct drm_i915_private *dev_priv,
 403                              struct i915_page_dma *p, const uint32_t val32)
 404 {
 405         uint64_t v = val32;
 406
 407         v = v << 32 | val32;
 408
 409         fill_page_dma(dev_priv, p, v);
 410 }
 411
 412 static int
 413 setup_scratch_page(struct drm_i915_private *dev_priv,
 414                    struct i915_page_dma *scratch,
 415                    gfp_t gfp)
 416 {
 417         return __setup_page_dma(dev_priv, scratch, gfp | __GFP_ZERO);
 418 }
 419
 420 static void cleanup_scratch_page(struct drm_i915_private *dev_priv,
 421                                  struct i915_page_dma *scratch)
 422 {
 423         cleanup_page_dma(dev_priv, scratch);
 424 }
 425
 426 static struct i915_page_table *alloc_pt(struct drm_i915_private *dev_priv)
 427 {
 428         struct i915_page_table *pt;
 429         const size_t count = INTEL_GEN(dev_priv) >= 8 ? GEN8_PTES : GEN6_PTES;
 430         int ret = -ENOMEM;
 431
 432         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 433         if (!pt)
 434                 return ERR_PTR(-ENOMEM);
 435
 436         pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
 437                                 GFP_KERNEL);
 438
 439         if (!pt->used_ptes)
 440                 goto fail_bitmap;
 441
 442         ret = setup_px(dev_priv, pt);
 443         if (ret)
 444                 goto fail_page_m;
 445
 446         return pt;
 447
 448 fail_page_m:
 449         kfree(pt->used_ptes);
 450 fail_bitmap:
 451         kfree(pt);
 452
 453         return ERR_PTR(ret);
 454 }
 455
 456 static void free_pt(struct drm_i915_private *dev_priv,
 457                     struct i915_page_table *pt)
 458 {
 459         cleanup_px(dev_priv, pt);
 460         kfree(pt->used_ptes);
 461         kfree(pt);
 462 }
 463
 464 static void gen8_initialize_pt(struct i915_address_space *vm,
 465                                struct i915_page_table *pt)
 466 {
 467         gen8_pte_t scratch_pte;
 468
 469         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 470                                       I915_CACHE_LLC);
 471
 472         fill_px(vm->i915, pt, scratch_pte);
 473 }
 474
 475 static void gen6_initialize_pt(struct i915_address_space *vm,
 476                                struct i915_page_table *pt)
 477 {
 478         gen6_pte_t scratch_pte;
 479
 480         WARN_ON(vm->scratch_page.daddr == 0);
 481
 482         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
 483                                      I915_CACHE_LLC, 0);
 484
 485         fill32_px(vm->i915, pt, scratch_pte);
 486 }
 487
 488 static struct i915_page_directory *alloc_pd(struct drm_i915_private *dev_priv)
 489 {
 490         struct i915_page_directory *pd;
 491         int ret = -ENOMEM;
 492
 493         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 494         if (!pd)
 495                 return ERR_PTR(-ENOMEM);
 496
 497         pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
 498                                 sizeof(*pd->used_pdes), GFP_KERNEL);
 499         if (!pd->used_pdes)
 500                 goto fail_bitmap;
 501
 502         ret = setup_px(dev_priv, pd);
 503         if (ret)
 504                 goto fail_page_m;
 505
 506         return pd;
 507
 508 fail_page_m:
 509         kfree(pd->used_pdes);
 510 fail_bitmap:
 511         kfree(pd);
 512
 513         return ERR_PTR(ret);
 514 }
 515
 516 static void free_pd(struct drm_i915_private *dev_priv,
 517                     struct i915_page_directory *pd)
 518 {
 519         if (px_page(pd)) {
 520                 cleanup_px(dev_priv, pd);
 521                 kfree(pd->used_pdes);
 522                 kfree(pd);
 523         }
 524 }
 525
 526 static void gen8_initialize_pd(struct i915_address_space *vm,
 527                                struct i915_page_directory *pd)
 528 {
 529         gen8_pde_t scratch_pde;
 530
 531         scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
 532
 533         fill_px(vm->i915, pd, scratch_pde);
 534 }
 535
 536 static int __pdp_init(struct drm_i915_private *dev_priv,
 537                       struct i915_page_directory_pointer *pdp)
 538 {
 539         size_t pdpes = I915_PDPES_PER_PDP(dev_priv);
 540
 541         pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
 542                                   sizeof(unsigned long),
 543                                   GFP_KERNEL);
 544         if (!pdp->used_pdpes)
 545                 return -ENOMEM;
 546
 547         pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
 548                                       GFP_KERNEL);
 549         if (!pdp->page_directory) {
 550                 kfree(pdp->used_pdpes);
 551                 /* the PDP might be the statically allocated top level. Keep it
 552                  * as clean as possible */
 553                 pdp->used_pdpes = NULL;
 554                 return -ENOMEM;
 555         }
 556
 557         return 0;
 558 }
 559
 560 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 561 {
 562         kfree(pdp->used_pdpes);
 563         kfree(pdp->page_directory);
 564         pdp->page_directory = NULL;
 565 }
 566
 567 static struct
 568 i915_page_directory_pointer *alloc_pdp(struct drm_i915_private *dev_priv)
 569 {
 570         struct i915_page_directory_pointer *pdp;
 571         int ret = -ENOMEM;
 572
 573         WARN_ON(!USES_FULL_48BIT_PPGTT(dev_priv));
 574
 575         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 576         if (!pdp)
 577                 return ERR_PTR(-ENOMEM);
 578
 579         ret = __pdp_init(dev_priv, pdp);
 580         if (ret)
 581                 goto fail_bitmap;
 582
 583         ret = setup_px(dev_priv, pdp);
 584         if (ret)
 585                 goto fail_page_m;
 586
 587         return pdp;
 588
 589 fail_page_m:
 590         __pdp_fini(pdp);
 591 fail_bitmap:
 592         kfree(pdp);
 593
 594         return ERR_PTR(ret);
 595 }
 596
 597 static void free_pdp(struct drm_i915_private *dev_priv,
 598                      struct i915_page_directory_pointer *pdp)
 599 {
 600         __pdp_fini(pdp);
 601         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
 602                 cleanup_px(dev_priv, pdp);
 603                 kfree(pdp);
 604         }
 605 }
 606
 607 static void gen8_initialize_pdp(struct i915_address_space *vm,
 608                                 struct i915_page_directory_pointer *pdp)
 609 {
 610         gen8_ppgtt_pdpe_t scratch_pdpe;
 611
 612         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 613
 614         fill_px(vm->i915, pdp, scratch_pdpe);
 615 }
 616
 617 static void gen8_initialize_pml4(struct i915_address_space *vm,
 618                                  struct i915_pml4 *pml4)
 619 {
 620         gen8_ppgtt_pml4e_t scratch_pml4e;
 621
 622         scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
 623                                           I915_CACHE_LLC);
 624
 625         fill_px(vm->i915, pml4, scratch_pml4e);
 626 }
 627
 628 static void
 629 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
 630                           struct i915_page_directory_pointer *pdp,
 631                           struct i915_page_directory *pd,
 632                           int index)
 633 {
 634         gen8_ppgtt_pdpe_t *page_directorypo;
 635
 636         if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)))
 637                 return;
 638
 639         page_directorypo = kmap_px(pdp);
 640         page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
 641         kunmap_px(ppgtt, page_directorypo);
 642 }
 643
 644 static void
 645 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
 646                                   struct i915_pml4 *pml4,
 647                                   struct i915_page_directory_pointer *pdp,
 648                                   int index)
 649 {
 650         gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
 651
 652         WARN_ON(!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)));
 653         pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 654         kunmap_px(ppgtt, pagemap);
 655 }
 656
 657 /* Broadwell Page Directory Pointer Descriptors */
 658 static int gen8_write_pdp(struct drm_i915_gem_request *req,
 659                           unsigned entry,
 660                           dma_addr_t addr)
 661 {
 662         struct intel_ring *ring = req->ring;
 663         struct intel_engine_cs *engine = req->engine;
 664         int ret;
 665
 666         BUG_ON(entry >= 4);
 667
 668         ret = intel_ring_begin(req, 6);
 669         if (ret)
 670                 return ret;
 671
 672         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 673         intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(engine, entry));
 674         intel_ring_emit(ring, upper_32_bits(addr));
 675         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 676         intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(engine, entry));
 677         intel_ring_emit(ring, lower_32_bits(addr));
 678         intel_ring_advance(ring);
 679
 680         return 0;
 681 }
 682
 683 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
 684                                  struct drm_i915_gem_request *req)
 685 {
 686         int i, ret;
 687
 688         for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
 689                 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
 690
 691                 ret = gen8_write_pdp(req, i, pd_daddr);
 692                 if (ret)
 693                         return ret;
 694         }
 695
 696         return 0;
 697 }
 698
 699 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
 700                               struct drm_i915_gem_request *req)
 701 {
 702         return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 703 }
 704
 705 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
 706  * the page table structures, we mark them dirty so that
 707  * context switching/execlist queuing code takes extra steps
 708  * to ensure that tlbs are flushed.
 709  */
 710 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 711 {
 712         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
 713 }
 714
 715 /* Removes entries from a single page table, releasing it if it's empty.
 716  * Caller can use the return value to update higher-level entries.
 717  */
 718 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
 719                                 struct i915_page_table *pt,
 720                                 uint64_t start,
 721                                 uint64_t length)
 722 {
 723         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 724         unsigned int num_entries = gen8_pte_count(start, length);
 725         unsigned int pte = gen8_pte_index(start);
 726         unsigned int pte_end = pte + num_entries;
 727         gen8_pte_t *pt_vaddr;
 728         gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
 729                                                  I915_CACHE_LLC);
 730
 731         if (WARN_ON(!px_page(pt)))
 732                 return false;
 733
 734         GEM_BUG_ON(pte_end > GEN8_PTES);
 735
 736         bitmap_clear(pt->used_ptes, pte, num_entries);
 737
 738         if (bitmap_empty(pt->used_ptes, GEN8_PTES))
 739                 return true;
 740
 741         pt_vaddr = kmap_px(pt);
 742
 743         while (pte < pte_end)
 744                 pt_vaddr[pte++] = scratch_pte;
 745
 746         kunmap_px(ppgtt, pt_vaddr);
 747
 748         return false;
 749 }
 750
 751 /* Removes entries from a single page dir, releasing it if it's empty.
 752  * Caller can use the return value to update higher-level entries
 753  */
 754 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 755                                 struct i915_page_directory *pd,
 756                                 uint64_t start,
 757                                 uint64_t length)
 758 {
 759         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 760         struct i915_page_table *pt;
 761         uint64_t pde;
 762         gen8_pde_t *pde_vaddr;
 763         gen8_pde_t scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt),
 764                                                  I915_CACHE_LLC);
 765
 766         gen8_for_each_pde(pt, pd, start, length, pde) {
 767                 if (WARN_ON(!pd->page_table[pde]))
 768                         break;
 769
 770                 if (gen8_ppgtt_clear_pt(vm, pt, start, length)) {
 771                         __clear_bit(pde, pd->used_pdes);
 772                         pde_vaddr = kmap_px(pd);
 773                         pde_vaddr[pde] = scratch_pde;
 774                         kunmap_px(ppgtt, pde_vaddr);
 775                         free_pt(vm->i915, pt);
 776                 }
 777         }
 778
 779         if (bitmap_empty(pd->used_pdes, I915_PDES))
 780                 return true;
 781
 782         return false;
 783 }
 784
 785 /* Removes entries from a single page dir pointer, releasing it if it's empty.
 786  * Caller can use the return value to update higher-level entries
 787  */
 788 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 789                                  struct i915_page_directory_pointer *pdp,
 790                                  uint64_t start,
 791                                  uint64_t length)
 792 {
 793         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 794         struct i915_page_directory *pd;
 795         uint64_t pdpe;
 796         gen8_ppgtt_pdpe_t *pdpe_vaddr;
 797         gen8_ppgtt_pdpe_t scratch_pdpe =
 798                 gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 799
 800         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 801                 if (WARN_ON(!pdp->page_directory[pdpe]))
 802                         break;
 803
 804                 if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
 805                         __clear_bit(pdpe, pdp->used_pdpes);
 806                         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
 807                                 pdpe_vaddr = kmap_px(pdp);
 808                                 pdpe_vaddr[pdpe] = scratch_pdpe;
 809                                 kunmap_px(ppgtt, pdpe_vaddr);
 810                         }
 811                         free_pd(vm->i915, pd);
 812                 }
 813         }
 814
 815         mark_tlbs_dirty(ppgtt);
 816
 817         if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)))
 818                 return true;
 819
 820         return false;
 821 }
 822
 823 /* Removes entries from a single pml4.
 824  * This is the top-level structure in 4-level page tables used on gen8+.
 825  * Empty entries are always scratch pml4e.
 826  */
 827 static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
 828                                   struct i915_pml4 *pml4,
 829                                   uint64_t start,
 830                                   uint64_t length)
 831 {
 832         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 833         struct i915_page_directory_pointer *pdp;
 834         uint64_t pml4e;
 835         gen8_ppgtt_pml4e_t *pml4e_vaddr;
 836         gen8_ppgtt_pml4e_t scratch_pml4e =
 837                 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC);
 838
 839         GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->i915));
 840
 841         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 842                 if (WARN_ON(!pml4->pdps[pml4e]))
 843                         break;
 844
 845                 if (gen8_ppgtt_clear_pdp(vm, pdp, start, length)) {
 846                         __clear_bit(pml4e, pml4->used_pml4es);
 847                         pml4e_vaddr = kmap_px(pml4);
 848                         pml4e_vaddr[pml4e] = scratch_pml4e;
 849                         kunmap_px(ppgtt, pml4e_vaddr);
 850                         free_pdp(vm->i915, pdp);
 851                 }
 852         }
 853 }
 854
 855 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 856                                    uint64_t start, uint64_t length)
 857 {
 858         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 859
 860         if (USES_FULL_48BIT_PPGTT(vm->i915))
 861                 gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
 862         else
 863                 gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
 864 }
 865
 866 static void
 867 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
 868                               struct i915_page_directory_pointer *pdp,
 869                               struct sg_page_iter *sg_iter,
 870                               uint64_t start,
 871                               enum i915_cache_level cache_level)
 872 {
 873         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 874         gen8_pte_t *pt_vaddr;
 875         unsigned pdpe = gen8_pdpe_index(start);
 876         unsigned pde = gen8_pde_index(start);
 877         unsigned pte = gen8_pte_index(start);
 878
 879         pt_vaddr = NULL;
 880
 881         while (__sg_page_iter_next(sg_iter)) {
 882                 if (pt_vaddr == NULL) {
 883                         struct i915_page_directory *pd = pdp->page_directory[pdpe];
 884                         struct i915_page_table *pt = pd->page_table[pde];
 885                         pt_vaddr = kmap_px(pt);
 886                 }
 887
 888                 pt_vaddr[pte] =
 889                         gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
 890                                         cache_level);
 891                 if (++pte == GEN8_PTES) {
 892                         kunmap_px(ppgtt, pt_vaddr);
 893                         pt_vaddr = NULL;
 894                         if (++pde == I915_PDES) {
 895                                 if (++pdpe == I915_PDPES_PER_PDP(vm->i915))
 896                                         break;
 897                                 pde = 0;
 898                         }
 899                         pte = 0;
 900                 }
 901         }
 902
 903         if (pt_vaddr)
 904                 kunmap_px(ppgtt, pt_vaddr);
 905 }
 906
 907 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 908                                       struct sg_table *pages,
 909                                       uint64_t start,
 910                                       enum i915_cache_level cache_level,
 911                                       u32 unused)
 912 {
 913         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 914         struct sg_page_iter sg_iter;
 915
 916         __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
 917
 918         if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
 919                 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
 920                                               cache_level);
 921         } else {
 922                 struct i915_page_directory_pointer *pdp;
 923                 uint64_t pml4e;
 924                 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
 925
 926                 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
 927                         gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
 928                                                       start, cache_level);
 929                 }
 930         }
 931 }
 932
 933 static void gen8_free_page_tables(struct drm_i915_private *dev_priv,
 934                                   struct i915_page_directory *pd)
 935 {
 936         int i;
 937
 938         if (!px_page(pd))
 939                 return;
 940
 941         for_each_set_bit(i, pd->used_pdes, I915_PDES) {
 942                 if (WARN_ON(!pd->page_table[i]))
 943                         continue;
 944
 945                 free_pt(dev_priv, pd->page_table[i]);
 946                 pd->page_table[i] = NULL;
 947         }
 948 }
 949
 950 static int gen8_init_scratch(struct i915_address_space *vm)
 951 {
 952         struct drm_i915_private *dev_priv = vm->i915;
 953         int ret;
 954
 955         ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
 956         if (ret)
 957                 return ret;
 958
 959         vm->scratch_pt = alloc_pt(dev_priv);
 960         if (IS_ERR(vm->scratch_pt)) {
 961                 ret = PTR_ERR(vm->scratch_pt);
 962                 goto free_scratch_page;
 963         }
 964
 965         vm->scratch_pd = alloc_pd(dev_priv);
 966         if (IS_ERR(vm->scratch_pd)) {
 967                 ret = PTR_ERR(vm->scratch_pd);
 968                 goto free_pt;
 969         }
 970
 971         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
 972                 vm->scratch_pdp = alloc_pdp(dev_priv);
 973                 if (IS_ERR(vm->scratch_pdp)) {
 974                         ret = PTR_ERR(vm->scratch_pdp);
 975                         goto free_pd;
 976                 }
 977         }
 978
 979         gen8_initialize_pt(vm, vm->scratch_pt);
 980         gen8_initialize_pd(vm, vm->scratch_pd);
 981         if (USES_FULL_48BIT_PPGTT(dev_priv))
 982                 gen8_initialize_pdp(vm, vm->scratch_pdp);
 983
 984         return 0;
 985
 986 free_pd:
 987         free_pd(dev_priv, vm->scratch_pd);
 988 free_pt:
 989         free_pt(dev_priv, vm->scratch_pt);
 990 free_scratch_page:
 991         cleanup_scratch_page(dev_priv, &vm->scratch_page);
 992
 993         return ret;
 994 }
 995
 996 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 997 {
 998         enum vgt_g2v_type msg;
 999         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1000         int i;
1001
1002         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1003                 u64 daddr = px_dma(&ppgtt->pml4);
1004
1005                 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1006                 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1007
1008                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1009                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1010         } else {
1011                 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
1012                         u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1013
1014                         I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1015                         I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1016                 }
1017
1018                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1019                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1020         }
1021
1022         I915_WRITE(vgtif_reg(g2v_notify), msg);
1023
1024         return 0;
1025 }
1026
1027 static void gen8_free_scratch(struct i915_address_space *vm)
1028 {
1029         struct drm_i915_private *dev_priv = vm->i915;
1030
1031         if (USES_FULL_48BIT_PPGTT(dev_priv))
1032                 free_pdp(dev_priv, vm->scratch_pdp);
1033         free_pd(dev_priv, vm->scratch_pd);
1034         free_pt(dev_priv, vm->scratch_pt);
1035         cleanup_scratch_page(dev_priv, &vm->scratch_page);
1036 }
1037
1038 static void gen8_ppgtt_cleanup_3lvl(struct drm_i915_private *dev_priv,
1039                                     struct i915_page_directory_pointer *pdp)
1040 {
1041         int i;
1042
1043         for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)) {
1044                 if (WARN_ON(!pdp->page_directory[i]))
1045                         continue;
1046
1047                 gen8_free_page_tables(dev_priv, pdp->page_directory[i]);
1048                 free_pd(dev_priv, pdp->page_directory[i]);
1049         }
1050
1051         free_pdp(dev_priv, pdp);
1052 }
1053
1054 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1055 {
1056         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1057         int i;
1058
1059         for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
1060                 if (WARN_ON(!ppgtt->pml4.pdps[i]))
1061                         continue;
1062
1063                 gen8_ppgtt_cleanup_3lvl(dev_priv, ppgtt->pml4.pdps[i]);
1064         }
1065
1066         cleanup_px(dev_priv, &ppgtt->pml4);
1067 }
1068
1069 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1070 {
1071         struct drm_i915_private *dev_priv = vm->i915;
1072         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1073
1074         if (intel_vgpu_active(dev_priv))
1075                 gen8_ppgtt_notify_vgt(ppgtt, false);
1076
1077         if (!USES_FULL_48BIT_PPGTT(dev_priv))
1078                 gen8_ppgtt_cleanup_3lvl(dev_priv, &ppgtt->pdp);
1079         else
1080                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1081
1082         gen8_free_scratch(vm);
1083 }
1084
1085 /**
1086  * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
1087  * @vm: Master vm structure.
1088  * @pd: Page directory for this address range.
1089  * @start:      Starting virtual address to begin allocations.
1090  * @length:     Size of the allocations.
1091  * @new_pts:    Bitmap set by function with new allocations. Likely used by the
1092  *              caller to free on error.
1093  *
1094  * Allocate the required number of page tables. Extremely similar to
1095  * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
1096  * the page directory boundary (instead of the page directory pointer). That
1097  * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
1098  * possible, and likely that the caller will need to use multiple calls of this
1099  * function to achieve the appropriate allocation.
1100  *
1101  * Return: 0 if success; negative error code otherwise.
1102  */
1103 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1104                                      struct i915_page_directory *pd,
1105                                      uint64_t start,
1106                                      uint64_t length,
1107                                      unsigned long *new_pts)
1108 {
1109         struct drm_i915_private *dev_priv = vm->i915;
1110         struct i915_page_table *pt;
1111         uint32_t pde;
1112
1113         gen8_for_each_pde(pt, pd, start, length, pde) {
1114                 /* Don't reallocate page tables */
1115                 if (test_bit(pde, pd->used_pdes)) {
1116                         /* Scratch is never allocated this way */
1117                         WARN_ON(pt == vm->scratch_pt);
1118                         continue;
1119                 }
1120
1121                 pt = alloc_pt(dev_priv);
1122                 if (IS_ERR(pt))
1123                         goto unwind_out;
1124
1125                 gen8_initialize_pt(vm, pt);
1126                 pd->page_table[pde] = pt;
1127                 __set_bit(pde, new_pts);
1128                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
1129         }
1130
1131         return 0;
1132
1133 unwind_out:
1134         for_each_set_bit(pde, new_pts, I915_PDES)
1135                 free_pt(dev_priv, pd->page_table[pde]);
1136
1137         return -ENOMEM;
1138 }
1139
1140 /**
1141  * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
1142  * @vm: Master vm structure.
1143  * @pdp:        Page directory pointer for this address range.
1144  * @start:      Starting virtual address to begin allocations.
1145  * @length:     Size of the allocations.
1146  * @new_pds:    Bitmap set by function with new allocations. Likely used by the
1147  *              caller to free on error.
1148  *
1149  * Allocate the required number of page directories starting at the pde index of
1150  * @start, and ending at the pde index @start + @length. This function will skip
1151  * over already allocated page directories within the range, and only allocate
1152  * new ones, setting the appropriate pointer within the pdp as well as the
1153  * correct position in the bitmap @new_pds.
1154  *
1155  * The function will only allocate the pages within the range for a give page
1156  * directory pointer. In other words, if @start + @length straddles a virtually
1157  * addressed PDP boundary (512GB for 4k pages), there will be more allocations
1158  * required by the caller, This is not currently possible, and the BUG in the
1159  * code will prevent it.
1160  *
1161  * Return: 0 if success; negative error code otherwise.
1162  */
1163 static int
1164 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1165                                   struct i915_page_directory_pointer *pdp,
1166                                   uint64_t start,
1167                                   uint64_t length,
1168                                   unsigned long *new_pds)
1169 {
1170         struct drm_i915_private *dev_priv = vm->i915;
1171         struct i915_page_directory *pd;
1172         uint32_t pdpe;
1173         uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1174
1175         WARN_ON(!bitmap_empty(new_pds, pdpes));
1176
1177         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1178                 if (test_bit(pdpe, pdp->used_pdpes))
1179                         continue;
1180
1181                 pd = alloc_pd(dev_priv);
1182                 if (IS_ERR(pd))
1183                         goto unwind_out;
1184
1185                 gen8_initialize_pd(vm, pd);
1186                 pdp->page_directory[pdpe] = pd;
1187                 __set_bit(pdpe, new_pds);
1188                 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
1189         }
1190
1191         return 0;
1192
1193 unwind_out:
1194         for_each_set_bit(pdpe, new_pds, pdpes)
1195                 free_pd(dev_priv, pdp->page_directory[pdpe]);
1196
1197         return -ENOMEM;
1198 }
1199
1200 /**
1201  * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
1202  * @vm: Master vm structure.
1203  * @pml4:       Page map level 4 for this address range.
1204  * @start:      Starting virtual address to begin allocations.
1205  * @length:     Size of the allocations.
1206  * @new_pdps:   Bitmap set by function with new allocations. Likely used by the
1207  *              caller to free on error.
1208  *
1209  * Allocate the required number of page directory pointers. Extremely similar to
1210  * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
1211  * The main difference is here we are limited by the pml4 boundary (instead of
1212  * the page directory pointer).
1213  *
1214  * Return: 0 if success; negative error code otherwise.
1215  */
1216 static int
1217 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1218                                   struct i915_pml4 *pml4,
1219                                   uint64_t start,
1220                                   uint64_t length,
1221                                   unsigned long *new_pdps)
1222 {
1223         struct drm_i915_private *dev_priv = vm->i915;
1224         struct i915_page_directory_pointer *pdp;
1225         uint32_t pml4e;
1226
1227         WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
1228
1229         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1230                 if (!test_bit(pml4e, pml4->used_pml4es)) {
1231                         pdp = alloc_pdp(dev_priv);
1232                         if (IS_ERR(pdp))
1233                                 goto unwind_out;
1234
1235                         gen8_initialize_pdp(vm, pdp);
1236                         pml4->pdps[pml4e] = pdp;
1237                         __set_bit(pml4e, new_pdps);
1238                         trace_i915_page_directory_pointer_entry_alloc(vm,
1239                                                                       pml4e,
1240                                                                       start,
1241                                                                       GEN8_PML4E_SHIFT);
1242                 }
1243         }
1244
1245         return 0;
1246
1247 unwind_out:
1248         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1249                 free_pdp(dev_priv, pml4->pdps[pml4e]);
1250
1251         return -ENOMEM;
1252 }
1253
1254 static void
1255 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
1256 {
1257         kfree(new_pts);
1258         kfree(new_pds);
1259 }
1260
1261 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both
1262  * of these are based on the number of PDPEs in the system.
1263  */
1264 static
1265 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
1266                                          unsigned long **new_pts,
1267                                          uint32_t pdpes)
1268 {
1269         unsigned long *pds;
1270         unsigned long *pts;
1271
1272         pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
1273         if (!pds)
1274                 return -ENOMEM;
1275
1276         pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
1277                       GFP_TEMPORARY);
1278         if (!pts)
1279                 goto err_out;
1280
1281         *new_pds = pds;
1282         *new_pts = pts;
1283
1284         return 0;
1285
1286 err_out:
1287         free_gen8_temp_bitmaps(pds, pts);
1288         return -ENOMEM;
1289 }
1290
1291 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1292                                     struct i915_page_directory_pointer *pdp,
1293                                     uint64_t start,
1294                                     uint64_t length)
1295 {
1296         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1297         unsigned long *new_page_dirs, *new_page_tables;
1298         struct drm_i915_private *dev_priv = vm->i915;
1299         struct i915_page_directory *pd;
1300         const uint64_t orig_start = start;
1301         const uint64_t orig_length = length;
1302         uint32_t pdpe;
1303         uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1304         int ret;
1305
1306         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1307         if (ret)
1308                 return ret;
1309
1310         /* Do the allocations first so we can easily bail out */
1311         ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
1312                                                 new_page_dirs);
1313         if (ret) {
1314                 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1315                 return ret;
1316         }
1317
1318         /* For every page directory referenced, allocate page tables */
1319         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1320                 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
1321                                                 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
1322                 if (ret)
1323                         goto err_out;
1324         }
1325
1326         start = orig_start;
1327         length = orig_length;
1328
1329         /* Allocations have completed successfully, so set the bitmaps, and do
1330          * the mappings. */
1331         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1332                 gen8_pde_t *const page_directory = kmap_px(pd);
1333                 struct i915_page_table *pt;
1334                 uint64_t pd_len = length;
1335                 uint64_t pd_start = start;
1336                 uint32_t pde;
1337
1338                 /* Every pd should be allocated, we just did that above. */
1339                 WARN_ON(!pd);
1340
1341                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1342                         /* Same reasoning as pd */
1343                         WARN_ON(!pt);
1344                         WARN_ON(!pd_len);
1345                         WARN_ON(!gen8_pte_count(pd_start, pd_len));
1346
1347                         /* Set our used ptes within the page table */
1348                         bitmap_set(pt->used_ptes,
1349                                    gen8_pte_index(pd_start),
1350                                    gen8_pte_count(pd_start, pd_len));
1351
1352                         /* Our pde is now pointing to the pagetable, pt */
1353                         __set_bit(pde, pd->used_pdes);
1354
1355                         /* Map the PDE to the page table */
1356                         page_directory[pde] = gen8_pde_encode(px_dma(pt),
1357                                                               I915_CACHE_LLC);
1358                         trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
1359                                                         gen8_pte_index(start),
1360                                                         gen8_pte_count(start, length),
1361                                                         GEN8_PTES);
1362
1363                         /* NB: We haven't yet mapped ptes to pages. At this
1364                          * point we're still relying on insert_entries() */
1365                 }
1366
1367                 kunmap_px(ppgtt, page_directory);
1368                 __set_bit(pdpe, pdp->used_pdpes);
1369                 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);
1370         }
1371
1372         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1373         mark_tlbs_dirty(ppgtt);
1374         return 0;
1375
1376 err_out:
1377         while (pdpe--) {
1378                 unsigned long temp;
1379
1380                 for_each_set_bit(temp, new_page_tables + pdpe *
1381                                 BITS_TO_LONGS(I915_PDES), I915_PDES)
1382                         free_pt(dev_priv,
1383                                 pdp->page_directory[pdpe]->page_table[temp]);
1384         }
1385
1386         for_each_set_bit(pdpe, new_page_dirs, pdpes)
1387                 free_pd(dev_priv, pdp->page_directory[pdpe]);
1388
1389         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1390         mark_tlbs_dirty(ppgtt);
1391         return ret;
1392 }
1393
1394 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1395                                     struct i915_pml4 *pml4,
1396                                     uint64_t start,
1397                                     uint64_t length)
1398 {
1399         DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
1400         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1401         struct i915_page_directory_pointer *pdp;
1402         uint64_t pml4e;
1403         int ret = 0;
1404
1405         /* Do the pml4 allocations first, so we don't need to track the newly
1406          * allocated tables below the pdp */
1407         bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
1408
1409         /* The pagedirectory and pagetable allocations are done in the shared 3
1410          * and 4 level code. Just allocate the pdps.
1411          */
1412         ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
1413                                                 new_pdps);
1414         if (ret)
1415                 return ret;
1416
1417         WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2,
1418              "The allocation has spanned more than 512GB. "
1419              "It is highly likely this is incorrect.");
1420
1421         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1422                 WARN_ON(!pdp);
1423
1424                 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
1425                 if (ret)
1426                         goto err_out;
1427
1428                 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);
1429         }
1430
1431         bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
1432                   GEN8_PML4ES_PER_PML4);
1433
1434         return 0;
1435
1436 err_out:
1437         for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1438                 gen8_ppgtt_cleanup_3lvl(vm->i915, pml4->pdps[pml4e]);
1439
1440         return ret;
1441 }
1442
1443 static int gen8_alloc_va_range(struct i915_address_space *vm,
1444                                uint64_t start, uint64_t length)
1445 {
1446         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1447
1448         if (USES_FULL_48BIT_PPGTT(vm->i915))
1449                 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1450         else
1451                 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
1452 }
1453
1454 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
1455                           uint64_t start, uint64_t length,
1456                           gen8_pte_t scratch_pte,
1457                           struct seq_file *m)
1458 {
1459         struct i915_page_directory *pd;
1460         uint32_t pdpe;
1461
1462         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1463                 struct i915_page_table *pt;
1464                 uint64_t pd_len = length;
1465                 uint64_t pd_start = start;
1466                 uint32_t pde;
1467
1468                 if (!test_bit(pdpe, pdp->used_pdpes))
1469                         continue;
1470
1471                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1472                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1473                         uint32_t  pte;
1474                         gen8_pte_t *pt_vaddr;
1475
1476                         if (!test_bit(pde, pd->used_pdes))
1477                                 continue;
1478
1479                         pt_vaddr = kmap_px(pt);
1480                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1481                                 uint64_t va =
1482                                         (pdpe << GEN8_PDPE_SHIFT) |
1483                                         (pde << GEN8_PDE_SHIFT) |
1484                                         (pte << GEN8_PTE_SHIFT);
1485                                 int i;
1486                                 bool found = false;
1487
1488                                 for (i = 0; i < 4; i++)
1489                                         if (pt_vaddr[pte + i] != scratch_pte)
1490                                                 found = true;
1491                                 if (!found)
1492                                         continue;
1493
1494                                 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1495                                 for (i = 0; i < 4; i++) {
1496                                         if (pt_vaddr[pte + i] != scratch_pte)
1497                                                 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1498                                         else
1499                                                 seq_puts(m, "  SCRATCH ");
1500                                 }
1501                                 seq_puts(m, "\n");
1502                         }
1503                         /* don't use kunmap_px, it could trigger
1504                          * an unnecessary flush.
1505                          */
1506                         kunmap_atomic(pt_vaddr);
1507                 }
1508         }
1509 }
1510
1511 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1512 {
1513         struct i915_address_space *vm = &ppgtt->base;
1514         uint64_t start = ppgtt->base.start;
1515         uint64_t length = ppgtt->base.total;
1516         gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
1517                                                  I915_CACHE_LLC);
1518
1519         if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
1520                 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
1521         } else {
1522                 uint64_t pml4e;
1523                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1524                 struct i915_page_directory_pointer *pdp;
1525
1526                 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1527                         if (!test_bit(pml4e, pml4->used_pml4es))
1528                                 continue;
1529
1530                         seq_printf(m, "    PML4E #%llu\n", pml4e);
1531                         gen8_dump_pdp(pdp, start, length, scratch_pte, m);
1532                 }
1533         }
1534 }
1535
1536 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1537 {
1538         unsigned long *new_page_dirs, *new_page_tables;
1539         uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev));
1540         int ret;
1541
1542         /* We allocate temp bitmap for page tables for no gain
1543          * but as this is for init only, lets keep the things simple
1544          */
1545         ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1546         if (ret)
1547                 return ret;
1548
1549         /* Allocate for all pdps regardless of how the ppgtt
1550          * was defined.
1551          */
1552         ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
1553                                                 0, 1ULL << 32,
1554                                                 new_page_dirs);
1555         if (!ret)
1556                 *ppgtt->pdp.used_pdpes = *new_page_dirs;
1557
1558         free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1559
1560         return ret;
1561 }
1562
1563 /*
1564  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1565  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1566  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1567  * space.
1568  *
1569  */
1570 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1571 {
1572         struct drm_i915_private *dev_priv = ppgtt->base.i915;
1573         int ret;
1574
1575         ret = gen8_init_scratch(&ppgtt->base);
1576         if (ret)
1577                 return ret;
1578
1579         ppgtt->base.start = 0;
1580         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1581         ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1582         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
1583         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1584         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1585         ppgtt->base.bind_vma = ppgtt_bind_vma;
1586         ppgtt->debug_dump = gen8_dump_ppgtt;
1587
1588         if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1589                 ret = setup_px(dev_priv, &ppgtt->pml4);
1590                 if (ret)
1591                         goto free_scratch;
1592
1593                 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1594
1595                 ppgtt->base.total = 1ULL << 48;
1596                 ppgtt->switch_mm = gen8_48b_mm_switch;
1597         } else {
1598                 ret = __pdp_init(dev_priv, &ppgtt->pdp);
1599                 if (ret)
1600                         goto free_scratch;
1601
1602                 ppgtt->base.total = 1ULL << 32;
1603                 ppgtt->switch_mm = gen8_legacy_mm_switch;
1604                 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
1605                                                               0, 0,
1606                                                               GEN8_PML4E_SHIFT);
1607
1608                 if (intel_vgpu_active(dev_priv)) {
1609                         ret = gen8_preallocate_top_level_pdps(ppgtt);
1610                         if (ret)
1611                                 goto free_scratch;
1612                 }
1613         }
1614
1615         if (intel_vgpu_active(dev_priv))
1616                 gen8_ppgtt_notify_vgt(ppgtt, true);
1617
1618         return 0;
1619
1620 free_scratch:
1621         gen8_free_scratch(&ppgtt->base);
1622         return ret;
1623 }
1624
1625 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1626 {
1627         struct i915_address_space *vm = &ppgtt->base;
1628         struct i915_page_table *unused;
1629         gen6_pte_t scratch_pte;
1630         uint32_t pd_entry;
1631         uint32_t  pte, pde;
1632         uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
1633
1634         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1635                                      I915_CACHE_LLC, 0);
1636
1637         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1638                 u32 expected;
1639                 gen6_pte_t *pt_vaddr;
1640                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1641                 pd_entry = readl(ppgtt->pd_addr + pde);
1642                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1643
1644                 if (pd_entry != expected)
1645                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1646                                    pde,
1647                                    pd_entry,
1648                                    expected);
1649                 seq_printf(m, "\tPDE: %x\n", pd_entry);
1650
1651                 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1652
1653                 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1654                         unsigned long va =
1655                                 (pde * PAGE_SIZE * GEN6_PTES) +
1656                                 (pte * PAGE_SIZE);
1657                         int i;
1658                         bool found = false;
1659                         for (i = 0; i < 4; i++)
1660                                 if (pt_vaddr[pte + i] != scratch_pte)
1661                                         found = true;
1662                         if (!found)
1663                                 continue;
1664
1665                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1666                         for (i = 0; i < 4; i++) {
1667                                 if (pt_vaddr[pte + i] != scratch_pte)
1668                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
1669                                 else
1670                                         seq_puts(m, "  SCRATCH ");
1671                         }
1672                         seq_puts(m, "\n");
1673                 }
1674                 kunmap_px(ppgtt, pt_vaddr);
1675         }
1676 }
1677
1678 /* Write pde (index) from the page directory @pd to the page table @pt */
1679 static void gen6_write_pde(struct i915_page_directory *pd,
1680                             const int pde, struct i915_page_table *pt)
1681 {
1682         /* Caller needs to make sure the write completes if necessary */
1683         struct i915_hw_ppgtt *ppgtt =
1684                 container_of(pd, struct i915_hw_ppgtt, pd);
1685         u32 pd_entry;
1686
1687         pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
1688         pd_entry |= GEN6_PDE_VALID;
1689
1690         writel(pd_entry, ppgtt->pd_addr + pde);
1691 }
1692
1693 /* Write all the page tables found in the ppgtt structure to incrementing page
1694  * directories. */
1695 static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1696                                   struct i915_page_directory *pd,
1697                                   uint32_t start, uint32_t length)
1698 {
1699         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1700         struct i915_page_table *pt;
1701         uint32_t pde;
1702
1703         gen6_for_each_pde(pt, pd, start, length, pde)
1704                 gen6_write_pde(pd, pde, pt);
1705
1706         /* Make sure write is complete before other code can use this page
1707          * table. Also require for WC mapped PTEs */
1708         readl(ggtt->gsm);
1709 }
1710
1711 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1712 {
1713         BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1714
1715         return (ppgtt->pd.base.ggtt_offset / 64) << 16;
1716 }
1717
1718 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1719                          struct drm_i915_gem_request *req)
1720 {
1721         struct intel_ring *ring = req->ring;
1722         struct intel_engine_cs *engine = req->engine;
1723         int ret;
1724
1725         /* NB: TLBs must be flushed and invalidated before a switch */
1726         ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1727         if (ret)
1728                 return ret;
1729
1730         ret = intel_ring_begin(req, 6);
1731         if (ret)
1732                 return ret;
1733
1734         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1735         intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
1736         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1737         intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
1738         intel_ring_emit(ring, get_pd_offset(ppgtt));
1739         intel_ring_emit(ring, MI_NOOP);
1740         intel_ring_advance(ring);
1741
1742         return 0;
1743 }
1744
1745 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1746                           struct drm_i915_gem_request *req)
1747 {
1748         struct intel_ring *ring = req->ring;
1749         struct intel_engine_cs *engine = req->engine;
1750         int ret;
1751
1752         /* NB: TLBs must be flushed and invalidated before a switch */
1753         ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1754         if (ret)
1755                 return ret;
1756
1757         ret = intel_ring_begin(req, 6);
1758         if (ret)
1759                 return ret;
1760
1761         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1762         intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
1763         intel_ring_emit(ring, PP_DIR_DCLV_2G);
1764         intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
1765         intel_ring_emit(ring, get_pd_offset(ppgtt));
1766         intel_ring_emit(ring, MI_NOOP);
1767         intel_ring_advance(ring);
1768
1769         /* XXX: RCS is the only one to auto invalidate the TLBs? */
1770         if (engine->id != RCS) {
1771                 ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1772                 if (ret)
1773                         return ret;
1774         }
1775
1776         return 0;
1777 }
1778
1779 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1780                           struct drm_i915_gem_request *req)
1781 {
1782         struct intel_engine_cs *engine = req->engine;
1783         struct drm_i915_private *dev_priv = req->i915;
1784
1785         I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1786         I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1787         return 0;
1788 }
1789
1790 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1791 {
1792         struct intel_engine_cs *engine;
1793         enum intel_engine_id id;
1794
1795         for_each_engine(engine, dev_priv, id) {
1796                 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1797                                  GEN8_GFX_PPGTT_48B : 0;
1798                 I915_WRITE(RING_MODE_GEN7(engine),
1799                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1800         }
1801 }
1802
1803 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1804 {
1805         struct intel_engine_cs *engine;
1806         uint32_t ecochk, ecobits;
1807         enum intel_engine_id id;
1808
1809         ecobits = I915_READ(GAC_ECO_BITS);
1810         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1811
1812         ecochk = I915_READ(GAM_ECOCHK);
1813         if (IS_HASWELL(dev_priv)) {
1814                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1815         } else {
1816                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1817                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1818         }
1819         I915_WRITE(GAM_ECOCHK, ecochk);
1820
1821         for_each_engine(engine, dev_priv, id) {
1822                 /* GFX_MODE is per-ring on gen7+ */
1823                 I915_WRITE(RING_MODE_GEN7(engine),
1824                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1825         }
1826 }
1827
1828 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1829 {
1830         uint32_t ecochk, gab_ctl, ecobits;
1831
1832         ecobits = I915_READ(GAC_ECO_BITS);
1833         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1834                    ECOBITS_PPGTT_CACHE64B);
1835
1836         gab_ctl = I915_READ(GAB_CTL);
1837         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1838
1839         ecochk = I915_READ(GAM_ECOCHK);
1840         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1841
1842         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1843 }
1844
1845 /* PPGTT support for Sandybdrige/Gen6 and later */
1846 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1847                                    uint64_t start,
1848                                    uint64_t length)
1849 {
1850         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1851         gen6_pte_t *pt_vaddr, scratch_pte;
1852         unsigned first_entry = start >> PAGE_SHIFT;
1853         unsigned num_entries = length >> PAGE_SHIFT;
1854         unsigned act_pt = first_entry / GEN6_PTES;
1855         unsigned first_pte = first_entry % GEN6_PTES;
1856         unsigned last_pte, i;
1857
1858         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1859                                      I915_CACHE_LLC, 0);
1860
1861         while (num_entries) {
1862                 last_pte = first_pte + num_entries;
1863                 if (last_pte > GEN6_PTES)
1864                         last_pte = GEN6_PTES;
1865
1866                 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1867
1868                 for (i = first_pte; i < last_pte; i++)
1869                         pt_vaddr[i] = scratch_pte;
1870
1871                 kunmap_px(ppgtt, pt_vaddr);
1872
1873                 num_entries -= last_pte - first_pte;
1874                 first_pte = 0;
1875                 act_pt++;
1876         }
1877 }
1878
1879 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1880                                       struct sg_table *pages,
1881                                       uint64_t start,
1882                                       enum i915_cache_level cache_level, u32 flags)
1883 {
1884         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1885         unsigned first_entry = start >> PAGE_SHIFT;
1886         unsigned act_pt = first_entry / GEN6_PTES;
1887         unsigned act_pte = first_entry % GEN6_PTES;
1888         gen6_pte_t *pt_vaddr = NULL;
1889         struct sgt_iter sgt_iter;
1890         dma_addr_t addr;
1891
1892         for_each_sgt_dma(addr, sgt_iter, pages) {
1893                 if (pt_vaddr == NULL)
1894                         pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1895
1896                 pt_vaddr[act_pte] =
1897                         vm->pte_encode(addr, cache_level, flags);
1898
1899                 if (++act_pte == GEN6_PTES) {
1900                         kunmap_px(ppgtt, pt_vaddr);
1901                         pt_vaddr = NULL;
1902                         act_pt++;
1903                         act_pte = 0;
1904                 }
1905         }
1906
1907         if (pt_vaddr)
1908                 kunmap_px(ppgtt, pt_vaddr);
1909 }
1910
1911 static int gen6_alloc_va_range(struct i915_address_space *vm,
1912                                uint64_t start_in, uint64_t length_in)
1913 {
1914         DECLARE_BITMAP(new_page_tables, I915_PDES);
1915         struct drm_i915_private *dev_priv = vm->i915;
1916         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1917         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1918         struct i915_page_table *pt;
1919         uint32_t start, length, start_save, length_save;
1920         uint32_t pde;
1921         int ret;
1922
1923         start = start_save = start_in;
1924         length = length_save = length_in;
1925
1926         bitmap_zero(new_page_tables, I915_PDES);
1927
1928         /* The allocation is done in two stages so that we can bail out with
1929          * minimal amount of pain. The first stage finds new page tables that
1930          * need allocation. The second stage marks use ptes within the page
1931          * tables.
1932          */
1933         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1934                 if (pt != vm->scratch_pt) {
1935                         WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1936                         continue;
1937                 }
1938
1939                 /* We've already allocated a page table */
1940                 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1941
1942                 pt = alloc_pt(dev_priv);
1943                 if (IS_ERR(pt)) {
1944                         ret = PTR_ERR(pt);
1945                         goto unwind_out;
1946                 }
1947
1948                 gen6_initialize_pt(vm, pt);
1949
1950                 ppgtt->pd.page_table[pde] = pt;
1951                 __set_bit(pde, new_page_tables);
1952                 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
1953         }
1954
1955         start = start_save;
1956         length = length_save;
1957
1958         gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1959                 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1960
1961                 bitmap_zero(tmp_bitmap, GEN6_PTES);
1962                 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1963                            gen6_pte_count(start, length));
1964
1965                 if (__test_and_clear_bit(pde, new_page_tables))
1966                         gen6_write_pde(&ppgtt->pd, pde, pt);
1967
1968                 trace_i915_page_table_entry_map(vm, pde, pt,
1969                                          gen6_pte_index(start),
1970                                          gen6_pte_count(start, length),
1971                                          GEN6_PTES);
1972                 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
1973                                 GEN6_PTES);
1974         }
1975
1976         WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1977
1978         /* Make sure write is complete before other code can use this page
1979          * table. Also require for WC mapped PTEs */
1980         readl(ggtt->gsm);
1981
1982         mark_tlbs_dirty(ppgtt);
1983         return 0;
1984
1985 unwind_out:
1986         for_each_set_bit(pde, new_page_tables, I915_PDES) {
1987                 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
1988
1989                 ppgtt->pd.page_table[pde] = vm->scratch_pt;
1990                 free_pt(dev_priv, pt);
1991         }
1992
1993         mark_tlbs_dirty(ppgtt);
1994         return ret;
1995 }
1996
1997 static int gen6_init_scratch(struct i915_address_space *vm)
1998 {
1999         struct drm_i915_private *dev_priv = vm->i915;
2000         int ret;
2001
2002         ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
2003         if (ret)
2004                 return ret;
2005
2006         vm->scratch_pt = alloc_pt(dev_priv);
2007         if (IS_ERR(vm->scratch_pt)) {
2008                 cleanup_scratch_page(dev_priv, &vm->scratch_page);
2009                 return PTR_ERR(vm->scratch_pt);
2010         }
2011
2012         gen6_initialize_pt(vm, vm->scratch_pt);
2013
2014         return 0;
2015 }
2016
2017 static void gen6_free_scratch(struct i915_address_space *vm)
2018 {
2019         struct drm_i915_private *dev_priv = vm->i915;
2020
2021         free_pt(dev_priv, vm->scratch_pt);
2022         cleanup_scratch_page(dev_priv, &vm->scratch_page);
2023 }
2024
2025 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2026 {
2027         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
2028         struct i915_page_directory *pd = &ppgtt->pd;
2029         struct drm_i915_private *dev_priv = vm->i915;
2030         struct i915_page_table *pt;
2031         uint32_t pde;
2032
2033         drm_mm_remove_node(&ppgtt->node);
2034
2035         gen6_for_all_pdes(pt, pd, pde)
2036                 if (pt != vm->scratch_pt)
2037                         free_pt(dev_priv, pt);
2038
2039         gen6_free_scratch(vm);
2040 }
2041
2042 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
2043 {
2044         struct i915_address_space *vm = &ppgtt->base;
2045         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2046         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2047         bool retried = false;
2048         int ret;
2049
2050         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2051          * allocator works in address space sizes, so it's multiplied by page
2052          * size. We allocate at the top of the GTT to avoid fragmentation.
2053          */
2054         BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
2055
2056         ret = gen6_init_scratch(vm);
2057         if (ret)
2058                 return ret;
2059
2060 alloc:
2061         ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm, &ppgtt->node,
2062                                                   GEN6_PD_SIZE, GEN6_PD_ALIGN,
2063                                                   I915_COLOR_UNEVICTABLE,
2064                                                   0, ggtt->base.total,
2065                                                   DRM_MM_TOPDOWN);
2066         if (ret == -ENOSPC && !retried) {
2067                 ret = i915_gem_evict_something(&ggtt->base,
2068                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
2069                                                I915_COLOR_UNEVICTABLE,
2070                                                0, ggtt->base.total,
2071                                                0);
2072                 if (ret)
2073                         goto err_out;
2074
2075                 retried = true;
2076                 goto alloc;
2077         }
2078
2079         if (ret)
2080                 goto err_out;
2081
2082
2083         if (ppgtt->node.start < ggtt->mappable_end)
2084                 DRM_DEBUG("Forced to use aperture for PDEs\n");
2085
2086         return 0;
2087
2088 err_out:
2089         gen6_free_scratch(vm);
2090         return ret;
2091 }
2092
2093 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2094 {
2095         return gen6_ppgtt_allocate_page_directories(ppgtt);
2096 }
2097
2098 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2099                                   uint64_t start, uint64_t length)
2100 {
2101         struct i915_page_table *unused;
2102         uint32_t pde;
2103
2104         gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
2105                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2106 }
2107
2108 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2109 {
2110         struct drm_i915_private *dev_priv = ppgtt->base.i915;
2111         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2112         int ret;
2113
2114         ppgtt->base.pte_encode = ggtt->base.pte_encode;
2115         if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
2116                 ppgtt->switch_mm = gen6_mm_switch;
2117         else if (IS_HASWELL(dev_priv))
2118                 ppgtt->switch_mm = hsw_mm_switch;
2119         else if (IS_GEN7(dev_priv))
2120                 ppgtt->switch_mm = gen7_mm_switch;
2121         else
2122                 BUG();
2123
2124         ret = gen6_ppgtt_alloc(ppgtt);
2125         if (ret)
2126                 return ret;
2127
2128         ppgtt->base.allocate_va_range = gen6_alloc_va_range;
2129         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2130         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2131         ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2132         ppgtt->base.bind_vma = ppgtt_bind_vma;
2133         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2134         ppgtt->base.start = 0;
2135         ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2136         ppgtt->debug_dump = gen6_dump_ppgtt;
2137
2138         ppgtt->pd.base.ggtt_offset =
2139                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2140
2141         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
2142                 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2143
2144         gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2145
2146         gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
2147
2148         DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2149                          ppgtt->node.size >> 20,
2150                          ppgtt->node.start / PAGE_SIZE);
2151
2152         DRM_DEBUG("Adding PPGTT at offset %x\n",
2153                   ppgtt->pd.base.ggtt_offset << 10);
2154
2155         return 0;
2156 }
2157
2158 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2159                            struct drm_i915_private *dev_priv)
2160 {
2161         ppgtt->base.i915 = dev_priv;
2162
2163         if (INTEL_INFO(dev_priv)->gen < 8)
2164                 return gen6_ppgtt_init(ppgtt);
2165         else
2166                 return gen8_ppgtt_init(ppgtt);
2167 }
2168
2169 static void i915_address_space_init(struct i915_address_space *vm,
2170                                     struct drm_i915_private *dev_priv,
2171                                     const char *name)
2172 {
2173         i915_gem_timeline_init(dev_priv, &vm->timeline, name);
2174         drm_mm_init(&vm->mm, vm->start, vm->total);
2175         INIT_LIST_HEAD(&vm->active_list);
2176         INIT_LIST_HEAD(&vm->inactive_list);
2177         INIT_LIST_HEAD(&vm->unbound_list);
2178         list_add_tail(&vm->global_link, &dev_priv->vm_list);
2179 }
2180
2181 static void i915_address_space_fini(struct i915_address_space *vm)
2182 {
2183         i915_gem_timeline_fini(&vm->timeline);
2184         drm_mm_takedown(&vm->mm);
2185         list_del(&vm->global_link);
2186 }
2187
2188 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2189 {
2190         /* This function is for gtt related workarounds. This function is
2191          * called on driver load and after a GPU reset, so you can place
2192          * workarounds here even if they get overwritten by GPU reset.
2193          */
2194         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt */
2195         if (IS_BROADWELL(dev_priv))
2196                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2197         else if (IS_CHERRYVIEW(dev_priv))
2198                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2199         else if (IS_SKYLAKE(dev_priv))
2200                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2201         else if (IS_BROXTON(dev_priv))
2202                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2203 }
2204
2205 static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2206                            struct drm_i915_private *dev_priv,
2207                            struct drm_i915_file_private *file_priv,
2208                            const char *name)
2209 {
2210         int ret;
2211
2212         ret = __hw_ppgtt_init(ppgtt, dev_priv);
2213         if (ret == 0) {
2214                 kref_init(&ppgtt->ref);
2215                 i915_address_space_init(&ppgtt->base, dev_priv, name);
2216                 ppgtt->base.file = file_priv;
2217         }
2218
2219         return ret;
2220 }
2221
2222 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2223 {
2224         gtt_write_workarounds(dev_priv);
2225
2226         /* In the case of execlists, PPGTT is enabled by the context descriptor
2227          * and the PDPs are contained within the context itself.  We don't
2228          * need to do anything here. */
2229         if (i915.enable_execlists)
2230                 return 0;
2231
2232         if (!USES_PPGTT(dev_priv))
2233                 return 0;
2234
2235         if (IS_GEN6(dev_priv))
2236                 gen6_ppgtt_enable(dev_priv);
2237         else if (IS_GEN7(dev_priv))
2238                 gen7_ppgtt_enable(dev_priv);
2239         else if (INTEL_GEN(dev_priv) >= 8)
2240                 gen8_ppgtt_enable(dev_priv);
2241         else
2242                 MISSING_CASE(INTEL_GEN(dev_priv));
2243
2244         return 0;
2245 }
2246
2247 struct i915_hw_ppgtt *
2248 i915_ppgtt_create(struct drm_i915_private *dev_priv,
2249                   struct drm_i915_file_private *fpriv,
2250                   const char *name)
2251 {
2252         struct i915_hw_ppgtt *ppgtt;
2253         int ret;
2254
2255         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2256         if (!ppgtt)
2257                 return ERR_PTR(-ENOMEM);
2258
2259         ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name);
2260         if (ret) {
2261                 kfree(ppgtt);
2262                 return ERR_PTR(ret);
2263         }
2264
2265         trace_i915_ppgtt_create(&ppgtt->base);
2266
2267         return ppgtt;
2268 }
2269
2270 void i915_ppgtt_release(struct kref *kref)
2271 {
2272         struct i915_hw_ppgtt *ppgtt =
2273                 container_of(kref, struct i915_hw_ppgtt, ref);
2274
2275         trace_i915_ppgtt_release(&ppgtt->base);
2276
2277         /* vmas should already be unbound and destroyed */
2278         WARN_ON(!list_empty(&ppgtt->base.active_list));
2279         WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2280         WARN_ON(!list_empty(&ppgtt->base.unbound_list));
2281
2282         i915_address_space_fini(&ppgtt->base);
2283
2284         ppgtt->base.cleanup(&ppgtt->base);
2285         kfree(ppgtt);
2286 }
2287
2288 /* Certain Gen5 chipsets require require idling the GPU before
2289  * unmapping anything from the GTT when VT-d is enabled.
2290  */
2291 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2292 {
2293 #ifdef CONFIG_INTEL_IOMMU
2294         /* Query intel_iommu to see if we need the workaround. Presumably that
2295          * was loaded first.
2296          */
2297         if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
2298                 return true;
2299 #endif
2300         return false;
2301 }
2302
2303 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2304 {
2305         struct intel_engine_cs *engine;
2306         enum intel_engine_id id;
2307
2308         if (INTEL_INFO(dev_priv)->gen < 6)
2309                 return;
2310
2311         for_each_engine(engine, dev_priv, id) {
2312                 u32 fault_reg;
2313                 fault_reg = I915_READ(RING_FAULT_REG(engine));
2314                 if (fault_reg & RING_FAULT_VALID) {
2315                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2316                                          "\tAddr: 0x%08lx\n"
2317                                          "\tAddress space: %s\n"
2318                                          "\tSource ID: %d\n"
2319                                          "\tType: %d\n",
2320                                          fault_reg & PAGE_MASK,
2321                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2322                                          RING_FAULT_SRCID(fault_reg),
2323                                          RING_FAULT_FAULT_TYPE(fault_reg));
2324                         I915_WRITE(RING_FAULT_REG(engine),
2325                                    fault_reg & ~RING_FAULT_VALID);
2326                 }
2327         }
2328
2329         /* Engine specific init may not have been done till this point. */
2330         if (dev_priv->engine[RCS])
2331                 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2332 }
2333
2334 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
2335 {
2336         if (INTEL_INFO(dev_priv)->gen < 6) {
2337                 intel_gtt_chipset_flush();
2338         } else {
2339                 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2340                 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2341         }
2342 }
2343
2344 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2345 {
2346         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2347
2348         /* Don't bother messing with faults pre GEN6 as we have little
2349          * documentation supporting that it's a good idea.
2350          */
2351         if (INTEL_GEN(dev_priv) < 6)
2352                 return;
2353
2354         i915_check_and_clear_faults(dev_priv);
2355
2356         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
2357
2358         i915_ggtt_flush(dev_priv);
2359 }
2360
2361 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2362                                struct sg_table *pages)
2363 {
2364         if (dma_map_sg(&obj->base.dev->pdev->dev,
2365                        pages->sgl, pages->nents,
2366                        PCI_DMA_BIDIRECTIONAL))
2367                 return 0;
2368
2369         return -ENOSPC;
2370 }
2371
2372 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2373 {
2374         writeq(pte, addr);
2375 }
2376
2377 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2378                                   dma_addr_t addr,
2379                                   uint64_t offset,
2380                                   enum i915_cache_level level,
2381                                   u32 unused)
2382 {
2383         struct drm_i915_private *dev_priv = vm->i915;
2384         gen8_pte_t __iomem *pte =
2385                 (gen8_pte_t __iomem *)dev_priv->ggtt.gsm +
2386                 (offset >> PAGE_SHIFT);
2387
2388         gen8_set_pte(pte, gen8_pte_encode(addr, level));
2389
2390         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2391         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2392 }
2393
2394 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2395                                      struct sg_table *st,
2396                                      uint64_t start,
2397                                      enum i915_cache_level level, u32 unused)
2398 {
2399         struct drm_i915_private *dev_priv = vm->i915;
2400         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2401         struct sgt_iter sgt_iter;
2402         gen8_pte_t __iomem *gtt_entries;
2403         gen8_pte_t gtt_entry;
2404         dma_addr_t addr;
2405         int i = 0;
2406
2407         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
2408
2409         for_each_sgt_dma(addr, sgt_iter, st) {
2410                 gtt_entry = gen8_pte_encode(addr, level);
2411                 gen8_set_pte(&gtt_entries[i++], gtt_entry);
2412         }
2413
2414         /*
2415          * XXX: This serves as a posting read to make sure that the PTE has
2416          * actually been updated. There is some concern that even though
2417          * registers and PTEs are within the same BAR that they are potentially
2418          * of NUMA access patterns. Therefore, even with the way we assume
2419          * hardware should work, we must keep this posting read for paranoia.
2420          */
2421         if (i != 0)
2422                 WARN_ON(readq(&gtt_entries[i-1]) != gtt_entry);
2423
2424         /* This next bit makes the above posting read even more important. We
2425          * want to flush the TLBs only after we're certain all the PTE updates
2426          * have finished.
2427          */
2428         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2429         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2430 }
2431
2432 struct insert_entries {
2433         struct i915_address_space *vm;
2434         struct sg_table *st;
2435         uint64_t start;
2436         enum i915_cache_level level;
2437         u32 flags;
2438 };
2439
2440 static int gen8_ggtt_insert_entries__cb(void *_arg)
2441 {
2442         struct insert_entries *arg = _arg;
2443         gen8_ggtt_insert_entries(arg->vm, arg->st,
2444                                  arg->start, arg->level, arg->flags);
2445         return 0;
2446 }
2447
2448 static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2449                                           struct sg_table *st,
2450                                           uint64_t start,
2451                                           enum i915_cache_level level,
2452                                           u32 flags)
2453 {
2454         struct insert_entries arg = { vm, st, start, level, flags };
2455         stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL);
2456 }
2457
2458 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2459                                   dma_addr_t addr,
2460                                   uint64_t offset,
2461                                   enum i915_cache_level level,
2462                                   u32 flags)
2463 {
2464         struct drm_i915_private *dev_priv = vm->i915;
2465         gen6_pte_t __iomem *pte =
2466                 (gen6_pte_t __iomem *)dev_priv->ggtt.gsm +
2467                 (offset >> PAGE_SHIFT);
2468
2469         iowrite32(vm->pte_encode(addr, level, flags), pte);
2470
2471         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2472         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2473 }
2474
2475 /*
2476  * Binds an object into the global gtt with the specified cache level. The object
2477  * will be accessible to the GPU via commands whose operands reference offsets
2478  * within the global GTT as well as accessible by the GPU through the GMADR
2479  * mapped BAR (dev_priv->mm.gtt->gtt).
2480  */
2481 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2482                                      struct sg_table *st,
2483                                      uint64_t start,
2484                                      enum i915_cache_level level, u32 flags)
2485 {
2486         struct drm_i915_private *dev_priv = vm->i915;
2487         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2488         struct sgt_iter sgt_iter;
2489         gen6_pte_t __iomem *gtt_entries;
2490         gen6_pte_t gtt_entry;
2491         dma_addr_t addr;
2492         int i = 0;
2493
2494         gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
2495
2496         for_each_sgt_dma(addr, sgt_iter, st) {
2497                 gtt_entry = vm->pte_encode(addr, level, flags);
2498                 iowrite32(gtt_entry, &gtt_entries[i++]);
2499         }
2500
2501         /* XXX: This serves as a posting read to make sure that the PTE has
2502          * actually been updated. There is some concern that even though
2503          * registers and PTEs are within the same BAR that they are potentially
2504          * of NUMA access patterns. Therefore, even with the way we assume
2505          * hardware should work, we must keep this posting read for paranoia.
2506          */
2507         if (i != 0)
2508                 WARN_ON(readl(&gtt_entries[i-1]) != gtt_entry);
2509
2510         /* This next bit makes the above posting read even more important. We
2511          * want to flush the TLBs only after we're certain all the PTE updates
2512          * have finished.
2513          */
2514         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2515         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2516 }
2517
2518 static void nop_clear_range(struct i915_address_space *vm,
2519                             uint64_t start, uint64_t length)
2520 {
2521 }
2522
2523 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2524                                   uint64_t start, uint64_t length)
2525 {
2526         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2527         unsigned first_entry = start >> PAGE_SHIFT;
2528         unsigned num_entries = length >> PAGE_SHIFT;
2529         gen8_pte_t scratch_pte, __iomem *gtt_base =
2530                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2531         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2532         int i;
2533
2534         if (WARN(num_entries > max_entries,
2535                  "First entry = %d; Num entries = %d (max=%d)\n",
2536                  first_entry, num_entries, max_entries))
2537                 num_entries = max_entries;
2538
2539         scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
2540                                       I915_CACHE_LLC);
2541         for (i = 0; i < num_entries; i++)
2542                 gen8_set_pte(&gtt_base[i], scratch_pte);
2543         readl(gtt_base);
2544 }
2545
2546 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2547                                   uint64_t start,
2548                                   uint64_t length)
2549 {
2550         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2551         unsigned first_entry = start >> PAGE_SHIFT;
2552         unsigned num_entries = length >> PAGE_SHIFT;
2553         gen6_pte_t scratch_pte, __iomem *gtt_base =
2554                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2555         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2556         int i;
2557
2558         if (WARN(num_entries > max_entries,
2559                  "First entry = %d; Num entries = %d (max=%d)\n",
2560                  first_entry, num_entries, max_entries))
2561                 num_entries = max_entries;
2562
2563         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2564                                      I915_CACHE_LLC, 0);
2565
2566         for (i = 0; i < num_entries; i++)
2567                 iowrite32(scratch_pte, &gtt_base[i]);
2568         readl(gtt_base);
2569 }
2570
2571 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2572                                   dma_addr_t addr,
2573                                   uint64_t offset,
2574                                   enum i915_cache_level cache_level,
2575                                   u32 unused)
2576 {
2577         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2578                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2579
2580         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2581 }
2582
2583 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2584                                      struct sg_table *pages,
2585                                      uint64_t start,
2586                                      enum i915_cache_level cache_level, u32 unused)
2587 {
2588         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2589                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2590
2591         intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2592
2593 }
2594
2595 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2596                                   uint64_t start,
2597                                   uint64_t length)
2598 {
2599         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2600 }
2601
2602 static int ggtt_bind_vma(struct i915_vma *vma,
2603                          enum i915_cache_level cache_level,
2604                          u32 flags)
2605 {
2606         struct drm_i915_private *i915 = vma->vm->i915;
2607         struct drm_i915_gem_object *obj = vma->obj;
2608         u32 pte_flags = 0;
2609         int ret;
2610
2611         ret = i915_get_ggtt_vma_pages(vma);
2612         if (ret)
2613                 return ret;
2614
2615         /* Currently applicable only to VLV */
2616         if (obj->gt_ro)
2617                 pte_flags |= PTE_READ_ONLY;
2618
2619         intel_runtime_pm_get(i915);
2620         vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
2621                                 cache_level, pte_flags);
2622         intel_runtime_pm_put(i915);
2623
2624         /*
2625          * Without aliasing PPGTT there's no difference between
2626          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2627          * upgrade to both bound if we bind either to avoid double-binding.
2628          */
2629         vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2630
2631         return 0;
2632 }
2633
2634 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2635                                  enum i915_cache_level cache_level,
2636                                  u32 flags)
2637 {
2638         struct drm_i915_private *i915 = vma->vm->i915;
2639         u32 pte_flags;
2640         int ret;
2641
2642         ret = i915_get_ggtt_vma_pages(vma);
2643         if (ret)
2644                 return ret;
2645
2646         /* Currently applicable only to VLV */
2647         pte_flags = 0;
2648         if (vma->obj->gt_ro)
2649                 pte_flags |= PTE_READ_ONLY;
2650
2651
2652         if (flags & I915_VMA_GLOBAL_BIND) {
2653                 intel_runtime_pm_get(i915);
2654                 vma->vm->insert_entries(vma->vm,
2655                                         vma->pages, vma->node.start,
2656                                         cache_level, pte_flags);
2657                 intel_runtime_pm_put(i915);
2658         }
2659
2660         if (flags & I915_VMA_LOCAL_BIND) {
2661                 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2662                 appgtt->base.insert_entries(&appgtt->base,
2663                                             vma->pages, vma->node.start,
2664                                             cache_level, pte_flags);
2665         }
2666
2667         return 0;
2668 }
2669
2670 static void ggtt_unbind_vma(struct i915_vma *vma)
2671 {
2672         struct drm_i915_private *i915 = vma->vm->i915;
2673         struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2674         const u64 size = min(vma->size, vma->node.size);
2675
2676         if (vma->flags & I915_VMA_GLOBAL_BIND) {
2677                 intel_runtime_pm_get(i915);
2678                 vma->vm->clear_range(vma->vm,
2679                                      vma->node.start, size);
2680                 intel_runtime_pm_put(i915);
2681         }
2682
2683         if (vma->flags & I915_VMA_LOCAL_BIND && appgtt)
2684                 appgtt->base.clear_range(&appgtt->base,
2685                                          vma->node.start, size);
2686 }
2687
2688 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2689                                struct sg_table *pages)
2690 {
2691         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2692         struct device *kdev = &dev_priv->drm.pdev->dev;
2693         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2694
2695         if (unlikely(ggtt->do_idle_maps)) {
2696                 if (i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED)) {
2697                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2698                         /* Wait a bit, in hopes it avoids the hang */
2699                         udelay(10);
2700                 }
2701         }
2702
2703         dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2704 }
2705
2706 static void i915_gtt_color_adjust(struct drm_mm_node *node,
2707                                   unsigned long color,
2708                                   u64 *start,
2709                                   u64 *end)
2710 {
2711         if (node->color != color)
2712                 *start += 4096;
2713
2714         node = list_first_entry_or_null(&node->node_list,
2715                                         struct drm_mm_node,
2716                                         node_list);
2717         if (node && node->allocated && node->color != color)
2718                 *end -= 4096;
2719 }
2720
2721 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2722 {
2723         /* Let GEM Manage all of the aperture.
2724          *
2725          * However, leave one page at the end still bound to the scratch page.
2726          * There are a number of places where the hardware apparently prefetches
2727          * past the end of the object, and we've seen multiple hangs with the
2728          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2729          * aperture.  One page should be enough to keep any prefetching inside
2730          * of the aperture.
2731          */
2732         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2733         unsigned long hole_start, hole_end;
2734         struct i915_hw_ppgtt *ppgtt;
2735         struct drm_mm_node *entry;
2736         int ret;
2737
2738         ret = intel_vgt_balloon(dev_priv);
2739         if (ret)
2740                 return ret;
2741
2742         /* Reserve a mappable slot for our lockless error capture */
2743         ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm,
2744                                                   &ggtt->error_capture,
2745                                                   4096, 0,
2746                                                   I915_COLOR_UNEVICTABLE,
2747                                                   0, ggtt->mappable_end,
2748                                                   0, 0);
2749         if (ret)
2750                 return ret;
2751
2752         /* Clear any non-preallocated blocks */
2753         drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2754                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2755                               hole_start, hole_end);
2756                 ggtt->base.clear_range(&ggtt->base, hole_start,
2757                                        hole_end - hole_start);
2758         }
2759
2760         /* And finally clear the reserved guard page */
2761         ggtt->base.clear_range(&ggtt->base,
2762                                ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2763
2764         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2765                 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2766                 if (!ppgtt) {
2767                         ret = -ENOMEM;
2768                         goto err;
2769                 }
2770
2771                 ret = __hw_ppgtt_init(ppgtt, dev_priv);
2772                 if (ret)
2773                         goto err_ppgtt;
2774
2775                 if (ppgtt->base.allocate_va_range) {
2776                         ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
2777                                                             ppgtt->base.total);
2778                         if (ret)
2779                                 goto err_ppgtt_cleanup;
2780                 }
2781
2782                 ppgtt->base.clear_range(&ppgtt->base,
2783                                         ppgtt->base.start,
2784                                         ppgtt->base.total);
2785
2786                 dev_priv->mm.aliasing_ppgtt = ppgtt;
2787                 WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2788                 ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2789         }
2790
2791         return 0;
2792
2793 err_ppgtt_cleanup:
2794         ppgtt->base.cleanup(&ppgtt->base);
2795 err_ppgtt:
2796         kfree(ppgtt);
2797 err:
2798         drm_mm_remove_node(&ggtt->error_capture);
2799         return ret;
2800 }
2801
2802 /**
2803  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2804  * @dev_priv: i915 device
2805  */
2806 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2807 {
2808         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2809
2810         if (dev_priv->mm.aliasing_ppgtt) {
2811                 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2812                 ppgtt->base.cleanup(&ppgtt->base);
2813                 kfree(ppgtt);
2814         }
2815
2816         i915_gem_cleanup_stolen(&dev_priv->drm);
2817
2818         if (drm_mm_node_allocated(&ggtt->error_capture))
2819                 drm_mm_remove_node(&ggtt->error_capture);
2820
2821         if (drm_mm_initialized(&ggtt->base.mm)) {
2822                 intel_vgt_deballoon(dev_priv);
2823
2824                 mutex_lock(&dev_priv->drm.struct_mutex);
2825                 i915_address_space_fini(&ggtt->base);
2826                 mutex_unlock(&dev_priv->drm.struct_mutex);
2827         }
2828
2829         ggtt->base.cleanup(&ggtt->base);
2830
2831         arch_phys_wc_del(ggtt->mtrr);
2832         io_mapping_fini(&ggtt->mappable);
2833 }
2834
2835 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2836 {
2837         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2838         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2839         return snb_gmch_ctl << 20;
2840 }
2841
2842 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2843 {
2844         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2845         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2846         if (bdw_gmch_ctl)
2847                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2848
2849 #ifdef CONFIG_X86_32
2850         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2851         if (bdw_gmch_ctl > 4)
2852                 bdw_gmch_ctl = 4;
2853 #endif
2854
2855         return bdw_gmch_ctl << 20;
2856 }
2857
2858 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2859 {
2860         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2861         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2862
2863         if (gmch_ctrl)
2864                 return 1 << (20 + gmch_ctrl);
2865
2866         return 0;
2867 }
2868
2869 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2870 {
2871         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2872         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2873         return snb_gmch_ctl << 25; /* 32 MB units */
2874 }
2875
2876 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2877 {
2878         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2879         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2880         return bdw_gmch_ctl << 25; /* 32 MB units */
2881 }
2882
2883 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2884 {
2885         gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2886         gmch_ctrl &= SNB_GMCH_GMS_MASK;
2887
2888         /*
2889          * 0x0  to 0x10: 32MB increments starting at 0MB
2890          * 0x11 to 0x16: 4MB increments starting at 8MB
2891          * 0x17 to 0x1d: 4MB increments start at 36MB
2892          */
2893         if (gmch_ctrl < 0x11)
2894                 return gmch_ctrl << 25;
2895         else if (gmch_ctrl < 0x17)
2896                 return (gmch_ctrl - 0x11 + 2) << 22;
2897         else
2898                 return (gmch_ctrl - 0x17 + 9) << 22;
2899 }
2900
2901 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2902 {
2903         gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2904         gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2905
2906         if (gen9_gmch_ctl < 0xf0)
2907                 return gen9_gmch_ctl << 25; /* 32 MB units */
2908         else
2909                 /* 4MB increments starting at 0xf0 for 4MB */
2910                 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2911 }
2912
2913 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2914 {
2915         struct drm_i915_private *dev_priv = ggtt->base.i915;
2916         struct pci_dev *pdev = dev_priv->drm.pdev;
2917         phys_addr_t phys_addr;
2918         int ret;
2919
2920         /* For Modern GENs the PTEs and register space are split in the BAR */
2921         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2922
2923         /*
2924          * On BXT writes larger than 64 bit to the GTT pagetable range will be
2925          * dropped. For WC mappings in general we have 64 byte burst writes
2926          * when the WC buffer is flushed, so we can't use it, but have to
2927          * resort to an uncached mapping. The WC issue is easily caught by the
2928          * readback check when writing GTT PTE entries.
2929          */
2930         if (IS_GEN9_LP(dev_priv))
2931                 ggtt->gsm = ioremap_nocache(phys_addr, size);
2932         else
2933                 ggtt->gsm = ioremap_wc(phys_addr, size);
2934         if (!ggtt->gsm) {
2935                 DRM_ERROR("Failed to map the ggtt page table\n");
2936                 return -ENOMEM;
2937         }
2938
2939         ret = setup_scratch_page(dev_priv, &ggtt->base.scratch_page, GFP_DMA32);
2940         if (ret) {
2941                 DRM_ERROR("Scratch setup failed\n");
2942                 /* iounmap will also get called at remove, but meh */
2943                 iounmap(ggtt->gsm);
2944                 return ret;
2945         }
2946
2947         return 0;
2948 }
2949
2950 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2951  * bits. When using advanced contexts each context stores its own PAT, but
2952  * writing this data shouldn't be harmful even in those cases. */
2953 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2954 {
2955         uint64_t pat;
2956
2957         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
2958               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2959               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2960               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
2961               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2962               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2963               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2964               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2965
2966         if (!USES_PPGTT(dev_priv))
2967                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2968                  * so RTL will always use the value corresponding to
2969                  * pat_sel = 000".
2970                  * So let's disable cache for GGTT to avoid screen corruptions.
2971                  * MOCS still can be used though.
2972                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2973                  * before this patch, i.e. the same uncached + snooping access
2974                  * like on gen6/7 seems to be in effect.
2975                  * - So this just fixes blitter/render access. Again it looks
2976                  * like it's not just uncached access, but uncached + snooping.
2977                  * So we can still hold onto all our assumptions wrt cpu
2978                  * clflushing on LLC machines.
2979                  */
2980                 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2981
2982         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2983          * write would work. */
2984         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2985         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
2986 }
2987
2988 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2989 {
2990         uint64_t pat;
2991
2992         /*
2993          * Map WB on BDW to snooped on CHV.
2994          *
2995          * Only the snoop bit has meaning for CHV, the rest is
2996          * ignored.
2997          *
2998          * The hardware will never snoop for certain types of accesses:
2999          * - CPU GTT (GMADR->GGTT->no snoop->memory)
3000          * - PPGTT page tables
3001          * - some other special cycles
3002          *
3003          * As with BDW, we also need to consider the following for GT accesses:
3004          * "For GGTT, there is NO pat_sel[2:0] from the entry,
3005          * so RTL will always use the value corresponding to
3006          * pat_sel = 000".
3007          * Which means we must set the snoop bit in PAT entry 0
3008          * in order to keep the global status page working.
3009          */
3010         pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
3011               GEN8_PPAT(1, 0) |
3012               GEN8_PPAT(2, 0) |
3013               GEN8_PPAT(3, 0) |
3014               GEN8_PPAT(4, CHV_PPAT_SNOOP) |
3015               GEN8_PPAT(5, CHV_PPAT_SNOOP) |
3016               GEN8_PPAT(6, CHV_PPAT_SNOOP) |
3017               GEN8_PPAT(7, CHV_PPAT_SNOOP);
3018
3019         I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3020         I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3021 }
3022
3023 static void gen6_gmch_remove(struct i915_address_space *vm)
3024 {
3025         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3026
3027         iounmap(ggtt->gsm);
3028         cleanup_scratch_page(vm->i915, &vm->scratch_page);
3029 }
3030
3031 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3032 {
3033         struct drm_i915_private *dev_priv = ggtt->base.i915;
3034         struct pci_dev *pdev = dev_priv->drm.pdev;
3035         unsigned int size;
3036         u16 snb_gmch_ctl;
3037
3038         /* TODO: We're not aware of mappable constraints on gen8 yet */
3039         ggtt->mappable_base = pci_resource_start(pdev, 2);
3040         ggtt->mappable_end = pci_resource_len(pdev, 2);
3041
3042         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
3043                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3044
3045         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3046
3047         if (INTEL_GEN(dev_priv) >= 9) {
3048                 ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
3049                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3050         } else if (IS_CHERRYVIEW(dev_priv)) {
3051                 ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
3052                 size = chv_get_total_gtt_size(snb_gmch_ctl);
3053         } else {
3054                 ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
3055                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3056         }
3057
3058         ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3059
3060         if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3061                 chv_setup_private_ppat(dev_priv);
3062         else
3063                 bdw_setup_private_ppat(dev_priv);
3064
3065         ggtt->base.cleanup = gen6_gmch_remove;
3066         ggtt->base.bind_vma = ggtt_bind_vma;
3067         ggtt->base.unbind_vma = ggtt_unbind_vma;
3068         ggtt->base.insert_page = gen8_ggtt_insert_page;
3069         ggtt->base.clear_range = nop_clear_range;
3070         if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3071                 ggtt->base.clear_range = gen8_ggtt_clear_range;
3072
3073         ggtt->base.insert_entries = gen8_ggtt_insert_entries;
3074         if (IS_CHERRYVIEW(dev_priv))
3075                 ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
3076
3077         return ggtt_probe_common(ggtt, size);
3078 }
3079
3080 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3081 {
3082         struct drm_i915_private *dev_priv = ggtt->base.i915;
3083         struct pci_dev *pdev = dev_priv->drm.pdev;
3084         unsigned int size;
3085         u16 snb_gmch_ctl;
3086
3087         ggtt->mappable_base = pci_resource_start(pdev, 2);
3088         ggtt->mappable_end = pci_resource_len(pdev, 2);
3089
3090         /* 64/512MB is the current min/max we actually know of, but this is just
3091          * a coarse sanity check.
3092          */
3093         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3094                 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
3095                 return -ENXIO;
3096         }
3097
3098         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
3099                 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3100         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3101
3102         ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
3103
3104         size = gen6_get_total_gtt_size(snb_gmch_ctl);
3105         ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3106
3107         ggtt->base.clear_range = gen6_ggtt_clear_range;
3108         ggtt->base.insert_page = gen6_ggtt_insert_page;
3109         ggtt->base.insert_entries = gen6_ggtt_insert_entries;
3110         ggtt->base.bind_vma = ggtt_bind_vma;
3111         ggtt->base.unbind_vma = ggtt_unbind_vma;
3112         ggtt->base.cleanup = gen6_gmch_remove;
3113
3114         if (HAS_EDRAM(dev_priv))
3115                 ggtt->base.pte_encode = iris_pte_encode;
3116         else if (IS_HASWELL(dev_priv))
3117                 ggtt->base.pte_encode = hsw_pte_encode;
3118         else if (IS_VALLEYVIEW(dev_priv))
3119                 ggtt->base.pte_encode = byt_pte_encode;
3120         else if (INTEL_GEN(dev_priv) >= 7)
3121                 ggtt->base.pte_encode = ivb_pte_encode;
3122         else
3123                 ggtt->base.pte_encode = snb_pte_encode;
3124
3125         return ggtt_probe_common(ggtt, size);
3126 }
3127
3128 static void i915_gmch_remove(struct i915_address_space *vm)
3129 {
3130         intel_gmch_remove();
3131 }
3132
3133 static int i915_gmch_probe(struct i915_ggtt *ggtt)
3134 {
3135         struct drm_i915_private *dev_priv = ggtt->base.i915;
3136         int ret;
3137
3138         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3139         if (!ret) {
3140                 DRM_ERROR("failed to set up gmch\n");
3141                 return -EIO;
3142         }
3143
3144         intel_gtt_get(&ggtt->base.total, &ggtt->stolen_size,
3145                       &ggtt->mappable_base, &ggtt->mappable_end);
3146
3147         ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3148         ggtt->base.insert_page = i915_ggtt_insert_page;
3149         ggtt->base.insert_entries = i915_ggtt_insert_entries;
3150         ggtt->base.clear_range = i915_ggtt_clear_range;
3151         ggtt->base.bind_vma = ggtt_bind_vma;
3152         ggtt->base.unbind_vma = ggtt_unbind_vma;
3153         ggtt->base.cleanup = i915_gmch_remove;
3154
3155         if (unlikely(ggtt->do_idle_maps))
3156                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3157
3158         return 0;
3159 }
3160
3161 /**
3162  * i915_ggtt_probe_hw - Probe GGTT hardware location
3163  * @dev_priv: i915 device
3164  */
3165 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3166 {
3167         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3168         int ret;
3169
3170         ggtt->base.i915 = dev_priv;
3171
3172         if (INTEL_GEN(dev_priv) <= 5)
3173                 ret = i915_gmch_probe(ggtt);
3174         else if (INTEL_GEN(dev_priv) < 8)
3175                 ret = gen6_gmch_probe(ggtt);
3176         else
3177                 ret = gen8_gmch_probe(ggtt);
3178         if (ret)
3179                 return ret;
3180
3181         if ((ggtt->base.total - 1) >> 32) {
3182                 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3183                           " of address space! Found %lldM!\n",
3184                           ggtt->base.total >> 20);
3185                 ggtt->base.total = 1ULL << 32;
3186                 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
3187         }
3188
3189         if (ggtt->mappable_end > ggtt->base.total) {
3190                 DRM_ERROR("mappable aperture extends past end of GGTT,"
3191                           " aperture=%llx, total=%llx\n",
3192                           ggtt->mappable_end, ggtt->base.total);
3193                 ggtt->mappable_end = ggtt->base.total;
3194         }
3195
3196         /* GMADR is the PCI mmio aperture into the global GTT. */
3197         DRM_INFO("Memory usable by graphics device = %lluM\n",
3198                  ggtt->base.total >> 20);
3199         DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
3200         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", ggtt->stolen_size >> 20);
3201 #ifdef CONFIG_INTEL_IOMMU
3202         if (intel_iommu_gfx_mapped)
3203                 DRM_INFO("VT-d active for gfx access\n");
3204 #endif
3205
3206         return 0;
3207 }
3208
3209 /**
3210  * i915_ggtt_init_hw - Initialize GGTT hardware
3211  * @dev_priv: i915 device
3212  */
3213 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3214 {
3215         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3216         int ret;
3217
3218         INIT_LIST_HEAD(&dev_priv->vm_list);
3219
3220         /* Subtract the guard page before address space initialization to
3221          * shrink the range used by drm_mm.
3222          */
3223         mutex_lock(&dev_priv->drm.struct_mutex);
3224         ggtt->base.total -= PAGE_SIZE;
3225         i915_address_space_init(&ggtt->base, dev_priv, "[global]");
3226         ggtt->base.total += PAGE_SIZE;
3227         if (!HAS_LLC(dev_priv))
3228                 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
3229         mutex_unlock(&dev_priv->drm.struct_mutex);
3230
3231         if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
3232                                 dev_priv->ggtt.mappable_base,
3233                                 dev_priv->ggtt.mappable_end)) {
3234                 ret = -EIO;
3235                 goto out_gtt_cleanup;
3236         }
3237
3238         ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
3239
3240         /*
3241          * Initialise stolen early so that we may reserve preallocated
3242          * objects for the BIOS to KMS transition.
3243          */
3244         ret = i915_gem_init_stolen(dev_priv);
3245         if (ret)
3246                 goto out_gtt_cleanup;
3247
3248         return 0;
3249
3250 out_gtt_cleanup:
3251         ggtt->base.cleanup(&ggtt->base);
3252         return ret;
3253 }
3254
3255 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3256 {
3257         if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3258                 return -EIO;
3259
3260         return 0;
3261 }
3262
3263 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3264 {
3265         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3266         struct drm_i915_gem_object *obj, *on;
3267
3268         i915_check_and_clear_faults(dev_priv);
3269
3270         /* First fill our portion of the GTT with scratch pages */
3271         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
3272
3273         ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
3274
3275         /* clflush objects bound into the GGTT and rebind them. */
3276         list_for_each_entry_safe(obj, on,
3277                                  &dev_priv->mm.bound_list, global_link) {
3278                 bool ggtt_bound = false;
3279                 struct i915_vma *vma;
3280
3281                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3282                         if (vma->vm != &ggtt->base)
3283                                 continue;
3284
3285                         if (!i915_vma_unbind(vma))
3286                                 continue;
3287
3288                         WARN_ON(i915_vma_bind(vma, obj->cache_level,
3289                                               PIN_UPDATE));
3290                         ggtt_bound = true;
3291                 }
3292
3293                 if (ggtt_bound)
3294                         WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3295         }
3296
3297         ggtt->base.closed = false;
3298
3299         if (INTEL_GEN(dev_priv) >= 8) {
3300                 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3301                         chv_setup_private_ppat(dev_priv);
3302                 else
3303                         bdw_setup_private_ppat(dev_priv);
3304
3305                 return;
3306         }
3307
3308         if (USES_PPGTT(dev_priv)) {
3309                 struct i915_address_space *vm;
3310
3311                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3312                         /* TODO: Perhaps it shouldn't be gen6 specific */
3313
3314                         struct i915_hw_ppgtt *ppgtt;
3315
3316                         if (i915_is_ggtt(vm))
3317                                 ppgtt = dev_priv->mm.aliasing_ppgtt;
3318                         else
3319                                 ppgtt = i915_vm_to_ppgtt(vm);
3320
3321                         gen6_write_page_range(dev_priv, &ppgtt->pd,
3322                                               0, ppgtt->base.total);
3323                 }
3324         }
3325
3326         i915_ggtt_flush(dev_priv);
3327 }
3328
3329 struct i915_vma *
3330 i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
3331                     struct i915_address_space *vm,
3332                     const struct i915_ggtt_view *view)
3333 {
3334         struct rb_node *rb;
3335
3336         rb = obj->vma_tree.rb_node;
3337         while (rb) {
3338                 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
3339                 long cmp;
3340
3341                 cmp = i915_vma_compare(vma, vm, view);
3342                 if (cmp == 0)
3343                         return vma;
3344
3345                 if (cmp < 0)
3346                         rb = rb->rb_right;
3347                 else
3348                         rb = rb->rb_left;
3349         }
3350
3351         return NULL;
3352 }
3353
3354 struct i915_vma *
3355 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
3356                                   struct i915_address_space *vm,
3357                                   const struct i915_ggtt_view *view)
3358 {
3359         struct i915_vma *vma;
3360
3361         lockdep_assert_held(&obj->base.dev->struct_mutex);
3362         GEM_BUG_ON(view && !i915_is_ggtt(vm));
3363
3364         vma = i915_gem_obj_to_vma(obj, vm, view);
3365         if (!vma) {
3366                 vma = i915_vma_create(obj, vm, view);
3367                 GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view));
3368         }
3369
3370         GEM_BUG_ON(i915_vma_is_closed(vma));
3371         return vma;
3372 }
3373
3374 static struct scatterlist *
3375 rotate_pages(const dma_addr_t *in, unsigned int offset,
3376              unsigned int width, unsigned int height,
3377              unsigned int stride,
3378              struct sg_table *st, struct scatterlist *sg)
3379 {
3380         unsigned int column, row;
3381         unsigned int src_idx;
3382
3383         for (column = 0; column < width; column++) {
3384                 src_idx = stride * (height - 1) + column;
3385                 for (row = 0; row < height; row++) {
3386                         st->nents++;
3387                         /* We don't need the pages, but need to initialize
3388                          * the entries so the sg list can be happily traversed.
3389                          * The only thing we need are DMA addresses.
3390                          */
3391                         sg_set_page(sg, NULL, PAGE_SIZE, 0);
3392                         sg_dma_address(sg) = in[offset + src_idx];
3393                         sg_dma_len(sg) = PAGE_SIZE;
3394                         sg = sg_next(sg);
3395                         src_idx -= stride;
3396                 }
3397         }
3398
3399         return sg;
3400 }
3401
3402 static struct sg_table *
3403 intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
3404                           struct drm_i915_gem_object *obj)
3405 {
3406         const size_t n_pages = obj->base.size / PAGE_SIZE;
3407         unsigned int size = intel_rotation_info_size(rot_info);
3408         struct sgt_iter sgt_iter;
3409         dma_addr_t dma_addr;
3410         unsigned long i;
3411         dma_addr_t *page_addr_list;
3412         struct sg_table *st;
3413         struct scatterlist *sg;
3414         int ret = -ENOMEM;
3415
3416         /* Allocate a temporary list of source pages for random access. */
3417         page_addr_list = drm_malloc_gfp(n_pages,
3418                                         sizeof(dma_addr_t),
3419                                         GFP_TEMPORARY);
3420         if (!page_addr_list)
3421                 return ERR_PTR(ret);
3422
3423         /* Allocate target SG list. */
3424         st = kmalloc(sizeof(*st), GFP_KERNEL);
3425         if (!st)
3426                 goto err_st_alloc;
3427
3428         ret = sg_alloc_table(st, size, GFP_KERNEL);
3429         if (ret)
3430                 goto err_sg_alloc;
3431
3432         /* Populate source page list from the object. */
3433         i = 0;
3434         for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3435                 page_addr_list[i++] = dma_addr;
3436
3437         GEM_BUG_ON(i != n_pages);
3438         st->nents = 0;
3439         sg = st->sgl;
3440
3441         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3442                 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3443                                   rot_info->plane[i].width, rot_info->plane[i].height,
3444                                   rot_info->plane[i].stride, st, sg);
3445         }
3446
3447         DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
3448                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3449
3450         drm_free_large(page_addr_list);
3451
3452         return st;
3453
3454 err_sg_alloc:
3455         kfree(st);
3456 err_st_alloc:
3457         drm_free_large(page_addr_list);
3458
3459         DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3460                       obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3461
3462         return ERR_PTR(ret);
3463 }
3464
3465 static struct sg_table *
3466 intel_partial_pages(const struct i915_ggtt_view *view,
3467                     struct drm_i915_gem_object *obj)
3468 {
3469         struct sg_table *st;
3470         struct scatterlist *sg, *iter;
3471         unsigned int count = view->params.partial.size;
3472         unsigned int offset;
3473         int ret = -ENOMEM;
3474
3475         st = kmalloc(sizeof(*st), GFP_KERNEL);
3476         if (!st)
3477                 goto err_st_alloc;
3478
3479         ret = sg_alloc_table(st, count, GFP_KERNEL);
3480         if (ret)
3481                 goto err_sg_alloc;
3482
3483         iter = i915_gem_object_get_sg(obj,
3484                                       view->params.partial.offset,
3485                                       &offset);
3486         GEM_BUG_ON(!iter);
3487
3488         sg = st->sgl;
3489         st->nents = 0;
3490         do {
3491                 unsigned int len;
3492
3493                 len = min(iter->length - (offset << PAGE_SHIFT),
3494                           count << PAGE_SHIFT);
3495                 sg_set_page(sg, NULL, len, 0);
3496                 sg_dma_address(sg) =
3497                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
3498                 sg_dma_len(sg) = len;
3499
3500                 st->nents++;
3501                 count -= len >> PAGE_SHIFT;
3502                 if (count == 0) {
3503                         sg_mark_end(sg);
3504                         return st;
3505                 }
3506
3507                 sg = __sg_next(sg);
3508                 iter = __sg_next(iter);
3509                 offset = 0;
3510         } while (1);
3511
3512 err_sg_alloc:
3513         kfree(st);
3514 err_st_alloc:
3515         return ERR_PTR(ret);
3516 }
3517
3518 static int
3519 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3520 {
3521         int ret = 0;
3522
3523         /* The vma->pages are only valid within the lifespan of the borrowed
3524          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3525          * must be the vma->pages. A simple rule is that vma->pages must only
3526          * be accessed when the obj->mm.pages are pinned.
3527          */
3528         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3529
3530         if (vma->pages)
3531                 return 0;
3532
3533         if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
3534                 vma->pages = vma->obj->mm.pages;
3535         else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
3536                 vma->pages =
3537                         intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
3538         else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
3539                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3540         else
3541                 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3542                           vma->ggtt_view.type);
3543
3544         if (!vma->pages) {
3545                 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
3546                           vma->ggtt_view.type);
3547                 ret = -EINVAL;
3548         } else if (IS_ERR(vma->pages)) {
3549                 ret = PTR_ERR(vma->pages);
3550                 vma->pages = NULL;
3551                 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3552                           vma->ggtt_view.type, ret);
3553         }
3554
3555         return ret;
3556 }
3557