drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/seq_file.h>
  26 #include <drm/drmP.h>
  27 #include <drm/i915_drm.h>
  28 #include "i915_drv.h"
  29 #include "i915_trace.h"
  30 #include "intel_drv.h"
  31
  32 #define GEN6_PPGTT_PD_ENTRIES 512
  33 #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
  34 typedef uint64_t gen8_gtt_pte_t;
  35 typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
  36
  37 /* PPGTT stuff */
  38 #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
  39 #define HSW_GTT_ADDR_ENCODE(addr)       ((addr) | (((addr) >> 28) & 0x7f0))
  40
  41 #define GEN6_PDE_VALID                  (1 << 0)
  42 /* gen6+ has bit 11-4 for physical addr bit 39-32 */
  43 #define GEN6_PDE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  44
  45 #define GEN6_PTE_VALID                  (1 << 0)
  46 #define GEN6_PTE_UNCACHED               (1 << 1)
  47 #define HSW_PTE_UNCACHED                (0)
  48 #define GEN6_PTE_CACHE_LLC              (2 << 1)
  49 #define GEN7_PTE_CACHE_L3_LLC           (3 << 1)
  50 #define GEN6_PTE_ADDR_ENCODE(addr)      GEN6_GTT_ADDR_ENCODE(addr)
  51 #define HSW_PTE_ADDR_ENCODE(addr)       HSW_GTT_ADDR_ENCODE(addr)
  52
  53 /* Cacheability Control is a 4-bit value. The low three bits are stored in *
  54  * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
  55  */
  56 #define HSW_CACHEABILITY_CONTROL(bits)  ((((bits) & 0x7) << 1) | \
  57                                          (((bits) & 0x8) << (11 - 3)))
  58 #define HSW_WB_LLC_AGE3                 HSW_CACHEABILITY_CONTROL(0x2)
  59 #define HSW_WB_LLC_AGE0                 HSW_CACHEABILITY_CONTROL(0x3)
  60 #define HSW_WB_ELLC_LLC_AGE0            HSW_CACHEABILITY_CONTROL(0xb)
  61 #define HSW_WB_ELLC_LLC_AGE3            HSW_CACHEABILITY_CONTROL(0x8)
  62 #define HSW_WT_ELLC_LLC_AGE0            HSW_CACHEABILITY_CONTROL(0x6)
  63 #define HSW_WT_ELLC_LLC_AGE3            HSW_CACHEABILITY_CONTROL(0x7)
  64
  65 #define GEN8_PTES_PER_PAGE              (PAGE_SIZE / sizeof(gen8_gtt_pte_t))
  66 #define GEN8_PDES_PER_PAGE              (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
  67 #define GEN8_LEGACY_PDPS                4
  68
  69 #define PPAT_UNCACHED_INDEX             (_PAGE_PWT | _PAGE_PCD)
  70 #define PPAT_CACHED_PDE_INDEX           0 /* WB LLC */
  71 #define PPAT_CACHED_INDEX               _PAGE_PAT /* WB LLCeLLC */
  72 #define PPAT_DISPLAY_ELLC_INDEX         _PAGE_PCD /* WT eLLC */
  73
  74 static void ppgtt_bind_vma(struct i915_vma *vma,
  75                            enum i915_cache_level cache_level,
  76                            u32 flags);
  77 static void ppgtt_unbind_vma(struct i915_vma *vma);
  78 static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
  79
  80 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
  81                                              enum i915_cache_level level,
  82                                              bool valid)
  83 {
  84         gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
  85         pte |= addr;
  86         if (level != I915_CACHE_NONE)
  87                 pte |= PPAT_CACHED_INDEX;
  88         else
  89                 pte |= PPAT_UNCACHED_INDEX;
  90         return pte;
  91 }
  92
  93 static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
  94                                              dma_addr_t addr,
  95                                              enum i915_cache_level level)
  96 {
  97         gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
  98         pde |= addr;
  99         if (level != I915_CACHE_NONE)
 100                 pde |= PPAT_CACHED_PDE_INDEX;
 101         else
 102                 pde |= PPAT_UNCACHED_INDEX;
 103         return pde;
 104 }
 105
 106 static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
 107                                      enum i915_cache_level level,
 108                                      bool valid)
 109 {
 110         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 111         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 112
 113         switch (level) {
 114         case I915_CACHE_L3_LLC:
 115         case I915_CACHE_LLC:
 116                 pte |= GEN6_PTE_CACHE_LLC;
 117                 break;
 118         case I915_CACHE_NONE:
 119                 pte |= GEN6_PTE_UNCACHED;
 120                 break;
 121         default:
 122                 WARN_ON(1);
 123         }
 124
 125         return pte;
 126 }
 127
 128 static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
 129                                      enum i915_cache_level level,
 130                                      bool valid)
 131 {
 132         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 133         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 134
 135         switch (level) {
 136         case I915_CACHE_L3_LLC:
 137                 pte |= GEN7_PTE_CACHE_L3_LLC;
 138                 break;
 139         case I915_CACHE_LLC:
 140                 pte |= GEN6_PTE_CACHE_LLC;
 141                 break;
 142         case I915_CACHE_NONE:
 143                 pte |= GEN6_PTE_UNCACHED;
 144                 break;
 145         default:
 146                 WARN_ON(1);
 147         }
 148
 149         return pte;
 150 }
 151
 152 #define BYT_PTE_WRITEABLE               (1 << 1)
 153 #define BYT_PTE_SNOOPED_BY_CPU_CACHES   (1 << 2)
 154
 155 static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
 156                                      enum i915_cache_level level,
 157                                      bool valid)
 158 {
 159         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 160         pte |= GEN6_PTE_ADDR_ENCODE(addr);
 161
 162         /* Mark the page as writeable.  Other platforms don't have a
 163          * setting for read-only/writable, so this matches that behavior.
 164          */
 165         pte |= BYT_PTE_WRITEABLE;
 166
 167         if (level != I915_CACHE_NONE)
 168                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 169
 170         return pte;
 171 }
 172
 173 static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
 174                                      enum i915_cache_level level,
 175                                      bool valid)
 176 {
 177         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 178         pte |= HSW_PTE_ADDR_ENCODE(addr);
 179
 180         if (level != I915_CACHE_NONE)
 181                 pte |= HSW_WB_LLC_AGE3;
 182
 183         return pte;
 184 }
 185
 186 static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 187                                       enum i915_cache_level level,
 188                                       bool valid)
 189 {
 190         gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
 191         pte |= HSW_PTE_ADDR_ENCODE(addr);
 192
 193         switch (level) {
 194         case I915_CACHE_NONE:
 195                 break;
 196         case I915_CACHE_WT:
 197                 pte |= HSW_WT_ELLC_LLC_AGE3;
 198                 break;
 199         default:
 200                 pte |= HSW_WB_ELLC_LLC_AGE3;
 201                 break;
 202         }
 203
 204         return pte;
 205 }
 206
 207 /* Broadwell Page Directory Pointer Descriptors */
 208 static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
 209                            uint64_t val, bool synchronous)
 210 {
 211         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 212         int ret;
 213
 214         BUG_ON(entry >= 4);
 215
 216         if (synchronous) {
 217                 I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);
 218                 I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);
 219                 return 0;
 220         }
 221
 222         ret = intel_ring_begin(ring, 6);
 223         if (ret)
 224                 return ret;
 225
 226         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 227         intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
 228         intel_ring_emit(ring, (u32)(val >> 32));
 229         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 230         intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
 231         intel_ring_emit(ring, (u32)(val));
 232         intel_ring_advance(ring);
 233
 234         return 0;
 235 }
 236
 237 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 238                           struct intel_ring_buffer *ring,
 239                           bool synchronous)
 240 {
 241         int i, ret;
 242
 243         /* bit of a hack to find the actual last used pd */
 244         int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
 245
 246         for (i = used_pd - 1; i >= 0; i--) {
 247                 dma_addr_t addr = ppgtt->pd_dma_addr[i];
 248                 ret = gen8_write_pdp(ring, i, addr, synchronous);
 249                 if (ret)
 250                         return ret;
 251         }
 252
 253         return 0;
 254 }
 255
 256 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 257                                    uint64_t start,
 258                                    uint64_t length,
 259                                    bool use_scratch)
 260 {
 261         struct i915_hw_ppgtt *ppgtt =
 262                 container_of(vm, struct i915_hw_ppgtt, base);
 263         gen8_gtt_pte_t *pt_vaddr, scratch_pte;
 264         unsigned first_entry = start >> PAGE_SHIFT;
 265         unsigned num_entries = length >> PAGE_SHIFT;
 266         unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
 267         unsigned first_pte = first_entry % GEN8_PTES_PER_PAGE;
 268         unsigned last_pte, i;
 269
 270         scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
 271                                       I915_CACHE_LLC, use_scratch);
 272
 273         while (num_entries) {
 274                 struct page *page_table = &ppgtt->gen8_pt_pages[act_pt];
 275
 276                 last_pte = first_pte + num_entries;
 277                 if (last_pte > GEN8_PTES_PER_PAGE)
 278                         last_pte = GEN8_PTES_PER_PAGE;
 279
 280                 pt_vaddr = kmap_atomic(page_table);
 281
 282                 for (i = first_pte; i < last_pte; i++)
 283                         pt_vaddr[i] = scratch_pte;
 284
 285                 kunmap_atomic(pt_vaddr);
 286
 287                 num_entries -= last_pte - first_pte;
 288                 first_pte = 0;
 289                 act_pt++;
 290         }
 291 }
 292
 293 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 294                                       struct sg_table *pages,
 295                                       uint64_t start,
 296                                       enum i915_cache_level cache_level)
 297 {
 298         struct i915_hw_ppgtt *ppgtt =
 299                 container_of(vm, struct i915_hw_ppgtt, base);
 300         gen8_gtt_pte_t *pt_vaddr;
 301         unsigned first_entry = start >> PAGE_SHIFT;
 302         unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
 303         unsigned act_pte = first_entry % GEN8_PTES_PER_PAGE;
 304         struct sg_page_iter sg_iter;
 305
 306         pt_vaddr = NULL;
 307         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 308                 if (pt_vaddr == NULL)
 309                         pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]);
 310
 311                 pt_vaddr[act_pte] =
 312                         gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
 313                                         cache_level, true);
 314                 if (++act_pte == GEN8_PTES_PER_PAGE) {
 315                         kunmap_atomic(pt_vaddr);
 316                         pt_vaddr = NULL;
 317                         act_pt++;
 318                         act_pte = 0;
 319                 }
 320         }
 321         if (pt_vaddr)
 322                 kunmap_atomic(pt_vaddr);
 323 }
 324
 325 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 326 {
 327         int i;
 328
 329         for (i = 0; i < ppgtt->num_pd_pages ; i++)
 330                 kfree(ppgtt->gen8_pt_dma_addr[i]);
 331
 332         __free_pages(ppgtt->gen8_pt_pages, get_order(ppgtt->num_pt_pages << PAGE_SHIFT));
 333         __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
 334 }
 335
 336 static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 337 {
 338         struct pci_dev *hwdev = ppgtt->base.dev->pdev;
 339         int i, j;
 340
 341         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 342                 /* TODO: In the future we'll support sparse mappings, so this
 343                  * will have to change. */
 344                 if (!ppgtt->pd_dma_addr[i])
 345                         continue;
 346
 347                 pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
 348                                PCI_DMA_BIDIRECTIONAL);
 349
 350                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 351                         dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 352                         if (addr)
 353                                 pci_unmap_page(hwdev, addr, PAGE_SIZE,
 354                                                PCI_DMA_BIDIRECTIONAL);
 355                 }
 356         }
 357 }
 358
 359 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 360 {
 361         struct i915_hw_ppgtt *ppgtt =
 362                 container_of(vm, struct i915_hw_ppgtt, base);
 363
 364         list_del(&vm->global_link);
 365         drm_mm_takedown(&vm->mm);
 366
 367         gen8_ppgtt_unmap_pages(ppgtt);
 368         gen8_ppgtt_free(ppgtt);
 369 }
 370
 371 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
 372                                            const int max_pdp)
 373 {
 374         struct page *pt_pages;
 375         const int num_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
 376
 377         pt_pages = alloc_pages(GFP_KERNEL, get_order(num_pt_pages << PAGE_SHIFT));
 378         if (!pt_pages)
 379                 return -ENOMEM;
 380
 381         ppgtt->gen8_pt_pages = pt_pages;
 382         ppgtt->num_pt_pages = 1 << get_order(num_pt_pages << PAGE_SHIFT);
 383
 384         return 0;
 385 }
 386
 387 static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
 388 {
 389         int i;
 390
 391         for (i = 0; i < ppgtt->num_pd_pages; i++) {
 392                 ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
 393                                                      sizeof(dma_addr_t),
 394                                                      GFP_KERNEL);
 395                 if (!ppgtt->gen8_pt_dma_addr[i])
 396                         return -ENOMEM;
 397         }
 398
 399         return 0;
 400 }
 401
 402 static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 403                                                 const int max_pdp)
 404 {
 405         ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
 406         if (!ppgtt->pd_pages)
 407                 return -ENOMEM;
 408
 409         ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
 410         BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
 411
 412         return 0;
 413 }
 414
 415 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 416                             const int max_pdp)
 417 {
 418         int ret;
 419
 420         ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
 421         if (ret)
 422                 return ret;
 423
 424         ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
 425         if (ret) {
 426                 __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
 427                 return ret;
 428         }
 429
 430         ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
 431
 432         ret = gen8_ppgtt_allocate_dma(ppgtt);
 433         if (ret)
 434                 gen8_ppgtt_free(ppgtt);
 435
 436         return ret;
 437 }
 438
 439 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 440                                              const int pd)
 441 {
 442         dma_addr_t pd_addr;
 443         int ret;
 444
 445         pd_addr = pci_map_page(ppgtt->base.dev->pdev,
 446                                &ppgtt->pd_pages[pd], 0,
 447                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 448
 449         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
 450         if (ret)
 451                 return ret;
 452
 453         ppgtt->pd_dma_addr[pd] = pd_addr;
 454
 455         return 0;
 456 }
 457
 458 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 459                                         const int pd,
 460                                         const int pt)
 461 {
 462         dma_addr_t pt_addr;
 463         struct page *p;
 464         int ret;
 465
 466         p = &ppgtt->gen8_pt_pages[pd * GEN8_PDES_PER_PAGE + pt];
 467         pt_addr = pci_map_page(ppgtt->base.dev->pdev,
 468                                p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 469         ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
 470         if (ret)
 471                 return ret;
 472
 473         ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
 474
 475         return 0;
 476 }
 477
 478 /**
 479  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 480  * with a net effect resembling a 2-level page table in normal x86 terms. Each
 481  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 482  * space.
 483  *
 484  * FIXME: split allocation into smaller pieces. For now we only ever do this
 485  * once, but with full PPGTT, the multiple contiguous allocations will be bad.
 486  * TODO: Do something with the size parameter
 487  */
 488 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 489 {
 490         const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
 491         const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
 492         int i, j, ret;
 493
 494         if (size % (1<<30))
 495                 DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
 496
 497         /* 1. Do all our allocations for page directories and page tables. */
 498         ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
 499         if (ret)
 500                 return ret;
 501
 502         /*
 503          * 2. Create DMA mappings for the page directories and page tables.
 504          */
 505         for (i = 0; i < max_pdp; i++) {
 506                 ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
 507                 if (ret)
 508                         goto bail;
 509
 510                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 511                         ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
 512                         if (ret)
 513                                 goto bail;
 514                 }
 515         }
 516
 517         /*
 518          * 3. Map all the page directory entires to point to the page tables
 519          * we've allocated.
 520          *
 521          * For now, the PPGTT helper functions all require that the PDEs are
 522          * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
 523          * will never need to touch the PDEs again.
 524          */
 525         for (i = 0; i < max_pdp; i++) {
 526                 gen8_ppgtt_pde_t *pd_vaddr;
 527                 pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
 528                 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
 529                         dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 530                         pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 531                                                       I915_CACHE_LLC);
 532                 }
 533                 kunmap_atomic(pd_vaddr);
 534         }
 535
 536         ppgtt->enable = gen8_ppgtt_enable;
 537         ppgtt->switch_mm = gen8_mm_switch;
 538         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
 539         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
 540         ppgtt->base.cleanup = gen8_ppgtt_cleanup;
 541         ppgtt->base.start = 0;
 542         ppgtt->base.total = ppgtt->num_pt_pages * GEN8_PTES_PER_PAGE * PAGE_SIZE;
 543
 544         ppgtt->base.clear_range(&ppgtt->base, 0,
 545                                 ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE,
 546                                 true);
 547
 548         DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
 549                          ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
 550         DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
 551                          ppgtt->num_pt_pages,
 552                          (ppgtt->num_pt_pages - min_pt_pages) +
 553                          size % (1<<30));
 554         return 0;
 555
 556 bail:
 557         gen8_ppgtt_unmap_pages(ppgtt);
 558         gen8_ppgtt_free(ppgtt);
 559         return ret;
 560 }
 561
 562 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 563 {
 564         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 565         struct i915_address_space *vm = &ppgtt->base;
 566         gen6_gtt_pte_t __iomem *pd_addr;
 567         gen6_gtt_pte_t scratch_pte;
 568         uint32_t pd_entry;
 569         int pte, pde;
 570
 571         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
 572
 573         pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
 574                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 575
 576         seq_printf(m, "  VM %p (pd_offset %x-%x):\n", vm,
 577                    ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries);
 578         for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
 579                 u32 expected;
 580                 gen6_gtt_pte_t *pt_vaddr;
 581                 dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde];
 582                 pd_entry = readl(pd_addr + pde);
 583                 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
 584
 585                 if (pd_entry != expected)
 586                         seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
 587                                    pde,
 588                                    pd_entry,
 589                                    expected);
 590                 seq_printf(m, "\tPDE: %x\n", pd_entry);
 591
 592                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
 593                 for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
 594                         unsigned long va =
 595                                 (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
 596                                 (pte * PAGE_SIZE);
 597                         int i;
 598                         bool found = false;
 599                         for (i = 0; i < 4; i++)
 600                                 if (pt_vaddr[pte + i] != scratch_pte)
 601                                         found = true;
 602                         if (!found)
 603                                 continue;
 604
 605                         seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
 606                         for (i = 0; i < 4; i++) {
 607                                 if (pt_vaddr[pte + i] != scratch_pte)
 608                                         seq_printf(m, " %08x", pt_vaddr[pte + i]);
 609                                 else
 610                                         seq_puts(m, "  SCRATCH ");
 611                         }
 612                         seq_puts(m, "\n");
 613                 }
 614                 kunmap_atomic(pt_vaddr);
 615         }
 616 }
 617
 618 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
 619 {
 620         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 621         gen6_gtt_pte_t __iomem *pd_addr;
 622         uint32_t pd_entry;
 623         int i;
 624
 625         WARN_ON(ppgtt->pd_offset & 0x3f);
 626         pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
 627                 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 628         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 629                 dma_addr_t pt_addr;
 630
 631                 pt_addr = ppgtt->pt_dma_addr[i];
 632                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
 633                 pd_entry |= GEN6_PDE_VALID;
 634
 635                 writel(pd_entry, pd_addr + i);
 636         }
 637         readl(pd_addr);
 638 }
 639
 640 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
 641 {
 642         BUG_ON(ppgtt->pd_offset & 0x3f);
 643
 644         return (ppgtt->pd_offset / 64) << 16;
 645 }
 646
 647 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 648                          struct intel_ring_buffer *ring,
 649                          bool synchronous)
 650 {
 651         struct drm_device *dev = ppgtt->base.dev;
 652         struct drm_i915_private *dev_priv = dev->dev_private;
 653         int ret;
 654
 655         /* If we're in reset, we can assume the GPU is sufficiently idle to
 656          * manually frob these bits. Ideally we could use the ring functions,
 657          * except our error handling makes it quite difficult (can't use
 658          * intel_ring_begin, ring->flush, or intel_ring_advance)
 659          *
 660          * FIXME: We should try not to special case reset
 661          */
 662         if (synchronous ||
 663             i915_reset_in_progress(&dev_priv->gpu_error)) {
 664                 WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
 665                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 666                 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
 667                 POSTING_READ(RING_PP_DIR_BASE(ring));
 668                 return 0;
 669         }
 670
 671         /* NB: TLBs must be flushed and invalidated before a switch */
 672         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 673         if (ret)
 674                 return ret;
 675
 676         ret = intel_ring_begin(ring, 6);
 677         if (ret)
 678                 return ret;
 679
 680         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
 681         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
 682         intel_ring_emit(ring, PP_DIR_DCLV_2G);
 683         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
 684         intel_ring_emit(ring, get_pd_offset(ppgtt));
 685         intel_ring_emit(ring, MI_NOOP);
 686         intel_ring_advance(ring);
 687
 688         return 0;
 689 }
 690
 691 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 692                           struct intel_ring_buffer *ring,
 693                           bool synchronous)
 694 {
 695         struct drm_device *dev = ppgtt->base.dev;
 696         struct drm_i915_private *dev_priv = dev->dev_private;
 697         int ret;
 698
 699         /* If we're in reset, we can assume the GPU is sufficiently idle to
 700          * manually frob these bits. Ideally we could use the ring functions,
 701          * except our error handling makes it quite difficult (can't use
 702          * intel_ring_begin, ring->flush, or intel_ring_advance)
 703          *
 704          * FIXME: We should try not to special case reset
 705          */
 706         if (synchronous ||
 707             i915_reset_in_progress(&dev_priv->gpu_error)) {
 708                 WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
 709                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 710                 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
 711                 POSTING_READ(RING_PP_DIR_BASE(ring));
 712                 return 0;
 713         }
 714
 715         /* NB: TLBs must be flushed and invalidated before a switch */
 716         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 717         if (ret)
 718                 return ret;
 719
 720         ret = intel_ring_begin(ring, 6);
 721         if (ret)
 722                 return ret;
 723
 724         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
 725         intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
 726         intel_ring_emit(ring, PP_DIR_DCLV_2G);
 727         intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
 728         intel_ring_emit(ring, get_pd_offset(ppgtt));
 729         intel_ring_emit(ring, MI_NOOP);
 730         intel_ring_advance(ring);
 731
 732         /* XXX: RCS is the only one to auto invalidate the TLBs? */
 733         if (ring->id != RCS) {
 734                 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
 735                 if (ret)
 736                         return ret;
 737         }
 738
 739         return 0;
 740 }
 741
 742 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
 743                           struct intel_ring_buffer *ring,
 744                           bool synchronous)
 745 {
 746         struct drm_device *dev = ppgtt->base.dev;
 747         struct drm_i915_private *dev_priv = dev->dev_private;
 748
 749         if (!synchronous)
 750                 return 0;
 751
 752         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 753         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
 754
 755         POSTING_READ(RING_PP_DIR_DCLV(ring));
 756
 757         return 0;
 758 }
 759
 760 static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
 761 {
 762         struct drm_device *dev = ppgtt->base.dev;
 763         struct drm_i915_private *dev_priv = dev->dev_private;
 764         struct intel_ring_buffer *ring;
 765         int j, ret;
 766
 767         for_each_ring(ring, dev_priv, j) {
 768                 I915_WRITE(RING_MODE_GEN7(ring),
 769                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 770
 771                 /* We promise to do a switch later with FULL PPGTT. If this is
 772                  * aliasing, this is the one and only switch we'll do */
 773                 if (USES_FULL_PPGTT(dev))
 774                         continue;
 775
 776                 ret = ppgtt->switch_mm(ppgtt, ring, true);
 777                 if (ret)
 778                         goto err_out;
 779         }
 780
 781         return 0;
 782
 783 err_out:
 784         for_each_ring(ring, dev_priv, j)
 785                 I915_WRITE(RING_MODE_GEN7(ring),
 786                            _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
 787         return ret;
 788 }
 789
 790 static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
 791 {
 792         struct drm_device *dev = ppgtt->base.dev;
 793         drm_i915_private_t *dev_priv = dev->dev_private;
 794         struct intel_ring_buffer *ring;
 795         uint32_t ecochk, ecobits;
 796         int i;
 797
 798         ecobits = I915_READ(GAC_ECO_BITS);
 799         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 800
 801         ecochk = I915_READ(GAM_ECOCHK);
 802         if (IS_HASWELL(dev)) {
 803                 ecochk |= ECOCHK_PPGTT_WB_HSW;
 804         } else {
 805                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
 806                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
 807         }
 808         I915_WRITE(GAM_ECOCHK, ecochk);
 809
 810         for_each_ring(ring, dev_priv, i) {
 811                 int ret;
 812                 /* GFX_MODE is per-ring on gen7+ */
 813                 I915_WRITE(RING_MODE_GEN7(ring),
 814                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 815
 816                 /* We promise to do a switch later with FULL PPGTT. If this is
 817                  * aliasing, this is the one and only switch we'll do */
 818                 if (USES_FULL_PPGTT(dev))
 819                         continue;
 820
 821                 ret = ppgtt->switch_mm(ppgtt, ring, true);
 822                 if (ret)
 823                         return ret;
 824         }
 825
 826         return 0;
 827 }
 828
 829 static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
 830 {
 831         struct drm_device *dev = ppgtt->base.dev;
 832         drm_i915_private_t *dev_priv = dev->dev_private;
 833         struct intel_ring_buffer *ring;
 834         uint32_t ecochk, gab_ctl, ecobits;
 835         int i;
 836
 837         ecobits = I915_READ(GAC_ECO_BITS);
 838         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
 839                    ECOBITS_PPGTT_CACHE64B);
 840
 841         gab_ctl = I915_READ(GAB_CTL);
 842         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 843
 844         ecochk = I915_READ(GAM_ECOCHK);
 845         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
 846
 847         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 848
 849         for_each_ring(ring, dev_priv, i) {
 850                 int ret = ppgtt->switch_mm(ppgtt, ring, true);
 851                 if (ret)
 852                         return ret;
 853         }
 854
 855         return 0;
 856 }
 857
 858 /* PPGTT support for Sandybdrige/Gen6 and later */
 859 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 860                                    uint64_t start,
 861                                    uint64_t length,
 862                                    bool use_scratch)
 863 {
 864         struct i915_hw_ppgtt *ppgtt =
 865                 container_of(vm, struct i915_hw_ppgtt, base);
 866         gen6_gtt_pte_t *pt_vaddr, scratch_pte;
 867         unsigned first_entry = start >> PAGE_SHIFT;
 868         unsigned num_entries = length >> PAGE_SHIFT;
 869         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 870         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 871         unsigned last_pte, i;
 872
 873         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
 874
 875         while (num_entries) {
 876                 last_pte = first_pte + num_entries;
 877                 if (last_pte > I915_PPGTT_PT_ENTRIES)
 878                         last_pte = I915_PPGTT_PT_ENTRIES;
 879
 880                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 881
 882                 for (i = first_pte; i < last_pte; i++)
 883                         pt_vaddr[i] = scratch_pte;
 884
 885                 kunmap_atomic(pt_vaddr);
 886
 887                 num_entries -= last_pte - first_pte;
 888                 first_pte = 0;
 889                 act_pt++;
 890         }
 891 }
 892
 893 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 894                                       struct sg_table *pages,
 895                                       uint64_t start,
 896                                       enum i915_cache_level cache_level)
 897 {
 898         struct i915_hw_ppgtt *ppgtt =
 899                 container_of(vm, struct i915_hw_ppgtt, base);
 900         gen6_gtt_pte_t *pt_vaddr;
 901         unsigned first_entry = start >> PAGE_SHIFT;
 902         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 903         unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 904         struct sg_page_iter sg_iter;
 905
 906         pt_vaddr = NULL;
 907         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 908                 if (pt_vaddr == NULL)
 909                         pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
 910
 911                 pt_vaddr[act_pte] =
 912                         vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
 913                                        cache_level, true);
 914                 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
 915                         kunmap_atomic(pt_vaddr);
 916                         pt_vaddr = NULL;
 917                         act_pt++;
 918                         act_pte = 0;
 919                 }
 920         }
 921         if (pt_vaddr)
 922                 kunmap_atomic(pt_vaddr);
 923 }
 924
 925 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 926 {
 927         struct i915_hw_ppgtt *ppgtt =
 928                 container_of(vm, struct i915_hw_ppgtt, base);
 929         int i;
 930
 931         list_del(&vm->global_link);
 932         drm_mm_takedown(&ppgtt->base.mm);
 933         drm_mm_remove_node(&ppgtt->node);
 934
 935         if (ppgtt->pt_dma_addr) {
 936                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 937                         pci_unmap_page(ppgtt->base.dev->pdev,
 938                                        ppgtt->pt_dma_addr[i],
 939                                        4096, PCI_DMA_BIDIRECTIONAL);
 940         }
 941
 942         kfree(ppgtt->pt_dma_addr);
 943         for (i = 0; i < ppgtt->num_pd_entries; i++)
 944                 __free_page(ppgtt->pt_pages[i]);
 945         kfree(ppgtt->pt_pages);
 946 }
 947
 948 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 949 {
 950 #define GEN6_PD_ALIGN (PAGE_SIZE * 16)
 951 #define GEN6_PD_SIZE (GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE)
 952         struct drm_device *dev = ppgtt->base.dev;
 953         struct drm_i915_private *dev_priv = dev->dev_private;
 954         bool retried = false;
 955         int i, ret;
 956
 957         /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
 958          * allocator works in address space sizes, so it's multiplied by page
 959          * size. We allocate at the top of the GTT to avoid fragmentation.
 960          */
 961         BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
 962 alloc:
 963         ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
 964                                                   &ppgtt->node, GEN6_PD_SIZE,
 965                                                   GEN6_PD_ALIGN, 0,
 966                                                   0, dev_priv->gtt.base.total,
 967                                                   DRM_MM_SEARCH_DEFAULT);
 968         if (ret == -ENOSPC && !retried) {
 969                 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
 970                                                GEN6_PD_SIZE, GEN6_PD_ALIGN,
 971                                                I915_CACHE_NONE, 0);
 972                 if (ret)
 973                         return ret;
 974
 975                 retried = true;
 976                 goto alloc;
 977         }
 978
 979         if (ppgtt->node.start < dev_priv->gtt.mappable_end)
 980                 DRM_DEBUG("Forced to use aperture for PDEs\n");
 981
 982         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
 983         ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
 984         if (IS_GEN6(dev)) {
 985                 ppgtt->enable = gen6_ppgtt_enable;
 986                 ppgtt->switch_mm = gen6_mm_switch;
 987         } else if (IS_HASWELL(dev)) {
 988                 ppgtt->enable = gen7_ppgtt_enable;
 989                 ppgtt->switch_mm = hsw_mm_switch;
 990         } else if (IS_GEN7(dev)) {
 991                 ppgtt->enable = gen7_ppgtt_enable;
 992                 ppgtt->switch_mm = gen7_mm_switch;
 993         } else
 994                 BUG();
 995         ppgtt->base.clear_range = gen6_ppgtt_clear_range;
 996         ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
 997         ppgtt->base.cleanup = gen6_ppgtt_cleanup;
 998         ppgtt->base.scratch = dev_priv->gtt.base.scratch;
 999         ppgtt->base.start = 0;
1000         ppgtt->base.total = GEN6_PPGTT_PD_ENTRIES * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
1001         ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
1002                                   GFP_KERNEL);
1003         if (!ppgtt->pt_pages) {
1004                 drm_mm_remove_node(&ppgtt->node);
1005                 return -ENOMEM;
1006         }
1007
1008         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1009                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
1010                 if (!ppgtt->pt_pages[i])
1011                         goto err_pt_alloc;
1012         }
1013
1014         ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
1015                                      GFP_KERNEL);
1016         if (!ppgtt->pt_dma_addr)
1017                 goto err_pt_alloc;
1018
1019         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1020                 dma_addr_t pt_addr;
1021
1022                 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
1023                                        PCI_DMA_BIDIRECTIONAL);
1024
1025                 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
1026                         ret = -EIO;
1027                         goto err_pd_pin;
1028
1029                 }
1030                 ppgtt->pt_dma_addr[i] = pt_addr;
1031         }
1032
1033         ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1034         ppgtt->debug_dump = gen6_dump_ppgtt;
1035
1036         DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
1037                          ppgtt->node.size >> 20,
1038                          ppgtt->node.start / PAGE_SIZE);
1039         ppgtt->pd_offset =
1040                 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
1041
1042         return 0;
1043
1044 err_pd_pin:
1045         if (ppgtt->pt_dma_addr) {
1046                 for (i--; i >= 0; i--)
1047                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
1048                                        4096, PCI_DMA_BIDIRECTIONAL);
1049         }
1050 err_pt_alloc:
1051         kfree(ppgtt->pt_dma_addr);
1052         for (i = 0; i < ppgtt->num_pd_entries; i++) {
1053                 if (ppgtt->pt_pages[i])
1054                         __free_page(ppgtt->pt_pages[i]);
1055         }
1056         kfree(ppgtt->pt_pages);
1057         drm_mm_remove_node(&ppgtt->node);
1058
1059         return ret;
1060 }
1061
1062 int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1063 {
1064         struct drm_i915_private *dev_priv = dev->dev_private;
1065         int ret = 0;
1066
1067         ppgtt->base.dev = dev;
1068
1069         if (INTEL_INFO(dev)->gen < 8)
1070                 ret = gen6_ppgtt_init(ppgtt);
1071         else if (IS_GEN8(dev))
1072                 ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
1073         else
1074                 BUG();
1075
1076         if (!ret) {
1077                 struct drm_i915_private *dev_priv = dev->dev_private;
1078                 kref_init(&ppgtt->ref);
1079                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1080                             ppgtt->base.total);
1081                 i915_init_vm(dev_priv, &ppgtt->base);
1082                 if (INTEL_INFO(dev)->gen < 8) {
1083                         gen6_write_pdes(ppgtt);
1084                         DRM_DEBUG("Adding PPGTT at offset %x\n",
1085                                   ppgtt->pd_offset << 10);
1086                 }
1087         }
1088
1089         return ret;
1090 }
1091
1092 static void
1093 ppgtt_bind_vma(struct i915_vma *vma,
1094                enum i915_cache_level cache_level,
1095                u32 flags)
1096 {
1097         WARN_ON(flags);
1098
1099         vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
1100                                 cache_level);
1101 }
1102
1103 static void ppgtt_unbind_vma(struct i915_vma *vma)
1104 {
1105         vma->vm->clear_range(vma->vm,
1106                              vma->node.start,
1107                              vma->obj->base.size,
1108                              true);
1109 }
1110
1111 extern int intel_iommu_gfx_mapped;
1112 /* Certain Gen5 chipsets require require idling the GPU before
1113  * unmapping anything from the GTT when VT-d is enabled.
1114  */
1115 static inline bool needs_idle_maps(struct drm_device *dev)
1116 {
1117 #ifdef CONFIG_INTEL_IOMMU
1118         /* Query intel_iommu to see if we need the workaround. Presumably that
1119          * was loaded first.
1120          */
1121         if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1122                 return true;
1123 #endif
1124         return false;
1125 }
1126
1127 static bool do_idling(struct drm_i915_private *dev_priv)
1128 {
1129         bool ret = dev_priv->mm.interruptible;
1130
1131         if (unlikely(dev_priv->gtt.do_idle_maps)) {
1132                 dev_priv->mm.interruptible = false;
1133                 if (i915_gpu_idle(dev_priv->dev)) {
1134                         DRM_ERROR("Couldn't idle GPU\n");
1135                         /* Wait a bit, in hopes it avoids the hang */
1136                         udelay(10);
1137                 }
1138         }
1139
1140         return ret;
1141 }
1142
1143 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1144 {
1145         if (unlikely(dev_priv->gtt.do_idle_maps))
1146                 dev_priv->mm.interruptible = interruptible;
1147 }
1148
1149 void i915_check_and_clear_faults(struct drm_device *dev)
1150 {
1151         struct drm_i915_private *dev_priv = dev->dev_private;
1152         struct intel_ring_buffer *ring;
1153         int i;
1154
1155         if (INTEL_INFO(dev)->gen < 6)
1156                 return;
1157
1158         for_each_ring(ring, dev_priv, i) {
1159                 u32 fault_reg;
1160                 fault_reg = I915_READ(RING_FAULT_REG(ring));
1161                 if (fault_reg & RING_FAULT_VALID) {
1162                         DRM_DEBUG_DRIVER("Unexpected fault\n"
1163                                          "\tAddr: 0x%08lx\\n"
1164                                          "\tAddress space: %s\n"
1165                                          "\tSource ID: %d\n"
1166                                          "\tType: %d\n",
1167                                          fault_reg & PAGE_MASK,
1168                                          fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1169                                          RING_FAULT_SRCID(fault_reg),
1170                                          RING_FAULT_FAULT_TYPE(fault_reg));
1171                         I915_WRITE(RING_FAULT_REG(ring),
1172                                    fault_reg & ~RING_FAULT_VALID);
1173                 }
1174         }
1175         POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1176 }
1177
1178 void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1179 {
1180         struct drm_i915_private *dev_priv = dev->dev_private;
1181
1182         /* Don't bother messing with faults pre GEN6 as we have little
1183          * documentation supporting that it's a good idea.
1184          */
1185         if (INTEL_INFO(dev)->gen < 6)
1186                 return;
1187
1188         i915_check_and_clear_faults(dev);
1189
1190         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1191                                        dev_priv->gtt.base.start,
1192                                        dev_priv->gtt.base.total,
1193                                        false);
1194 }
1195
1196 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1197 {
1198         struct drm_i915_private *dev_priv = dev->dev_private;
1199         struct drm_i915_gem_object *obj;
1200         struct i915_address_space *vm;
1201
1202         i915_check_and_clear_faults(dev);
1203
1204         /* First fill our portion of the GTT with scratch pages */
1205         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1206                                        dev_priv->gtt.base.start,
1207                                        dev_priv->gtt.base.total,
1208                                        true);
1209
1210         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1211                 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1212                                                            &dev_priv->gtt.base);
1213                 if (!vma)
1214                         continue;
1215
1216                 i915_gem_clflush_object(obj, obj->pin_display);
1217                 /* The bind_vma code tries to be smart about tracking mappings.
1218                  * Unfortunately above, we've just wiped out the mappings
1219                  * without telling our object about it. So we need to fake it.
1220                  */
1221                 obj->has_global_gtt_mapping = 0;
1222                 vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
1223         }
1224
1225
1226         if (INTEL_INFO(dev)->gen >= 8)
1227                 return;
1228
1229         list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1230                 /* TODO: Perhaps it shouldn't be gen6 specific */
1231                 if (i915_is_ggtt(vm)) {
1232                         if (dev_priv->mm.aliasing_ppgtt)
1233                                 gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
1234                         continue;
1235                 }
1236
1237                 gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
1238         }
1239
1240         i915_gem_chipset_flush(dev);
1241 }
1242
1243 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1244 {
1245         if (obj->has_dma_mapping)
1246                 return 0;
1247
1248         if (!dma_map_sg(&obj->base.dev->pdev->dev,
1249                         obj->pages->sgl, obj->pages->nents,
1250                         PCI_DMA_BIDIRECTIONAL))
1251                 return -ENOSPC;
1252
1253         return 0;
1254 }
1255
1256 static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
1257 {
1258 #ifdef writeq
1259         writeq(pte, addr);
1260 #else
1261         iowrite32((u32)pte, addr);
1262         iowrite32(pte >> 32, addr + 4);
1263 #endif
1264 }
1265
1266 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1267                                      struct sg_table *st,
1268                                      uint64_t start,
1269                                      enum i915_cache_level level)
1270 {
1271         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1272         unsigned first_entry = start >> PAGE_SHIFT;
1273         gen8_gtt_pte_t __iomem *gtt_entries =
1274                 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1275         int i = 0;
1276         struct sg_page_iter sg_iter;
1277         dma_addr_t addr;
1278
1279         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1280                 addr = sg_dma_address(sg_iter.sg) +
1281                         (sg_iter.sg_pgoffset << PAGE_SHIFT);
1282                 gen8_set_pte(&gtt_entries[i],
1283                              gen8_pte_encode(addr, level, true));
1284                 i++;
1285         }
1286
1287         /*
1288          * XXX: This serves as a posting read to make sure that the PTE has
1289          * actually been updated. There is some concern that even though
1290          * registers and PTEs are within the same BAR that they are potentially
1291          * of NUMA access patterns. Therefore, even with the way we assume
1292          * hardware should work, we must keep this posting read for paranoia.
1293          */
1294         if (i != 0)
1295                 WARN_ON(readq(&gtt_entries[i-1])
1296                         != gen8_pte_encode(addr, level, true));
1297
1298         /* This next bit makes the above posting read even more important. We
1299          * want to flush the TLBs only after we're certain all the PTE updates
1300          * have finished.
1301          */
1302         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1303         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1304 }
1305
1306 /*
1307  * Binds an object into the global gtt with the specified cache level. The object
1308  * will be accessible to the GPU via commands whose operands reference offsets
1309  * within the global GTT as well as accessible by the GPU through the GMADR
1310  * mapped BAR (dev_priv->mm.gtt->gtt).
1311  */
1312 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1313                                      struct sg_table *st,
1314                                      uint64_t start,
1315                                      enum i915_cache_level level)
1316 {
1317         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1318         unsigned first_entry = start >> PAGE_SHIFT;
1319         gen6_gtt_pte_t __iomem *gtt_entries =
1320                 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1321         int i = 0;
1322         struct sg_page_iter sg_iter;
1323         dma_addr_t addr;
1324
1325         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1326                 addr = sg_page_iter_dma_address(&sg_iter);
1327                 iowrite32(vm->pte_encode(addr, level, true), &gtt_entries[i]);
1328                 i++;
1329         }
1330
1331         /* XXX: This serves as a posting read to make sure that the PTE has
1332          * actually been updated. There is some concern that even though
1333          * registers and PTEs are within the same BAR that they are potentially
1334          * of NUMA access patterns. Therefore, even with the way we assume
1335          * hardware should work, we must keep this posting read for paranoia.
1336          */
1337         if (i != 0)
1338                 WARN_ON(readl(&gtt_entries[i-1]) !=
1339                         vm->pte_encode(addr, level, true));
1340
1341         /* This next bit makes the above posting read even more important. We
1342          * want to flush the TLBs only after we're certain all the PTE updates
1343          * have finished.
1344          */
1345         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1346         POSTING_READ(GFX_FLSH_CNTL_GEN6);
1347 }
1348
1349 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1350                                   uint64_t start,
1351                                   uint64_t length,
1352                                   bool use_scratch)
1353 {
1354         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1355         unsigned first_entry = start >> PAGE_SHIFT;
1356         unsigned num_entries = length >> PAGE_SHIFT;
1357         gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
1358                 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1359         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1360         int i;
1361
1362         if (WARN(num_entries > max_entries,
1363                  "First entry = %d; Num entries = %d (max=%d)\n",
1364                  first_entry, num_entries, max_entries))
1365                 num_entries = max_entries;
1366
1367         scratch_pte = gen8_pte_encode(vm->scratch.addr,
1368                                       I915_CACHE_LLC,
1369                                       use_scratch);
1370         for (i = 0; i < num_entries; i++)
1371                 gen8_set_pte(&gtt_base[i], scratch_pte);
1372         readl(gtt_base);
1373 }
1374
1375 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1376                                   uint64_t start,
1377                                   uint64_t length,
1378                                   bool use_scratch)
1379 {
1380         struct drm_i915_private *dev_priv = vm->dev->dev_private;
1381         unsigned first_entry = start >> PAGE_SHIFT;
1382         unsigned num_entries = length >> PAGE_SHIFT;
1383         gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1384                 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1385         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1386         int i;
1387
1388         if (WARN(num_entries > max_entries,
1389                  "First entry = %d; Num entries = %d (max=%d)\n",
1390                  first_entry, num_entries, max_entries))
1391                 num_entries = max_entries;
1392
1393         scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch);
1394
1395         for (i = 0; i < num_entries; i++)
1396                 iowrite32(scratch_pte, &gtt_base[i]);
1397         readl(gtt_base);
1398 }
1399
1400
1401 static void i915_ggtt_bind_vma(struct i915_vma *vma,
1402                                enum i915_cache_level cache_level,
1403                                u32 unused)
1404 {
1405         const unsigned long entry = vma->node.start >> PAGE_SHIFT;
1406         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1407                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1408
1409         BUG_ON(!i915_is_ggtt(vma->vm));
1410         intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
1411         vma->obj->has_global_gtt_mapping = 1;
1412 }
1413
1414 static void i915_ggtt_clear_range(struct i915_address_space *vm,
1415                                   uint64_t start,
1416                                   uint64_t length,
1417                                   bool unused)
1418 {
1419         unsigned first_entry = start >> PAGE_SHIFT;
1420         unsigned num_entries = length >> PAGE_SHIFT;
1421         intel_gtt_clear_range(first_entry, num_entries);
1422 }
1423
1424 static void i915_ggtt_unbind_vma(struct i915_vma *vma)
1425 {
1426         const unsigned int first = vma->node.start >> PAGE_SHIFT;
1427         const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
1428
1429         BUG_ON(!i915_is_ggtt(vma->vm));
1430         vma->obj->has_global_gtt_mapping = 0;
1431         intel_gtt_clear_range(first, size);
1432 }
1433
1434 static void ggtt_bind_vma(struct i915_vma *vma,
1435                           enum i915_cache_level cache_level,
1436                           u32 flags)
1437 {
1438         struct drm_device *dev = vma->vm->dev;
1439         struct drm_i915_private *dev_priv = dev->dev_private;
1440         struct drm_i915_gem_object *obj = vma->obj;
1441
1442         /* If there is no aliasing PPGTT, or the caller needs a global mapping,
1443          * or we have a global mapping already but the cacheability flags have
1444          * changed, set the global PTEs.
1445          *
1446          * If there is an aliasing PPGTT it is anecdotally faster, so use that
1447          * instead if none of the above hold true.
1448          *
1449          * NB: A global mapping should only be needed for special regions like
1450          * "gtt mappable", SNB errata, or if specified via special execbuf
1451          * flags. At all other times, the GPU will use the aliasing PPGTT.
1452          */
1453         if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
1454                 if (!obj->has_global_gtt_mapping ||
1455                     (cache_level != obj->cache_level)) {
1456                         vma->vm->insert_entries(vma->vm, obj->pages,
1457                                                 vma->node.start,
1458                                                 cache_level);
1459                         obj->has_global_gtt_mapping = 1;
1460                 }
1461         }
1462
1463         if (dev_priv->mm.aliasing_ppgtt &&
1464             (!obj->has_aliasing_ppgtt_mapping ||
1465              (cache_level != obj->cache_level))) {
1466                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1467                 appgtt->base.insert_entries(&appgtt->base,
1468                                             vma->obj->pages,
1469                                             vma->node.start,
1470                                             cache_level);
1471                 vma->obj->has_aliasing_ppgtt_mapping = 1;
1472         }
1473 }
1474
1475 static void ggtt_unbind_vma(struct i915_vma *vma)
1476 {
1477         struct drm_device *dev = vma->vm->dev;
1478         struct drm_i915_private *dev_priv = dev->dev_private;
1479         struct drm_i915_gem_object *obj = vma->obj;
1480
1481         if (obj->has_global_gtt_mapping) {
1482                 vma->vm->clear_range(vma->vm,
1483                                      vma->node.start,
1484                                      obj->base.size,
1485                                      true);
1486                 obj->has_global_gtt_mapping = 0;
1487         }
1488
1489         if (obj->has_aliasing_ppgtt_mapping) {
1490                 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1491                 appgtt->base.clear_range(&appgtt->base,
1492                                          vma->node.start,
1493                                          obj->base.size,
1494                                          true);
1495                 obj->has_aliasing_ppgtt_mapping = 0;
1496         }
1497 }
1498
1499 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1500 {
1501         struct drm_device *dev = obj->base.dev;
1502         struct drm_i915_private *dev_priv = dev->dev_private;
1503         bool interruptible;
1504
1505         interruptible = do_idling(dev_priv);
1506
1507         if (!obj->has_dma_mapping)
1508                 dma_unmap_sg(&dev->pdev->dev,
1509                              obj->pages->sgl, obj->pages->nents,
1510                              PCI_DMA_BIDIRECTIONAL);
1511
1512         undo_idling(dev_priv, interruptible);
1513 }
1514
1515 static void i915_gtt_color_adjust(struct drm_mm_node *node,
1516                                   unsigned long color,
1517                                   unsigned long *start,
1518                                   unsigned long *end)
1519 {
1520         if (node->color != color)
1521                 *start += 4096;
1522
1523         if (!list_empty(&node->node_list)) {
1524                 node = list_entry(node->node_list.next,
1525                                   struct drm_mm_node,
1526                                   node_list);
1527                 if (node->allocated && node->color != color)
1528                         *end -= 4096;
1529         }
1530 }
1531
1532 void i915_gem_setup_global_gtt(struct drm_device *dev,
1533                                unsigned long start,
1534                                unsigned long mappable_end,
1535                                unsigned long end)
1536 {
1537         /* Let GEM Manage all of the aperture.
1538          *
1539          * However, leave one page at the end still bound to the scratch page.
1540          * There are a number of places where the hardware apparently prefetches
1541          * past the end of the object, and we've seen multiple hangs with the
1542          * GPU head pointer stuck in a batchbuffer bound at the last page of the
1543          * aperture.  One page should be enough to keep any prefetching inside
1544          * of the aperture.
1545          */
1546         struct drm_i915_private *dev_priv = dev->dev_private;
1547         struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
1548         struct drm_mm_node *entry;
1549         struct drm_i915_gem_object *obj;
1550         unsigned long hole_start, hole_end;
1551
1552         BUG_ON(mappable_end > end);
1553
1554         /* Subtract the guard page ... */
1555         drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
1556         if (!HAS_LLC(dev))
1557                 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
1558
1559         /* Mark any preallocated objects as occupied */
1560         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1561                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
1562                 int ret;
1563                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
1564                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
1565
1566                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
1567                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
1568                 if (ret)
1569                         DRM_DEBUG_KMS("Reservation failed\n");
1570                 obj->has_global_gtt_mapping = 1;
1571         }
1572
1573         dev_priv->gtt.base.start = start;
1574         dev_priv->gtt.base.total = end - start;
1575
1576         /* Clear any non-preallocated blocks */
1577         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
1578                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1579                               hole_start, hole_end);
1580                 ggtt_vm->clear_range(ggtt_vm, hole_start,
1581                                      hole_end - hole_start, true);
1582         }
1583
1584         /* And finally clear the reserved guard page */
1585         ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
1586 }
1587
1588 void i915_gem_init_global_gtt(struct drm_device *dev)
1589 {
1590         struct drm_i915_private *dev_priv = dev->dev_private;
1591         unsigned long gtt_size, mappable_size;
1592
1593         gtt_size = dev_priv->gtt.base.total;
1594         mappable_size = dev_priv->gtt.mappable_end;
1595
1596         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
1597 }
1598
1599 static int setup_scratch_page(struct drm_device *dev)
1600 {
1601         struct drm_i915_private *dev_priv = dev->dev_private;
1602         struct page *page;
1603         dma_addr_t dma_addr;
1604
1605         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
1606         if (page == NULL)
1607                 return -ENOMEM;
1608         get_page(page);
1609         set_pages_uc(page, 1);
1610
1611 #ifdef CONFIG_INTEL_IOMMU
1612         dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
1613                                 PCI_DMA_BIDIRECTIONAL);
1614         if (pci_dma_mapping_error(dev->pdev, dma_addr))
1615                 return -EINVAL;
1616 #else
1617         dma_addr = page_to_phys(page);
1618 #endif
1619         dev_priv->gtt.base.scratch.page = page;
1620         dev_priv->gtt.base.scratch.addr = dma_addr;
1621
1622         return 0;
1623 }
1624
1625 static void teardown_scratch_page(struct drm_device *dev)
1626 {
1627         struct drm_i915_private *dev_priv = dev->dev_private;
1628         struct page *page = dev_priv->gtt.base.scratch.page;
1629
1630         set_pages_wb(page, 1);
1631         pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
1632                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
1633         put_page(page);
1634         __free_page(page);
1635 }
1636
1637 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1638 {
1639         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1640         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1641         return snb_gmch_ctl << 20;
1642 }
1643
1644 static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1645 {
1646         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1647         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1648         if (bdw_gmch_ctl)
1649                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1650         if (bdw_gmch_ctl > 4) {
1651                 WARN_ON(!i915.preliminary_hw_support);
1652                 return 4<<20;
1653         }
1654
1655         return bdw_gmch_ctl << 20;
1656 }
1657
1658 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1659 {
1660         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1661         snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
1662         return snb_gmch_ctl << 25; /* 32 MB units */
1663 }
1664
1665 static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
1666 {
1667         bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1668         bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1669         return bdw_gmch_ctl << 25; /* 32 MB units */
1670 }
1671
1672 static int ggtt_probe_common(struct drm_device *dev,
1673                              size_t gtt_size)
1674 {
1675         struct drm_i915_private *dev_priv = dev->dev_private;
1676         phys_addr_t gtt_bus_addr;
1677         int ret;
1678
1679         /* For Modern GENs the PTEs and register space are split in the BAR */
1680         gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
1681                 (pci_resource_len(dev->pdev, 0) / 2);
1682
1683         dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
1684         if (!dev_priv->gtt.gsm) {
1685                 DRM_ERROR("Failed to map the gtt page table\n");
1686                 return -ENOMEM;
1687         }
1688
1689         ret = setup_scratch_page(dev);
1690         if (ret) {
1691                 DRM_ERROR("Scratch setup failed\n");
1692                 /* iounmap will also get called at remove, but meh */
1693                 iounmap(dev_priv->gtt.gsm);
1694         }
1695
1696         return ret;
1697 }
1698
1699 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
1700  * bits. When using advanced contexts each context stores its own PAT, but
1701  * writing this data shouldn't be harmful even in those cases. */
1702 static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv)
1703 {
1704 #define GEN8_PPAT_UC            (0<<0)
1705 #define GEN8_PPAT_WC            (1<<0)
1706 #define GEN8_PPAT_WT            (2<<0)
1707 #define GEN8_PPAT_WB            (3<<0)
1708 #define GEN8_PPAT_ELLC_OVERRIDE (0<<2)
1709 /* FIXME(BDW): Bspec is completely confused about cache control bits. */
1710 #define GEN8_PPAT_LLC           (1<<2)
1711 #define GEN8_PPAT_LLCELLC       (2<<2)
1712 #define GEN8_PPAT_LLCeLLC       (3<<2)
1713 #define GEN8_PPAT_AGE(x)        (x<<4)
1714 #define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8))
1715         uint64_t pat;
1716
1717         pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
1718               GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
1719               GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
1720               GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
1721               GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
1722               GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
1723               GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
1724               GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
1725
1726         /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
1727          * write would work. */
1728         I915_WRITE(GEN8_PRIVATE_PAT, pat);
1729         I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1730 }
1731
1732 static int gen8_gmch_probe(struct drm_device *dev,
1733                            size_t *gtt_total,
1734                            size_t *stolen,
1735                            phys_addr_t *mappable_base,
1736                            unsigned long *mappable_end)
1737 {
1738         struct drm_i915_private *dev_priv = dev->dev_private;
1739         unsigned int gtt_size;
1740         u16 snb_gmch_ctl;
1741         int ret;
1742
1743         /* TODO: We're not aware of mappable constraints on gen8 yet */
1744         *mappable_base = pci_resource_start(dev->pdev, 2);
1745         *mappable_end = pci_resource_len(dev->pdev, 2);
1746
1747         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
1748                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
1749
1750         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1751
1752         *stolen = gen8_get_stolen_size(snb_gmch_ctl);
1753
1754         gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
1755         *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
1756
1757         gen8_setup_private_ppat(dev_priv);
1758
1759         ret = ggtt_probe_common(dev, gtt_size);
1760
1761         dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
1762         dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
1763
1764         return ret;
1765 }
1766
1767 static int gen6_gmch_probe(struct drm_device *dev,
1768                            size_t *gtt_total,
1769                            size_t *stolen,
1770                            phys_addr_t *mappable_base,
1771                            unsigned long *mappable_end)
1772 {
1773         struct drm_i915_private *dev_priv = dev->dev_private;
1774         unsigned int gtt_size;
1775         u16 snb_gmch_ctl;
1776         int ret;
1777
1778         *mappable_base = pci_resource_start(dev->pdev, 2);
1779         *mappable_end = pci_resource_len(dev->pdev, 2);
1780
1781         /* 64/512MB is the current min/max we actually know of, but this is just
1782          * a coarse sanity check.
1783          */
1784         if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
1785                 DRM_ERROR("Unknown GMADR size (%lx)\n",
1786                           dev_priv->gtt.mappable_end);
1787                 return -ENXIO;
1788         }
1789
1790         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
1791                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
1792         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1793
1794         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
1795
1796         gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
1797         *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
1798
1799         ret = ggtt_probe_common(dev, gtt_size);
1800
1801         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
1802         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
1803
1804         return ret;
1805 }
1806
1807 static void gen6_gmch_remove(struct i915_address_space *vm)
1808 {
1809
1810         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
1811
1812         drm_mm_takedown(&vm->mm);
1813         iounmap(gtt->gsm);
1814         teardown_scratch_page(vm->dev);
1815 }
1816
1817 static int i915_gmch_probe(struct drm_device *dev,
1818                            size_t *gtt_total,
1819                            size_t *stolen,
1820                            phys_addr_t *mappable_base,
1821                            unsigned long *mappable_end)
1822 {
1823         struct drm_i915_private *dev_priv = dev->dev_private;
1824         int ret;
1825
1826         ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
1827         if (!ret) {
1828                 DRM_ERROR("failed to set up gmch\n");
1829                 return -EIO;
1830         }
1831
1832         intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
1833
1834         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
1835         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
1836
1837         if (unlikely(dev_priv->gtt.do_idle_maps))
1838                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
1839
1840         return 0;
1841 }
1842
1843 static void i915_gmch_remove(struct i915_address_space *vm)
1844 {
1845         intel_gmch_remove();
1846 }
1847
1848 int i915_gem_gtt_init(struct drm_device *dev)
1849 {
1850         struct drm_i915_private *dev_priv = dev->dev_private;
1851         struct i915_gtt *gtt = &dev_priv->gtt;
1852         int ret;
1853
1854         if (INTEL_INFO(dev)->gen <= 5) {
1855                 gtt->gtt_probe = i915_gmch_probe;
1856                 gtt->base.cleanup = i915_gmch_remove;
1857         } else if (INTEL_INFO(dev)->gen < 8) {
1858                 gtt->gtt_probe = gen6_gmch_probe;
1859                 gtt->base.cleanup = gen6_gmch_remove;
1860                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
1861                         gtt->base.pte_encode = iris_pte_encode;
1862                 else if (IS_HASWELL(dev))
1863                         gtt->base.pte_encode = hsw_pte_encode;
1864                 else if (IS_VALLEYVIEW(dev))
1865                         gtt->base.pte_encode = byt_pte_encode;
1866                 else if (INTEL_INFO(dev)->gen >= 7)
1867                         gtt->base.pte_encode = ivb_pte_encode;
1868                 else
1869                         gtt->base.pte_encode = snb_pte_encode;
1870         } else {
1871                 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
1872                 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
1873         }
1874
1875         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
1876                              &gtt->mappable_base, &gtt->mappable_end);
1877         if (ret)
1878                 return ret;
1879
1880         gtt->base.dev = dev;
1881
1882         /* GMADR is the PCI mmio aperture into the global GTT. */
1883         DRM_INFO("Memory usable by graphics device = %zdM\n",
1884                  gtt->base.total >> 20);
1885         DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
1886         DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
1887
1888         return 0;
1889 }
1890
1891 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
1892                                               struct i915_address_space *vm)
1893 {
1894         struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
1895         if (vma == NULL)
1896                 return ERR_PTR(-ENOMEM);
1897
1898         INIT_LIST_HEAD(&vma->vma_link);
1899         INIT_LIST_HEAD(&vma->mm_list);
1900         INIT_LIST_HEAD(&vma->exec_list);
1901         vma->vm = vm;
1902         vma->obj = obj;
1903
1904         switch (INTEL_INFO(vm->dev)->gen) {
1905         case 8:
1906         case 7:
1907         case 6:
1908                 if (i915_is_ggtt(vm)) {
1909                         vma->unbind_vma = ggtt_unbind_vma;
1910                         vma->bind_vma = ggtt_bind_vma;
1911                 } else {
1912                         vma->unbind_vma = ppgtt_unbind_vma;
1913                         vma->bind_vma = ppgtt_bind_vma;
1914                 }
1915                 break;
1916         case 5:
1917         case 4:
1918         case 3:
1919         case 2:
1920                 BUG_ON(!i915_is_ggtt(vm));
1921                 vma->unbind_vma = i915_ggtt_unbind_vma;
1922                 vma->bind_vma = i915_ggtt_bind_vma;
1923                 break;
1924         default:
1925                 BUG();
1926         }
1927
1928         /* Keep GGTT vmas first to make debug easier */
1929         if (i915_is_ggtt(vm))
1930                 list_add(&vma->vma_link, &obj->vma_list);
1931         else
1932                 list_add_tail(&vma->vma_link, &obj->vma_list);
1933
1934         return vma;
1935 }
1936
1937 struct i915_vma *
1938 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
1939                                   struct i915_address_space *vm)
1940 {
1941         struct i915_vma *vma;
1942
1943         vma = i915_gem_obj_to_vma(obj, vm);
1944         if (!vma)
1945                 vma = __i915_gem_vma_create(obj, vm);
1946
1947         return vma;
1948 }