2 * Copyright © 2010 Daniel Vetter
3 * Copyright © 2011-2014 Intel Corporation
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <linux/seq_file.h>
27 #include <linux/stop_machine.h>
29 #include <drm/i915_drm.h>
31 #include "i915_vgpu.h"
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include "intel_frontbuffer.h"
36 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
39 * DOC: Global GTT views
41 * Background and previous state
43 * Historically objects could exists (be bound) in global GTT space only as
44 * singular instances with a view representing all of the object's backing pages
45 * in a linear fashion. This view will be called a normal view.
47 * To support multiple views of the same object, where the number of mapped
48 * pages is not equal to the backing store, or where the layout of the pages
49 * is not linear, concept of a GGTT view was added.
51 * One example of an alternative view is a stereo display driven by a single
52 * image. In this case we would have a framebuffer looking like this
58 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
59 * rendering. In contrast, fed to the display engine would be an alternative
60 * view which could look something like this:
65 * In this example both the size and layout of pages in the alternative view is
66 * different from the normal view.
68 * Implementation and usage
70 * GGTT views are implemented using VMAs and are distinguished via enum
71 * i915_ggtt_view_type and struct i915_ggtt_view.
73 * A new flavour of core GEM functions which work with GGTT bound objects were
74 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
75 * renaming in large amounts of code. They take the struct i915_ggtt_view
76 * parameter encapsulating all metadata required to implement a view.
78 * As a helper for callers which are only interested in the normal view,
79 * globally const i915_ggtt_view_normal singleton instance exists. All old core
80 * GEM API functions, the ones not taking the view parameter, are operating on,
81 * or with the normal GGTT view.
83 * Code wanting to add or use a new GGTT view needs to:
85 * 1. Add a new enum with a suitable name.
86 * 2. Extend the metadata in the i915_ggtt_view structure if required.
87 * 3. Add support to i915_get_vma_pages().
89 * New views are required to build a scatter-gather table from within the
90 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
91 * exists for the lifetime of an VMA.
93 * Core API is designed to have copy semantics which means that passed in
94 * struct i915_ggtt_view does not need to be persistent (left around after
95 * calling the core API functions).
100 i915_get_ggtt_vma_pages(struct i915_vma *vma);
102 const struct i915_ggtt_view i915_ggtt_view_normal = {
103 .type = I915_GGTT_VIEW_NORMAL,
105 const struct i915_ggtt_view i915_ggtt_view_rotated = {
106 .type = I915_GGTT_VIEW_ROTATED,
109 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
112 bool has_aliasing_ppgtt;
114 bool has_full_48bit_ppgtt;
116 has_aliasing_ppgtt = dev_priv->info.has_aliasing_ppgtt;
117 has_full_ppgtt = dev_priv->info.has_full_ppgtt;
118 has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
120 if (intel_vgpu_active(dev_priv)) {
121 /* emulation is too hard */
122 has_full_ppgtt = false;
123 has_full_48bit_ppgtt = false;
126 if (!has_aliasing_ppgtt)
130 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
131 * execlists, the sole mechanism available to submit work.
133 if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
136 if (enable_ppgtt == 1)
139 if (enable_ppgtt == 2 && has_full_ppgtt)
142 if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
145 #ifdef CONFIG_INTEL_IOMMU
146 /* Disable ppgtt on SNB if VT-d is on. */
147 if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
148 DRM_INFO("Disabling PPGTT because VT-d is on\n");
153 /* Early VLV doesn't have this */
154 if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
155 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
159 if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
160 return has_full_48bit_ppgtt ? 3 : 2;
162 return has_aliasing_ppgtt ? 1 : 0;
165 static int ppgtt_bind_vma(struct i915_vma *vma,
166 enum i915_cache_level cache_level,
171 vma->pages = vma->obj->mm.pages;
173 /* Currently applicable only to VLV */
175 pte_flags |= PTE_READ_ONLY;
177 vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
178 cache_level, pte_flags);
183 static void ppgtt_unbind_vma(struct i915_vma *vma)
185 vma->vm->clear_range(vma->vm,
190 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
191 enum i915_cache_level level)
193 gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
197 case I915_CACHE_NONE:
198 pte |= PPAT_UNCACHED_INDEX;
201 pte |= PPAT_DISPLAY_ELLC_INDEX;
204 pte |= PPAT_CACHED_INDEX;
211 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
212 const enum i915_cache_level level)
214 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
216 if (level != I915_CACHE_NONE)
217 pde |= PPAT_CACHED_PDE_INDEX;
219 pde |= PPAT_UNCACHED_INDEX;
223 #define gen8_pdpe_encode gen8_pde_encode
224 #define gen8_pml4e_encode gen8_pde_encode
226 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
227 enum i915_cache_level level,
230 gen6_pte_t pte = GEN6_PTE_VALID;
231 pte |= GEN6_PTE_ADDR_ENCODE(addr);
234 case I915_CACHE_L3_LLC:
236 pte |= GEN6_PTE_CACHE_LLC;
238 case I915_CACHE_NONE:
239 pte |= GEN6_PTE_UNCACHED;
248 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
249 enum i915_cache_level level,
252 gen6_pte_t pte = GEN6_PTE_VALID;
253 pte |= GEN6_PTE_ADDR_ENCODE(addr);
256 case I915_CACHE_L3_LLC:
257 pte |= GEN7_PTE_CACHE_L3_LLC;
260 pte |= GEN6_PTE_CACHE_LLC;
262 case I915_CACHE_NONE:
263 pte |= GEN6_PTE_UNCACHED;
272 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
273 enum i915_cache_level level,
276 gen6_pte_t pte = GEN6_PTE_VALID;
277 pte |= GEN6_PTE_ADDR_ENCODE(addr);
279 if (!(flags & PTE_READ_ONLY))
280 pte |= BYT_PTE_WRITEABLE;
282 if (level != I915_CACHE_NONE)
283 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
288 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
289 enum i915_cache_level level,
292 gen6_pte_t pte = GEN6_PTE_VALID;
293 pte |= HSW_PTE_ADDR_ENCODE(addr);
295 if (level != I915_CACHE_NONE)
296 pte |= HSW_WB_LLC_AGE3;
301 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
302 enum i915_cache_level level,
305 gen6_pte_t pte = GEN6_PTE_VALID;
306 pte |= HSW_PTE_ADDR_ENCODE(addr);
309 case I915_CACHE_NONE:
312 pte |= HSW_WT_ELLC_LLC_AGE3;
315 pte |= HSW_WB_ELLC_LLC_AGE3;
322 static int __setup_page_dma(struct drm_i915_private *dev_priv,
323 struct i915_page_dma *p, gfp_t flags)
325 struct device *kdev = &dev_priv->drm.pdev->dev;
327 p->page = alloc_page(flags);
331 p->daddr = dma_map_page(kdev,
332 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
334 if (dma_mapping_error(kdev, p->daddr)) {
335 __free_page(p->page);
342 static int setup_page_dma(struct drm_i915_private *dev_priv,
343 struct i915_page_dma *p)
345 return __setup_page_dma(dev_priv, p, I915_GFP_DMA);
348 static void cleanup_page_dma(struct drm_i915_private *dev_priv,
349 struct i915_page_dma *p)
351 struct pci_dev *pdev = dev_priv->drm.pdev;
353 if (WARN_ON(!p->page))
356 dma_unmap_page(&pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
357 __free_page(p->page);
358 memset(p, 0, sizeof(*p));
361 static void *kmap_page_dma(struct i915_page_dma *p)
363 return kmap_atomic(p->page);
366 /* We use the flushing unmap only with ppgtt structures:
367 * page directories, page tables and scratch pages.
369 static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr)
371 /* There are only few exceptions for gen >=6. chv and bxt.
372 * And we are not sure about the latter so play safe for now.
374 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
375 drm_clflush_virt_range(vaddr, PAGE_SIZE);
377 kunmap_atomic(vaddr);
380 #define kmap_px(px) kmap_page_dma(px_base(px))
381 #define kunmap_px(ppgtt, vaddr) \
382 kunmap_page_dma((ppgtt)->base.i915, (vaddr))
384 #define setup_px(dev_priv, px) setup_page_dma((dev_priv), px_base(px))
385 #define cleanup_px(dev_priv, px) cleanup_page_dma((dev_priv), px_base(px))
386 #define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v))
387 #define fill32_px(dev_priv, px, v) \
388 fill_page_dma_32((dev_priv), px_base(px), (v))
390 static void fill_page_dma(struct drm_i915_private *dev_priv,
391 struct i915_page_dma *p, const uint64_t val)
394 uint64_t * const vaddr = kmap_page_dma(p);
396 for (i = 0; i < 512; i++)
399 kunmap_page_dma(dev_priv, vaddr);
402 static void fill_page_dma_32(struct drm_i915_private *dev_priv,
403 struct i915_page_dma *p, const uint32_t val32)
409 fill_page_dma(dev_priv, p, v);
413 setup_scratch_page(struct drm_i915_private *dev_priv,
414 struct i915_page_dma *scratch,
417 return __setup_page_dma(dev_priv, scratch, gfp | __GFP_ZERO);
420 static void cleanup_scratch_page(struct drm_i915_private *dev_priv,
421 struct i915_page_dma *scratch)
423 cleanup_page_dma(dev_priv, scratch);
426 static struct i915_page_table *alloc_pt(struct drm_i915_private *dev_priv)
428 struct i915_page_table *pt;
429 const size_t count = INTEL_GEN(dev_priv) >= 8 ? GEN8_PTES : GEN6_PTES;
432 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
434 return ERR_PTR(-ENOMEM);
436 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
442 ret = setup_px(dev_priv, pt);
449 kfree(pt->used_ptes);
456 static void free_pt(struct drm_i915_private *dev_priv,
457 struct i915_page_table *pt)
459 cleanup_px(dev_priv, pt);
460 kfree(pt->used_ptes);
464 static void gen8_initialize_pt(struct i915_address_space *vm,
465 struct i915_page_table *pt)
467 gen8_pte_t scratch_pte;
469 scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
472 fill_px(vm->i915, pt, scratch_pte);
475 static void gen6_initialize_pt(struct i915_address_space *vm,
476 struct i915_page_table *pt)
478 gen6_pte_t scratch_pte;
480 WARN_ON(vm->scratch_page.daddr == 0);
482 scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
485 fill32_px(vm->i915, pt, scratch_pte);
488 static struct i915_page_directory *alloc_pd(struct drm_i915_private *dev_priv)
490 struct i915_page_directory *pd;
493 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
495 return ERR_PTR(-ENOMEM);
497 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
498 sizeof(*pd->used_pdes), GFP_KERNEL);
502 ret = setup_px(dev_priv, pd);
509 kfree(pd->used_pdes);
516 static void free_pd(struct drm_i915_private *dev_priv,
517 struct i915_page_directory *pd)
520 cleanup_px(dev_priv, pd);
521 kfree(pd->used_pdes);
526 static void gen8_initialize_pd(struct i915_address_space *vm,
527 struct i915_page_directory *pd)
529 gen8_pde_t scratch_pde;
531 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
533 fill_px(vm->i915, pd, scratch_pde);
536 static int __pdp_init(struct drm_i915_private *dev_priv,
537 struct i915_page_directory_pointer *pdp)
539 size_t pdpes = I915_PDPES_PER_PDP(dev_priv);
541 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
542 sizeof(unsigned long),
544 if (!pdp->used_pdpes)
547 pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
549 if (!pdp->page_directory) {
550 kfree(pdp->used_pdpes);
551 /* the PDP might be the statically allocated top level. Keep it
552 * as clean as possible */
553 pdp->used_pdpes = NULL;
560 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
562 kfree(pdp->used_pdpes);
563 kfree(pdp->page_directory);
564 pdp->page_directory = NULL;
568 i915_page_directory_pointer *alloc_pdp(struct drm_i915_private *dev_priv)
570 struct i915_page_directory_pointer *pdp;
573 WARN_ON(!USES_FULL_48BIT_PPGTT(dev_priv));
575 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
577 return ERR_PTR(-ENOMEM);
579 ret = __pdp_init(dev_priv, pdp);
583 ret = setup_px(dev_priv, pdp);
597 static void free_pdp(struct drm_i915_private *dev_priv,
598 struct i915_page_directory_pointer *pdp)
601 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
602 cleanup_px(dev_priv, pdp);
607 static void gen8_initialize_pdp(struct i915_address_space *vm,
608 struct i915_page_directory_pointer *pdp)
610 gen8_ppgtt_pdpe_t scratch_pdpe;
612 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
614 fill_px(vm->i915, pdp, scratch_pdpe);
617 static void gen8_initialize_pml4(struct i915_address_space *vm,
618 struct i915_pml4 *pml4)
620 gen8_ppgtt_pml4e_t scratch_pml4e;
622 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
625 fill_px(vm->i915, pml4, scratch_pml4e);
629 gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
630 struct i915_page_directory_pointer *pdp,
631 struct i915_page_directory *pd,
634 gen8_ppgtt_pdpe_t *page_directorypo;
636 if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)))
639 page_directorypo = kmap_px(pdp);
640 page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
641 kunmap_px(ppgtt, page_directorypo);
645 gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
646 struct i915_pml4 *pml4,
647 struct i915_page_directory_pointer *pdp,
650 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
652 WARN_ON(!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)));
653 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
654 kunmap_px(ppgtt, pagemap);
657 /* Broadwell Page Directory Pointer Descriptors */
658 static int gen8_write_pdp(struct drm_i915_gem_request *req,
662 struct intel_ring *ring = req->ring;
663 struct intel_engine_cs *engine = req->engine;
668 ret = intel_ring_begin(req, 6);
672 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
673 intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(engine, entry));
674 intel_ring_emit(ring, upper_32_bits(addr));
675 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
676 intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(engine, entry));
677 intel_ring_emit(ring, lower_32_bits(addr));
678 intel_ring_advance(ring);
683 static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
684 struct drm_i915_gem_request *req)
688 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
689 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
691 ret = gen8_write_pdp(req, i, pd_daddr);
699 static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
700 struct drm_i915_gem_request *req)
702 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
705 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
706 * the page table structures, we mark them dirty so that
707 * context switching/execlist queuing code takes extra steps
708 * to ensure that tlbs are flushed.
710 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
712 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.i915)->ring_mask;
715 /* Removes entries from a single page table, releasing it if it's empty.
716 * Caller can use the return value to update higher-level entries.
718 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
719 struct i915_page_table *pt,
723 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
724 unsigned int num_entries = gen8_pte_count(start, length);
725 unsigned int pte = gen8_pte_index(start);
726 unsigned int pte_end = pte + num_entries;
727 gen8_pte_t *pt_vaddr;
728 gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
731 if (WARN_ON(!px_page(pt)))
734 GEM_BUG_ON(pte_end > GEN8_PTES);
736 bitmap_clear(pt->used_ptes, pte, num_entries);
738 if (bitmap_empty(pt->used_ptes, GEN8_PTES))
741 pt_vaddr = kmap_px(pt);
743 while (pte < pte_end)
744 pt_vaddr[pte++] = scratch_pte;
746 kunmap_px(ppgtt, pt_vaddr);
751 /* Removes entries from a single page dir, releasing it if it's empty.
752 * Caller can use the return value to update higher-level entries
754 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
755 struct i915_page_directory *pd,
759 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
760 struct i915_page_table *pt;
762 gen8_pde_t *pde_vaddr;
763 gen8_pde_t scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt),
766 gen8_for_each_pde(pt, pd, start, length, pde) {
767 if (WARN_ON(!pd->page_table[pde]))
770 if (gen8_ppgtt_clear_pt(vm, pt, start, length)) {
771 __clear_bit(pde, pd->used_pdes);
772 pde_vaddr = kmap_px(pd);
773 pde_vaddr[pde] = scratch_pde;
774 kunmap_px(ppgtt, pde_vaddr);
775 free_pt(vm->i915, pt);
779 if (bitmap_empty(pd->used_pdes, I915_PDES))
785 /* Removes entries from a single page dir pointer, releasing it if it's empty.
786 * Caller can use the return value to update higher-level entries
788 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
789 struct i915_page_directory_pointer *pdp,
793 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
794 struct i915_page_directory *pd;
796 gen8_ppgtt_pdpe_t *pdpe_vaddr;
797 gen8_ppgtt_pdpe_t scratch_pdpe =
798 gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
800 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
801 if (WARN_ON(!pdp->page_directory[pdpe]))
804 if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
805 __clear_bit(pdpe, pdp->used_pdpes);
806 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
807 pdpe_vaddr = kmap_px(pdp);
808 pdpe_vaddr[pdpe] = scratch_pdpe;
809 kunmap_px(ppgtt, pdpe_vaddr);
811 free_pd(vm->i915, pd);
815 mark_tlbs_dirty(ppgtt);
817 if (bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)))
823 /* Removes entries from a single pml4.
824 * This is the top-level structure in 4-level page tables used on gen8+.
825 * Empty entries are always scratch pml4e.
827 static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
828 struct i915_pml4 *pml4,
832 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
833 struct i915_page_directory_pointer *pdp;
835 gen8_ppgtt_pml4e_t *pml4e_vaddr;
836 gen8_ppgtt_pml4e_t scratch_pml4e =
837 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC);
839 GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->i915));
841 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
842 if (WARN_ON(!pml4->pdps[pml4e]))
845 if (gen8_ppgtt_clear_pdp(vm, pdp, start, length)) {
846 __clear_bit(pml4e, pml4->used_pml4es);
847 pml4e_vaddr = kmap_px(pml4);
848 pml4e_vaddr[pml4e] = scratch_pml4e;
849 kunmap_px(ppgtt, pml4e_vaddr);
850 free_pdp(vm->i915, pdp);
855 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
856 uint64_t start, uint64_t length)
858 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
860 if (USES_FULL_48BIT_PPGTT(vm->i915))
861 gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
863 gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
867 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
868 struct i915_page_directory_pointer *pdp,
869 struct sg_page_iter *sg_iter,
871 enum i915_cache_level cache_level)
873 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
874 gen8_pte_t *pt_vaddr;
875 unsigned pdpe = gen8_pdpe_index(start);
876 unsigned pde = gen8_pde_index(start);
877 unsigned pte = gen8_pte_index(start);
881 while (__sg_page_iter_next(sg_iter)) {
882 if (pt_vaddr == NULL) {
883 struct i915_page_directory *pd = pdp->page_directory[pdpe];
884 struct i915_page_table *pt = pd->page_table[pde];
885 pt_vaddr = kmap_px(pt);
889 gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
891 if (++pte == GEN8_PTES) {
892 kunmap_px(ppgtt, pt_vaddr);
894 if (++pde == I915_PDES) {
895 if (++pdpe == I915_PDPES_PER_PDP(vm->i915))
904 kunmap_px(ppgtt, pt_vaddr);
907 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
908 struct sg_table *pages,
910 enum i915_cache_level cache_level,
913 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
914 struct sg_page_iter sg_iter;
916 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
918 if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
919 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
922 struct i915_page_directory_pointer *pdp;
924 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
926 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
927 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
933 static void gen8_free_page_tables(struct drm_i915_private *dev_priv,
934 struct i915_page_directory *pd)
941 for_each_set_bit(i, pd->used_pdes, I915_PDES) {
942 if (WARN_ON(!pd->page_table[i]))
945 free_pt(dev_priv, pd->page_table[i]);
946 pd->page_table[i] = NULL;
950 static int gen8_init_scratch(struct i915_address_space *vm)
952 struct drm_i915_private *dev_priv = vm->i915;
955 ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
959 vm->scratch_pt = alloc_pt(dev_priv);
960 if (IS_ERR(vm->scratch_pt)) {
961 ret = PTR_ERR(vm->scratch_pt);
962 goto free_scratch_page;
965 vm->scratch_pd = alloc_pd(dev_priv);
966 if (IS_ERR(vm->scratch_pd)) {
967 ret = PTR_ERR(vm->scratch_pd);
971 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
972 vm->scratch_pdp = alloc_pdp(dev_priv);
973 if (IS_ERR(vm->scratch_pdp)) {
974 ret = PTR_ERR(vm->scratch_pdp);
979 gen8_initialize_pt(vm, vm->scratch_pt);
980 gen8_initialize_pd(vm, vm->scratch_pd);
981 if (USES_FULL_48BIT_PPGTT(dev_priv))
982 gen8_initialize_pdp(vm, vm->scratch_pdp);
987 free_pd(dev_priv, vm->scratch_pd);
989 free_pt(dev_priv, vm->scratch_pt);
991 cleanup_scratch_page(dev_priv, &vm->scratch_page);
996 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
998 enum vgt_g2v_type msg;
999 struct drm_i915_private *dev_priv = ppgtt->base.i915;
1002 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1003 u64 daddr = px_dma(&ppgtt->pml4);
1005 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1006 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1008 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1009 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1011 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
1012 u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1014 I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1015 I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1018 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1019 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1022 I915_WRITE(vgtif_reg(g2v_notify), msg);
1027 static void gen8_free_scratch(struct i915_address_space *vm)
1029 struct drm_i915_private *dev_priv = vm->i915;
1031 if (USES_FULL_48BIT_PPGTT(dev_priv))
1032 free_pdp(dev_priv, vm->scratch_pdp);
1033 free_pd(dev_priv, vm->scratch_pd);
1034 free_pt(dev_priv, vm->scratch_pt);
1035 cleanup_scratch_page(dev_priv, &vm->scratch_page);
1038 static void gen8_ppgtt_cleanup_3lvl(struct drm_i915_private *dev_priv,
1039 struct i915_page_directory_pointer *pdp)
1043 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)) {
1044 if (WARN_ON(!pdp->page_directory[i]))
1047 gen8_free_page_tables(dev_priv, pdp->page_directory[i]);
1048 free_pd(dev_priv, pdp->page_directory[i]);
1051 free_pdp(dev_priv, pdp);
1054 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1056 struct drm_i915_private *dev_priv = ppgtt->base.i915;
1059 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
1060 if (WARN_ON(!ppgtt->pml4.pdps[i]))
1063 gen8_ppgtt_cleanup_3lvl(dev_priv, ppgtt->pml4.pdps[i]);
1066 cleanup_px(dev_priv, &ppgtt->pml4);
1069 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1071 struct drm_i915_private *dev_priv = vm->i915;
1072 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1074 if (intel_vgpu_active(dev_priv))
1075 gen8_ppgtt_notify_vgt(ppgtt, false);
1077 if (!USES_FULL_48BIT_PPGTT(dev_priv))
1078 gen8_ppgtt_cleanup_3lvl(dev_priv, &ppgtt->pdp);
1080 gen8_ppgtt_cleanup_4lvl(ppgtt);
1082 gen8_free_scratch(vm);
1086 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
1087 * @vm: Master vm structure.
1088 * @pd: Page directory for this address range.
1089 * @start: Starting virtual address to begin allocations.
1090 * @length: Size of the allocations.
1091 * @new_pts: Bitmap set by function with new allocations. Likely used by the
1092 * caller to free on error.
1094 * Allocate the required number of page tables. Extremely similar to
1095 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
1096 * the page directory boundary (instead of the page directory pointer). That
1097 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
1098 * possible, and likely that the caller will need to use multiple calls of this
1099 * function to achieve the appropriate allocation.
1101 * Return: 0 if success; negative error code otherwise.
1103 static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1104 struct i915_page_directory *pd,
1107 unsigned long *new_pts)
1109 struct drm_i915_private *dev_priv = vm->i915;
1110 struct i915_page_table *pt;
1113 gen8_for_each_pde(pt, pd, start, length, pde) {
1114 /* Don't reallocate page tables */
1115 if (test_bit(pde, pd->used_pdes)) {
1116 /* Scratch is never allocated this way */
1117 WARN_ON(pt == vm->scratch_pt);
1121 pt = alloc_pt(dev_priv);
1125 gen8_initialize_pt(vm, pt);
1126 pd->page_table[pde] = pt;
1127 __set_bit(pde, new_pts);
1128 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
1134 for_each_set_bit(pde, new_pts, I915_PDES)
1135 free_pt(dev_priv, pd->page_table[pde]);
1141 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
1142 * @vm: Master vm structure.
1143 * @pdp: Page directory pointer for this address range.
1144 * @start: Starting virtual address to begin allocations.
1145 * @length: Size of the allocations.
1146 * @new_pds: Bitmap set by function with new allocations. Likely used by the
1147 * caller to free on error.
1149 * Allocate the required number of page directories starting at the pde index of
1150 * @start, and ending at the pde index @start + @length. This function will skip
1151 * over already allocated page directories within the range, and only allocate
1152 * new ones, setting the appropriate pointer within the pdp as well as the
1153 * correct position in the bitmap @new_pds.
1155 * The function will only allocate the pages within the range for a give page
1156 * directory pointer. In other words, if @start + @length straddles a virtually
1157 * addressed PDP boundary (512GB for 4k pages), there will be more allocations
1158 * required by the caller, This is not currently possible, and the BUG in the
1159 * code will prevent it.
1161 * Return: 0 if success; negative error code otherwise.
1164 gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1165 struct i915_page_directory_pointer *pdp,
1168 unsigned long *new_pds)
1170 struct drm_i915_private *dev_priv = vm->i915;
1171 struct i915_page_directory *pd;
1173 uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1175 WARN_ON(!bitmap_empty(new_pds, pdpes));
1177 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1178 if (test_bit(pdpe, pdp->used_pdpes))
1181 pd = alloc_pd(dev_priv);
1185 gen8_initialize_pd(vm, pd);
1186 pdp->page_directory[pdpe] = pd;
1187 __set_bit(pdpe, new_pds);
1188 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
1194 for_each_set_bit(pdpe, new_pds, pdpes)
1195 free_pd(dev_priv, pdp->page_directory[pdpe]);
1201 * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
1202 * @vm: Master vm structure.
1203 * @pml4: Page map level 4 for this address range.
1204 * @start: Starting virtual address to begin allocations.
1205 * @length: Size of the allocations.
1206 * @new_pdps: Bitmap set by function with new allocations. Likely used by the
1207 * caller to free on error.
1209 * Allocate the required number of page directory pointers. Extremely similar to
1210 * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
1211 * The main difference is here we are limited by the pml4 boundary (instead of
1212 * the page directory pointer).
1214 * Return: 0 if success; negative error code otherwise.
1217 gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1218 struct i915_pml4 *pml4,
1221 unsigned long *new_pdps)
1223 struct drm_i915_private *dev_priv = vm->i915;
1224 struct i915_page_directory_pointer *pdp;
1227 WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
1229 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1230 if (!test_bit(pml4e, pml4->used_pml4es)) {
1231 pdp = alloc_pdp(dev_priv);
1235 gen8_initialize_pdp(vm, pdp);
1236 pml4->pdps[pml4e] = pdp;
1237 __set_bit(pml4e, new_pdps);
1238 trace_i915_page_directory_pointer_entry_alloc(vm,
1248 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1249 free_pdp(dev_priv, pml4->pdps[pml4e]);
1255 free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
1261 /* Fills in the page directory bitmap, and the array of page tables bitmap. Both
1262 * of these are based on the number of PDPEs in the system.
1265 int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
1266 unsigned long **new_pts,
1272 pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
1276 pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
1287 free_gen8_temp_bitmaps(pds, pts);
1291 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1292 struct i915_page_directory_pointer *pdp,
1296 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1297 unsigned long *new_page_dirs, *new_page_tables;
1298 struct drm_i915_private *dev_priv = vm->i915;
1299 struct i915_page_directory *pd;
1300 const uint64_t orig_start = start;
1301 const uint64_t orig_length = length;
1303 uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1306 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1310 /* Do the allocations first so we can easily bail out */
1311 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
1314 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1318 /* For every page directory referenced, allocate page tables */
1319 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1320 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
1321 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
1327 length = orig_length;
1329 /* Allocations have completed successfully, so set the bitmaps, and do
1331 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1332 gen8_pde_t *const page_directory = kmap_px(pd);
1333 struct i915_page_table *pt;
1334 uint64_t pd_len = length;
1335 uint64_t pd_start = start;
1338 /* Every pd should be allocated, we just did that above. */
1341 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1342 /* Same reasoning as pd */
1345 WARN_ON(!gen8_pte_count(pd_start, pd_len));
1347 /* Set our used ptes within the page table */
1348 bitmap_set(pt->used_ptes,
1349 gen8_pte_index(pd_start),
1350 gen8_pte_count(pd_start, pd_len));
1352 /* Our pde is now pointing to the pagetable, pt */
1353 __set_bit(pde, pd->used_pdes);
1355 /* Map the PDE to the page table */
1356 page_directory[pde] = gen8_pde_encode(px_dma(pt),
1358 trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
1359 gen8_pte_index(start),
1360 gen8_pte_count(start, length),
1363 /* NB: We haven't yet mapped ptes to pages. At this
1364 * point we're still relying on insert_entries() */
1367 kunmap_px(ppgtt, page_directory);
1368 __set_bit(pdpe, pdp->used_pdpes);
1369 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);
1372 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1373 mark_tlbs_dirty(ppgtt);
1380 for_each_set_bit(temp, new_page_tables + pdpe *
1381 BITS_TO_LONGS(I915_PDES), I915_PDES)
1383 pdp->page_directory[pdpe]->page_table[temp]);
1386 for_each_set_bit(pdpe, new_page_dirs, pdpes)
1387 free_pd(dev_priv, pdp->page_directory[pdpe]);
1389 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1390 mark_tlbs_dirty(ppgtt);
1394 static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1395 struct i915_pml4 *pml4,
1399 DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
1400 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1401 struct i915_page_directory_pointer *pdp;
1405 /* Do the pml4 allocations first, so we don't need to track the newly
1406 * allocated tables below the pdp */
1407 bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
1409 /* The pagedirectory and pagetable allocations are done in the shared 3
1410 * and 4 level code. Just allocate the pdps.
1412 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
1417 WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2,
1418 "The allocation has spanned more than 512GB. "
1419 "It is highly likely this is incorrect.");
1421 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1424 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
1428 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);
1431 bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
1432 GEN8_PML4ES_PER_PML4);
1437 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1438 gen8_ppgtt_cleanup_3lvl(vm->i915, pml4->pdps[pml4e]);
1443 static int gen8_alloc_va_range(struct i915_address_space *vm,
1444 uint64_t start, uint64_t length)
1446 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1448 if (USES_FULL_48BIT_PPGTT(vm->i915))
1449 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1451 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
1454 static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
1455 uint64_t start, uint64_t length,
1456 gen8_pte_t scratch_pte,
1459 struct i915_page_directory *pd;
1462 gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1463 struct i915_page_table *pt;
1464 uint64_t pd_len = length;
1465 uint64_t pd_start = start;
1468 if (!test_bit(pdpe, pdp->used_pdpes))
1471 seq_printf(m, "\tPDPE #%d\n", pdpe);
1472 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1474 gen8_pte_t *pt_vaddr;
1476 if (!test_bit(pde, pd->used_pdes))
1479 pt_vaddr = kmap_px(pt);
1480 for (pte = 0; pte < GEN8_PTES; pte += 4) {
1482 (pdpe << GEN8_PDPE_SHIFT) |
1483 (pde << GEN8_PDE_SHIFT) |
1484 (pte << GEN8_PTE_SHIFT);
1488 for (i = 0; i < 4; i++)
1489 if (pt_vaddr[pte + i] != scratch_pte)
1494 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1495 for (i = 0; i < 4; i++) {
1496 if (pt_vaddr[pte + i] != scratch_pte)
1497 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1499 seq_puts(m, " SCRATCH ");
1503 /* don't use kunmap_px, it could trigger
1504 * an unnecessary flush.
1506 kunmap_atomic(pt_vaddr);
1511 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1513 struct i915_address_space *vm = &ppgtt->base;
1514 uint64_t start = ppgtt->base.start;
1515 uint64_t length = ppgtt->base.total;
1516 gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
1519 if (!USES_FULL_48BIT_PPGTT(vm->i915)) {
1520 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
1523 struct i915_pml4 *pml4 = &ppgtt->pml4;
1524 struct i915_page_directory_pointer *pdp;
1526 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1527 if (!test_bit(pml4e, pml4->used_pml4es))
1530 seq_printf(m, " PML4E #%llu\n", pml4e);
1531 gen8_dump_pdp(pdp, start, length, scratch_pte, m);
1536 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1538 unsigned long *new_page_dirs, *new_page_tables;
1539 uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev));
1542 /* We allocate temp bitmap for page tables for no gain
1543 * but as this is for init only, lets keep the things simple
1545 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1549 /* Allocate for all pdps regardless of how the ppgtt
1552 ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
1556 *ppgtt->pdp.used_pdpes = *new_page_dirs;
1558 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1564 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1565 * with a net effect resembling a 2-level page table in normal x86 terms. Each
1566 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1570 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1572 struct drm_i915_private *dev_priv = ppgtt->base.i915;
1575 ret = gen8_init_scratch(&ppgtt->base);
1579 ppgtt->base.start = 0;
1580 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1581 ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1582 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
1583 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1584 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1585 ppgtt->base.bind_vma = ppgtt_bind_vma;
1586 ppgtt->debug_dump = gen8_dump_ppgtt;
1588 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1589 ret = setup_px(dev_priv, &ppgtt->pml4);
1593 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1595 ppgtt->base.total = 1ULL << 48;
1596 ppgtt->switch_mm = gen8_48b_mm_switch;
1598 ret = __pdp_init(dev_priv, &ppgtt->pdp);
1602 ppgtt->base.total = 1ULL << 32;
1603 ppgtt->switch_mm = gen8_legacy_mm_switch;
1604 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
1608 if (intel_vgpu_active(dev_priv)) {
1609 ret = gen8_preallocate_top_level_pdps(ppgtt);
1615 if (intel_vgpu_active(dev_priv))
1616 gen8_ppgtt_notify_vgt(ppgtt, true);
1621 gen8_free_scratch(&ppgtt->base);
1625 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1627 struct i915_address_space *vm = &ppgtt->base;
1628 struct i915_page_table *unused;
1629 gen6_pte_t scratch_pte;
1632 uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
1634 scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1637 gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
1639 gen6_pte_t *pt_vaddr;
1640 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1641 pd_entry = readl(ppgtt->pd_addr + pde);
1642 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1644 if (pd_entry != expected)
1645 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1649 seq_printf(m, "\tPDE: %x\n", pd_entry);
1651 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1653 for (pte = 0; pte < GEN6_PTES; pte+=4) {
1655 (pde * PAGE_SIZE * GEN6_PTES) +
1659 for (i = 0; i < 4; i++)
1660 if (pt_vaddr[pte + i] != scratch_pte)
1665 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1666 for (i = 0; i < 4; i++) {
1667 if (pt_vaddr[pte + i] != scratch_pte)
1668 seq_printf(m, " %08x", pt_vaddr[pte + i]);
1670 seq_puts(m, " SCRATCH ");
1674 kunmap_px(ppgtt, pt_vaddr);
1678 /* Write pde (index) from the page directory @pd to the page table @pt */
1679 static void gen6_write_pde(struct i915_page_directory *pd,
1680 const int pde, struct i915_page_table *pt)
1682 /* Caller needs to make sure the write completes if necessary */
1683 struct i915_hw_ppgtt *ppgtt =
1684 container_of(pd, struct i915_hw_ppgtt, pd);
1687 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
1688 pd_entry |= GEN6_PDE_VALID;
1690 writel(pd_entry, ppgtt->pd_addr + pde);
1693 /* Write all the page tables found in the ppgtt structure to incrementing page
1695 static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1696 struct i915_page_directory *pd,
1697 uint32_t start, uint32_t length)
1699 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1700 struct i915_page_table *pt;
1703 gen6_for_each_pde(pt, pd, start, length, pde)
1704 gen6_write_pde(pd, pde, pt);
1706 /* Make sure write is complete before other code can use this page
1707 * table. Also require for WC mapped PTEs */
1711 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1713 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1715 return (ppgtt->pd.base.ggtt_offset / 64) << 16;
1718 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1719 struct drm_i915_gem_request *req)
1721 struct intel_ring *ring = req->ring;
1722 struct intel_engine_cs *engine = req->engine;
1725 /* NB: TLBs must be flushed and invalidated before a switch */
1726 ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1730 ret = intel_ring_begin(req, 6);
1734 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1735 intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
1736 intel_ring_emit(ring, PP_DIR_DCLV_2G);
1737 intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
1738 intel_ring_emit(ring, get_pd_offset(ppgtt));
1739 intel_ring_emit(ring, MI_NOOP);
1740 intel_ring_advance(ring);
1745 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1746 struct drm_i915_gem_request *req)
1748 struct intel_ring *ring = req->ring;
1749 struct intel_engine_cs *engine = req->engine;
1752 /* NB: TLBs must be flushed and invalidated before a switch */
1753 ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1757 ret = intel_ring_begin(req, 6);
1761 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1762 intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(engine));
1763 intel_ring_emit(ring, PP_DIR_DCLV_2G);
1764 intel_ring_emit_reg(ring, RING_PP_DIR_BASE(engine));
1765 intel_ring_emit(ring, get_pd_offset(ppgtt));
1766 intel_ring_emit(ring, MI_NOOP);
1767 intel_ring_advance(ring);
1769 /* XXX: RCS is the only one to auto invalidate the TLBs? */
1770 if (engine->id != RCS) {
1771 ret = engine->emit_flush(req, EMIT_INVALIDATE | EMIT_FLUSH);
1779 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1780 struct drm_i915_gem_request *req)
1782 struct intel_engine_cs *engine = req->engine;
1783 struct drm_i915_private *dev_priv = req->i915;
1785 I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1786 I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
1790 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1792 struct intel_engine_cs *engine;
1793 enum intel_engine_id id;
1795 for_each_engine(engine, dev_priv, id) {
1796 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1797 GEN8_GFX_PPGTT_48B : 0;
1798 I915_WRITE(RING_MODE_GEN7(engine),
1799 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1803 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1805 struct intel_engine_cs *engine;
1806 uint32_t ecochk, ecobits;
1807 enum intel_engine_id id;
1809 ecobits = I915_READ(GAC_ECO_BITS);
1810 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1812 ecochk = I915_READ(GAM_ECOCHK);
1813 if (IS_HASWELL(dev_priv)) {
1814 ecochk |= ECOCHK_PPGTT_WB_HSW;
1816 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1817 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1819 I915_WRITE(GAM_ECOCHK, ecochk);
1821 for_each_engine(engine, dev_priv, id) {
1822 /* GFX_MODE is per-ring on gen7+ */
1823 I915_WRITE(RING_MODE_GEN7(engine),
1824 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1828 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1830 uint32_t ecochk, gab_ctl, ecobits;
1832 ecobits = I915_READ(GAC_ECO_BITS);
1833 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1834 ECOBITS_PPGTT_CACHE64B);
1836 gab_ctl = I915_READ(GAB_CTL);
1837 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1839 ecochk = I915_READ(GAM_ECOCHK);
1840 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1842 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1845 /* PPGTT support for Sandybdrige/Gen6 and later */
1846 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1850 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1851 gen6_pte_t *pt_vaddr, scratch_pte;
1852 unsigned first_entry = start >> PAGE_SHIFT;
1853 unsigned num_entries = length >> PAGE_SHIFT;
1854 unsigned act_pt = first_entry / GEN6_PTES;
1855 unsigned first_pte = first_entry % GEN6_PTES;
1856 unsigned last_pte, i;
1858 scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
1861 while (num_entries) {
1862 last_pte = first_pte + num_entries;
1863 if (last_pte > GEN6_PTES)
1864 last_pte = GEN6_PTES;
1866 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1868 for (i = first_pte; i < last_pte; i++)
1869 pt_vaddr[i] = scratch_pte;
1871 kunmap_px(ppgtt, pt_vaddr);
1873 num_entries -= last_pte - first_pte;
1879 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1880 struct sg_table *pages,
1882 enum i915_cache_level cache_level, u32 flags)
1884 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1885 unsigned first_entry = start >> PAGE_SHIFT;
1886 unsigned act_pt = first_entry / GEN6_PTES;
1887 unsigned act_pte = first_entry % GEN6_PTES;
1888 gen6_pte_t *pt_vaddr = NULL;
1889 struct sgt_iter sgt_iter;
1892 for_each_sgt_dma(addr, sgt_iter, pages) {
1893 if (pt_vaddr == NULL)
1894 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1897 vm->pte_encode(addr, cache_level, flags);
1899 if (++act_pte == GEN6_PTES) {
1900 kunmap_px(ppgtt, pt_vaddr);
1908 kunmap_px(ppgtt, pt_vaddr);
1911 static int gen6_alloc_va_range(struct i915_address_space *vm,
1912 uint64_t start_in, uint64_t length_in)
1914 DECLARE_BITMAP(new_page_tables, I915_PDES);
1915 struct drm_i915_private *dev_priv = vm->i915;
1916 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1917 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1918 struct i915_page_table *pt;
1919 uint32_t start, length, start_save, length_save;
1923 start = start_save = start_in;
1924 length = length_save = length_in;
1926 bitmap_zero(new_page_tables, I915_PDES);
1928 /* The allocation is done in two stages so that we can bail out with
1929 * minimal amount of pain. The first stage finds new page tables that
1930 * need allocation. The second stage marks use ptes within the page
1933 gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1934 if (pt != vm->scratch_pt) {
1935 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1939 /* We've already allocated a page table */
1940 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1942 pt = alloc_pt(dev_priv);
1948 gen6_initialize_pt(vm, pt);
1950 ppgtt->pd.page_table[pde] = pt;
1951 __set_bit(pde, new_page_tables);
1952 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
1956 length = length_save;
1958 gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
1959 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1961 bitmap_zero(tmp_bitmap, GEN6_PTES);
1962 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1963 gen6_pte_count(start, length));
1965 if (__test_and_clear_bit(pde, new_page_tables))
1966 gen6_write_pde(&ppgtt->pd, pde, pt);
1968 trace_i915_page_table_entry_map(vm, pde, pt,
1969 gen6_pte_index(start),
1970 gen6_pte_count(start, length),
1972 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
1976 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1978 /* Make sure write is complete before other code can use this page
1979 * table. Also require for WC mapped PTEs */
1982 mark_tlbs_dirty(ppgtt);
1986 for_each_set_bit(pde, new_page_tables, I915_PDES) {
1987 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
1989 ppgtt->pd.page_table[pde] = vm->scratch_pt;
1990 free_pt(dev_priv, pt);
1993 mark_tlbs_dirty(ppgtt);
1997 static int gen6_init_scratch(struct i915_address_space *vm)
1999 struct drm_i915_private *dev_priv = vm->i915;
2002 ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
2006 vm->scratch_pt = alloc_pt(dev_priv);
2007 if (IS_ERR(vm->scratch_pt)) {
2008 cleanup_scratch_page(dev_priv, &vm->scratch_page);
2009 return PTR_ERR(vm->scratch_pt);
2012 gen6_initialize_pt(vm, vm->scratch_pt);
2017 static void gen6_free_scratch(struct i915_address_space *vm)
2019 struct drm_i915_private *dev_priv = vm->i915;
2021 free_pt(dev_priv, vm->scratch_pt);
2022 cleanup_scratch_page(dev_priv, &vm->scratch_page);
2025 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2027 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
2028 struct i915_page_directory *pd = &ppgtt->pd;
2029 struct drm_i915_private *dev_priv = vm->i915;
2030 struct i915_page_table *pt;
2033 drm_mm_remove_node(&ppgtt->node);
2035 gen6_for_all_pdes(pt, pd, pde)
2036 if (pt != vm->scratch_pt)
2037 free_pt(dev_priv, pt);
2039 gen6_free_scratch(vm);
2042 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
2044 struct i915_address_space *vm = &ppgtt->base;
2045 struct drm_i915_private *dev_priv = ppgtt->base.i915;
2046 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2047 bool retried = false;
2050 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2051 * allocator works in address space sizes, so it's multiplied by page
2052 * size. We allocate at the top of the GTT to avoid fragmentation.
2054 BUG_ON(!drm_mm_initialized(&ggtt->base.mm));
2056 ret = gen6_init_scratch(vm);
2061 ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm, &ppgtt->node,
2062 GEN6_PD_SIZE, GEN6_PD_ALIGN,
2063 I915_COLOR_UNEVICTABLE,
2064 0, ggtt->base.total,
2066 if (ret == -ENOSPC && !retried) {
2067 ret = i915_gem_evict_something(&ggtt->base,
2068 GEN6_PD_SIZE, GEN6_PD_ALIGN,
2069 I915_COLOR_UNEVICTABLE,
2070 0, ggtt->base.total,
2083 if (ppgtt->node.start < ggtt->mappable_end)
2084 DRM_DEBUG("Forced to use aperture for PDEs\n");
2089 gen6_free_scratch(vm);
2093 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2095 return gen6_ppgtt_allocate_page_directories(ppgtt);
2098 static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2099 uint64_t start, uint64_t length)
2101 struct i915_page_table *unused;
2104 gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
2105 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
2108 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2110 struct drm_i915_private *dev_priv = ppgtt->base.i915;
2111 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2114 ppgtt->base.pte_encode = ggtt->base.pte_encode;
2115 if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
2116 ppgtt->switch_mm = gen6_mm_switch;
2117 else if (IS_HASWELL(dev_priv))
2118 ppgtt->switch_mm = hsw_mm_switch;
2119 else if (IS_GEN7(dev_priv))
2120 ppgtt->switch_mm = gen7_mm_switch;
2124 ret = gen6_ppgtt_alloc(ppgtt);
2128 ppgtt->base.allocate_va_range = gen6_alloc_va_range;
2129 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2130 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
2131 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2132 ppgtt->base.bind_vma = ppgtt_bind_vma;
2133 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
2134 ppgtt->base.start = 0;
2135 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2136 ppgtt->debug_dump = gen6_dump_ppgtt;
2138 ppgtt->pd.base.ggtt_offset =
2139 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
2141 ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm +
2142 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
2144 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
2146 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
2148 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
2149 ppgtt->node.size >> 20,
2150 ppgtt->node.start / PAGE_SIZE);
2152 DRM_DEBUG("Adding PPGTT at offset %x\n",
2153 ppgtt->pd.base.ggtt_offset << 10);
2158 static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2159 struct drm_i915_private *dev_priv)
2161 ppgtt->base.i915 = dev_priv;
2163 if (INTEL_INFO(dev_priv)->gen < 8)
2164 return gen6_ppgtt_init(ppgtt);
2166 return gen8_ppgtt_init(ppgtt);
2169 static void i915_address_space_init(struct i915_address_space *vm,
2170 struct drm_i915_private *dev_priv,
2173 i915_gem_timeline_init(dev_priv, &vm->timeline, name);
2174 drm_mm_init(&vm->mm, vm->start, vm->total);
2175 INIT_LIST_HEAD(&vm->active_list);
2176 INIT_LIST_HEAD(&vm->inactive_list);
2177 INIT_LIST_HEAD(&vm->unbound_list);
2178 list_add_tail(&vm->global_link, &dev_priv->vm_list);
2181 static void i915_address_space_fini(struct i915_address_space *vm)
2183 i915_gem_timeline_fini(&vm->timeline);
2184 drm_mm_takedown(&vm->mm);
2185 list_del(&vm->global_link);
2188 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2190 /* This function is for gtt related workarounds. This function is
2191 * called on driver load and after a GPU reset, so you can place
2192 * workarounds here even if they get overwritten by GPU reset.
2194 /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt */
2195 if (IS_BROADWELL(dev_priv))
2196 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2197 else if (IS_CHERRYVIEW(dev_priv))
2198 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2199 else if (IS_SKYLAKE(dev_priv))
2200 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2201 else if (IS_BROXTON(dev_priv))
2202 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2205 static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2206 struct drm_i915_private *dev_priv,
2207 struct drm_i915_file_private *file_priv,
2212 ret = __hw_ppgtt_init(ppgtt, dev_priv);
2214 kref_init(&ppgtt->ref);
2215 i915_address_space_init(&ppgtt->base, dev_priv, name);
2216 ppgtt->base.file = file_priv;
2222 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2224 gtt_write_workarounds(dev_priv);
2226 /* In the case of execlists, PPGTT is enabled by the context descriptor
2227 * and the PDPs are contained within the context itself. We don't
2228 * need to do anything here. */
2229 if (i915.enable_execlists)
2232 if (!USES_PPGTT(dev_priv))
2235 if (IS_GEN6(dev_priv))
2236 gen6_ppgtt_enable(dev_priv);
2237 else if (IS_GEN7(dev_priv))
2238 gen7_ppgtt_enable(dev_priv);
2239 else if (INTEL_GEN(dev_priv) >= 8)
2240 gen8_ppgtt_enable(dev_priv);
2242 MISSING_CASE(INTEL_GEN(dev_priv));
2247 struct i915_hw_ppgtt *
2248 i915_ppgtt_create(struct drm_i915_private *dev_priv,
2249 struct drm_i915_file_private *fpriv,
2252 struct i915_hw_ppgtt *ppgtt;
2255 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2257 return ERR_PTR(-ENOMEM);
2259 ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name);
2262 return ERR_PTR(ret);
2265 trace_i915_ppgtt_create(&ppgtt->base);
2270 void i915_ppgtt_release(struct kref *kref)
2272 struct i915_hw_ppgtt *ppgtt =
2273 container_of(kref, struct i915_hw_ppgtt, ref);
2275 trace_i915_ppgtt_release(&ppgtt->base);
2277 /* vmas should already be unbound and destroyed */
2278 WARN_ON(!list_empty(&ppgtt->base.active_list));
2279 WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2280 WARN_ON(!list_empty(&ppgtt->base.unbound_list));
2282 i915_address_space_fini(&ppgtt->base);
2284 ppgtt->base.cleanup(&ppgtt->base);
2288 /* Certain Gen5 chipsets require require idling the GPU before
2289 * unmapping anything from the GTT when VT-d is enabled.
2291 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2293 #ifdef CONFIG_INTEL_IOMMU
2294 /* Query intel_iommu to see if we need the workaround. Presumably that
2297 if (IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_iommu_gfx_mapped)
2303 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2305 struct intel_engine_cs *engine;
2306 enum intel_engine_id id;
2308 if (INTEL_INFO(dev_priv)->gen < 6)
2311 for_each_engine(engine, dev_priv, id) {
2313 fault_reg = I915_READ(RING_FAULT_REG(engine));
2314 if (fault_reg & RING_FAULT_VALID) {
2315 DRM_DEBUG_DRIVER("Unexpected fault\n"
2317 "\tAddress space: %s\n"
2320 fault_reg & PAGE_MASK,
2321 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2322 RING_FAULT_SRCID(fault_reg),
2323 RING_FAULT_FAULT_TYPE(fault_reg));
2324 I915_WRITE(RING_FAULT_REG(engine),
2325 fault_reg & ~RING_FAULT_VALID);
2329 /* Engine specific init may not have been done till this point. */
2330 if (dev_priv->engine[RCS])
2331 POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2334 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
2336 if (INTEL_INFO(dev_priv)->gen < 6) {
2337 intel_gtt_chipset_flush();
2339 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2340 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2344 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2346 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2348 /* Don't bother messing with faults pre GEN6 as we have little
2349 * documentation supporting that it's a good idea.
2351 if (INTEL_GEN(dev_priv) < 6)
2354 i915_check_and_clear_faults(dev_priv);
2356 ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
2358 i915_ggtt_flush(dev_priv);
2361 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2362 struct sg_table *pages)
2364 if (dma_map_sg(&obj->base.dev->pdev->dev,
2365 pages->sgl, pages->nents,
2366 PCI_DMA_BIDIRECTIONAL))
2372 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2377 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2380 enum i915_cache_level level,
2383 struct drm_i915_private *dev_priv = vm->i915;
2384 gen8_pte_t __iomem *pte =
2385 (gen8_pte_t __iomem *)dev_priv->ggtt.gsm +
2386 (offset >> PAGE_SHIFT);
2388 gen8_set_pte(pte, gen8_pte_encode(addr, level));
2390 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2391 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2394 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2395 struct sg_table *st,
2397 enum i915_cache_level level, u32 unused)
2399 struct drm_i915_private *dev_priv = vm->i915;
2400 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2401 struct sgt_iter sgt_iter;
2402 gen8_pte_t __iomem *gtt_entries;
2403 gen8_pte_t gtt_entry;
2407 gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
2409 for_each_sgt_dma(addr, sgt_iter, st) {
2410 gtt_entry = gen8_pte_encode(addr, level);
2411 gen8_set_pte(>t_entries[i++], gtt_entry);
2415 * XXX: This serves as a posting read to make sure that the PTE has
2416 * actually been updated. There is some concern that even though
2417 * registers and PTEs are within the same BAR that they are potentially
2418 * of NUMA access patterns. Therefore, even with the way we assume
2419 * hardware should work, we must keep this posting read for paranoia.
2422 WARN_ON(readq(>t_entries[i-1]) != gtt_entry);
2424 /* This next bit makes the above posting read even more important. We
2425 * want to flush the TLBs only after we're certain all the PTE updates
2428 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2429 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2432 struct insert_entries {
2433 struct i915_address_space *vm;
2434 struct sg_table *st;
2436 enum i915_cache_level level;
2440 static int gen8_ggtt_insert_entries__cb(void *_arg)
2442 struct insert_entries *arg = _arg;
2443 gen8_ggtt_insert_entries(arg->vm, arg->st,
2444 arg->start, arg->level, arg->flags);
2448 static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2449 struct sg_table *st,
2451 enum i915_cache_level level,
2454 struct insert_entries arg = { vm, st, start, level, flags };
2455 stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL);
2458 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2461 enum i915_cache_level level,
2464 struct drm_i915_private *dev_priv = vm->i915;
2465 gen6_pte_t __iomem *pte =
2466 (gen6_pte_t __iomem *)dev_priv->ggtt.gsm +
2467 (offset >> PAGE_SHIFT);
2469 iowrite32(vm->pte_encode(addr, level, flags), pte);
2471 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2472 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2476 * Binds an object into the global gtt with the specified cache level. The object
2477 * will be accessible to the GPU via commands whose operands reference offsets
2478 * within the global GTT as well as accessible by the GPU through the GMADR
2479 * mapped BAR (dev_priv->mm.gtt->gtt).
2481 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2482 struct sg_table *st,
2484 enum i915_cache_level level, u32 flags)
2486 struct drm_i915_private *dev_priv = vm->i915;
2487 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2488 struct sgt_iter sgt_iter;
2489 gen6_pte_t __iomem *gtt_entries;
2490 gen6_pte_t gtt_entry;
2494 gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
2496 for_each_sgt_dma(addr, sgt_iter, st) {
2497 gtt_entry = vm->pte_encode(addr, level, flags);
2498 iowrite32(gtt_entry, >t_entries[i++]);
2501 /* XXX: This serves as a posting read to make sure that the PTE has
2502 * actually been updated. There is some concern that even though
2503 * registers and PTEs are within the same BAR that they are potentially
2504 * of NUMA access patterns. Therefore, even with the way we assume
2505 * hardware should work, we must keep this posting read for paranoia.
2508 WARN_ON(readl(>t_entries[i-1]) != gtt_entry);
2510 /* This next bit makes the above posting read even more important. We
2511 * want to flush the TLBs only after we're certain all the PTE updates
2514 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2515 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2518 static void nop_clear_range(struct i915_address_space *vm,
2519 uint64_t start, uint64_t length)
2523 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2524 uint64_t start, uint64_t length)
2526 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2527 unsigned first_entry = start >> PAGE_SHIFT;
2528 unsigned num_entries = length >> PAGE_SHIFT;
2529 gen8_pte_t scratch_pte, __iomem *gtt_base =
2530 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2531 const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2534 if (WARN(num_entries > max_entries,
2535 "First entry = %d; Num entries = %d (max=%d)\n",
2536 first_entry, num_entries, max_entries))
2537 num_entries = max_entries;
2539 scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
2541 for (i = 0; i < num_entries; i++)
2542 gen8_set_pte(>t_base[i], scratch_pte);
2546 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2550 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2551 unsigned first_entry = start >> PAGE_SHIFT;
2552 unsigned num_entries = length >> PAGE_SHIFT;
2553 gen6_pte_t scratch_pte, __iomem *gtt_base =
2554 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2555 const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2558 if (WARN(num_entries > max_entries,
2559 "First entry = %d; Num entries = %d (max=%d)\n",
2560 first_entry, num_entries, max_entries))
2561 num_entries = max_entries;
2563 scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2566 for (i = 0; i < num_entries; i++)
2567 iowrite32(scratch_pte, >t_base[i]);
2571 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2574 enum i915_cache_level cache_level,
2577 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2578 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2580 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2583 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2584 struct sg_table *pages,
2586 enum i915_cache_level cache_level, u32 unused)
2588 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2589 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2591 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2595 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2599 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2602 static int ggtt_bind_vma(struct i915_vma *vma,
2603 enum i915_cache_level cache_level,
2606 struct drm_i915_private *i915 = vma->vm->i915;
2607 struct drm_i915_gem_object *obj = vma->obj;
2611 ret = i915_get_ggtt_vma_pages(vma);
2615 /* Currently applicable only to VLV */
2617 pte_flags |= PTE_READ_ONLY;
2619 intel_runtime_pm_get(i915);
2620 vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
2621 cache_level, pte_flags);
2622 intel_runtime_pm_put(i915);
2625 * Without aliasing PPGTT there's no difference between
2626 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2627 * upgrade to both bound if we bind either to avoid double-binding.
2629 vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2634 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2635 enum i915_cache_level cache_level,
2638 struct drm_i915_private *i915 = vma->vm->i915;
2642 ret = i915_get_ggtt_vma_pages(vma);
2646 /* Currently applicable only to VLV */
2648 if (vma->obj->gt_ro)
2649 pte_flags |= PTE_READ_ONLY;
2652 if (flags & I915_VMA_GLOBAL_BIND) {
2653 intel_runtime_pm_get(i915);
2654 vma->vm->insert_entries(vma->vm,
2655 vma->pages, vma->node.start,
2656 cache_level, pte_flags);
2657 intel_runtime_pm_put(i915);
2660 if (flags & I915_VMA_LOCAL_BIND) {
2661 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2662 appgtt->base.insert_entries(&appgtt->base,
2663 vma->pages, vma->node.start,
2664 cache_level, pte_flags);
2670 static void ggtt_unbind_vma(struct i915_vma *vma)
2672 struct drm_i915_private *i915 = vma->vm->i915;
2673 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2674 const u64 size = min(vma->size, vma->node.size);
2676 if (vma->flags & I915_VMA_GLOBAL_BIND) {
2677 intel_runtime_pm_get(i915);
2678 vma->vm->clear_range(vma->vm,
2679 vma->node.start, size);
2680 intel_runtime_pm_put(i915);
2683 if (vma->flags & I915_VMA_LOCAL_BIND && appgtt)
2684 appgtt->base.clear_range(&appgtt->base,
2685 vma->node.start, size);
2688 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2689 struct sg_table *pages)
2691 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2692 struct device *kdev = &dev_priv->drm.pdev->dev;
2693 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2695 if (unlikely(ggtt->do_idle_maps)) {
2696 if (i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED)) {
2697 DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2698 /* Wait a bit, in hopes it avoids the hang */
2703 dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2706 static void i915_gtt_color_adjust(struct drm_mm_node *node,
2707 unsigned long color,
2711 if (node->color != color)
2714 node = list_first_entry_or_null(&node->node_list,
2717 if (node && node->allocated && node->color != color)
2721 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2723 /* Let GEM Manage all of the aperture.
2725 * However, leave one page at the end still bound to the scratch page.
2726 * There are a number of places where the hardware apparently prefetches
2727 * past the end of the object, and we've seen multiple hangs with the
2728 * GPU head pointer stuck in a batchbuffer bound at the last page of the
2729 * aperture. One page should be enough to keep any prefetching inside
2732 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2733 unsigned long hole_start, hole_end;
2734 struct i915_hw_ppgtt *ppgtt;
2735 struct drm_mm_node *entry;
2738 ret = intel_vgt_balloon(dev_priv);
2742 /* Reserve a mappable slot for our lockless error capture */
2743 ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm,
2744 &ggtt->error_capture,
2746 I915_COLOR_UNEVICTABLE,
2747 0, ggtt->mappable_end,
2752 /* Clear any non-preallocated blocks */
2753 drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
2754 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2755 hole_start, hole_end);
2756 ggtt->base.clear_range(&ggtt->base, hole_start,
2757 hole_end - hole_start);
2760 /* And finally clear the reserved guard page */
2761 ggtt->base.clear_range(&ggtt->base,
2762 ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
2764 if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2765 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2771 ret = __hw_ppgtt_init(ppgtt, dev_priv);
2775 if (ppgtt->base.allocate_va_range) {
2776 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
2779 goto err_ppgtt_cleanup;
2782 ppgtt->base.clear_range(&ppgtt->base,
2786 dev_priv->mm.aliasing_ppgtt = ppgtt;
2787 WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
2788 ggtt->base.bind_vma = aliasing_gtt_bind_vma;
2794 ppgtt->base.cleanup(&ppgtt->base);
2798 drm_mm_remove_node(&ggtt->error_capture);
2803 * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2804 * @dev_priv: i915 device
2806 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2808 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2810 if (dev_priv->mm.aliasing_ppgtt) {
2811 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2812 ppgtt->base.cleanup(&ppgtt->base);
2816 i915_gem_cleanup_stolen(&dev_priv->drm);
2818 if (drm_mm_node_allocated(&ggtt->error_capture))
2819 drm_mm_remove_node(&ggtt->error_capture);
2821 if (drm_mm_initialized(&ggtt->base.mm)) {
2822 intel_vgt_deballoon(dev_priv);
2824 mutex_lock(&dev_priv->drm.struct_mutex);
2825 i915_address_space_fini(&ggtt->base);
2826 mutex_unlock(&dev_priv->drm.struct_mutex);
2829 ggtt->base.cleanup(&ggtt->base);
2831 arch_phys_wc_del(ggtt->mtrr);
2832 io_mapping_fini(&ggtt->mappable);
2835 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2837 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2838 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2839 return snb_gmch_ctl << 20;
2842 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2844 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2845 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2847 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2849 #ifdef CONFIG_X86_32
2850 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2851 if (bdw_gmch_ctl > 4)
2855 return bdw_gmch_ctl << 20;
2858 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2860 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2861 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2864 return 1 << (20 + gmch_ctrl);
2869 static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2871 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2872 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2873 return snb_gmch_ctl << 25; /* 32 MB units */
2876 static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2878 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2879 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2880 return bdw_gmch_ctl << 25; /* 32 MB units */
2883 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2885 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2886 gmch_ctrl &= SNB_GMCH_GMS_MASK;
2889 * 0x0 to 0x10: 32MB increments starting at 0MB
2890 * 0x11 to 0x16: 4MB increments starting at 8MB
2891 * 0x17 to 0x1d: 4MB increments start at 36MB
2893 if (gmch_ctrl < 0x11)
2894 return gmch_ctrl << 25;
2895 else if (gmch_ctrl < 0x17)
2896 return (gmch_ctrl - 0x11 + 2) << 22;
2898 return (gmch_ctrl - 0x17 + 9) << 22;
2901 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2903 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2904 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2906 if (gen9_gmch_ctl < 0xf0)
2907 return gen9_gmch_ctl << 25; /* 32 MB units */
2909 /* 4MB increments starting at 0xf0 for 4MB */
2910 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2913 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2915 struct drm_i915_private *dev_priv = ggtt->base.i915;
2916 struct pci_dev *pdev = dev_priv->drm.pdev;
2917 phys_addr_t phys_addr;
2920 /* For Modern GENs the PTEs and register space are split in the BAR */
2921 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
2924 * On BXT writes larger than 64 bit to the GTT pagetable range will be
2925 * dropped. For WC mappings in general we have 64 byte burst writes
2926 * when the WC buffer is flushed, so we can't use it, but have to
2927 * resort to an uncached mapping. The WC issue is easily caught by the
2928 * readback check when writing GTT PTE entries.
2930 if (IS_GEN9_LP(dev_priv))
2931 ggtt->gsm = ioremap_nocache(phys_addr, size);
2933 ggtt->gsm = ioremap_wc(phys_addr, size);
2935 DRM_ERROR("Failed to map the ggtt page table\n");
2939 ret = setup_scratch_page(dev_priv, &ggtt->base.scratch_page, GFP_DMA32);
2941 DRM_ERROR("Scratch setup failed\n");
2942 /* iounmap will also get called at remove, but meh */
2950 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2951 * bits. When using advanced contexts each context stores its own PAT, but
2952 * writing this data shouldn't be harmful even in those cases. */
2953 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2957 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
2958 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2959 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2960 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
2961 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2962 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2963 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2964 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2966 if (!USES_PPGTT(dev_priv))
2967 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2968 * so RTL will always use the value corresponding to
2970 * So let's disable cache for GGTT to avoid screen corruptions.
2971 * MOCS still can be used though.
2972 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2973 * before this patch, i.e. the same uncached + snooping access
2974 * like on gen6/7 seems to be in effect.
2975 * - So this just fixes blitter/render access. Again it looks
2976 * like it's not just uncached access, but uncached + snooping.
2977 * So we can still hold onto all our assumptions wrt cpu
2978 * clflushing on LLC machines.
2980 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2982 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2983 * write would work. */
2984 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2985 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
2988 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2993 * Map WB on BDW to snooped on CHV.
2995 * Only the snoop bit has meaning for CHV, the rest is
2998 * The hardware will never snoop for certain types of accesses:
2999 * - CPU GTT (GMADR->GGTT->no snoop->memory)
3000 * - PPGTT page tables
3001 * - some other special cycles
3003 * As with BDW, we also need to consider the following for GT accesses:
3004 * "For GGTT, there is NO pat_sel[2:0] from the entry,
3005 * so RTL will always use the value corresponding to
3007 * Which means we must set the snoop bit in PAT entry 0
3008 * in order to keep the global status page working.
3010 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
3014 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
3015 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
3016 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
3017 GEN8_PPAT(7, CHV_PPAT_SNOOP);
3019 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
3020 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
3023 static void gen6_gmch_remove(struct i915_address_space *vm)
3025 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3028 cleanup_scratch_page(vm->i915, &vm->scratch_page);
3031 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
3033 struct drm_i915_private *dev_priv = ggtt->base.i915;
3034 struct pci_dev *pdev = dev_priv->drm.pdev;
3038 /* TODO: We're not aware of mappable constraints on gen8 yet */
3039 ggtt->mappable_base = pci_resource_start(pdev, 2);
3040 ggtt->mappable_end = pci_resource_len(pdev, 2);
3042 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(39)))
3043 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
3045 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3047 if (INTEL_GEN(dev_priv) >= 9) {
3048 ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
3049 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3050 } else if (IS_CHERRYVIEW(dev_priv)) {
3051 ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
3052 size = chv_get_total_gtt_size(snb_gmch_ctl);
3054 ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
3055 size = gen8_get_total_gtt_size(snb_gmch_ctl);
3058 ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
3060 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3061 chv_setup_private_ppat(dev_priv);
3063 bdw_setup_private_ppat(dev_priv);
3065 ggtt->base.cleanup = gen6_gmch_remove;
3066 ggtt->base.bind_vma = ggtt_bind_vma;
3067 ggtt->base.unbind_vma = ggtt_unbind_vma;
3068 ggtt->base.insert_page = gen8_ggtt_insert_page;
3069 ggtt->base.clear_range = nop_clear_range;
3070 if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
3071 ggtt->base.clear_range = gen8_ggtt_clear_range;
3073 ggtt->base.insert_entries = gen8_ggtt_insert_entries;
3074 if (IS_CHERRYVIEW(dev_priv))
3075 ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
3077 return ggtt_probe_common(ggtt, size);
3080 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3082 struct drm_i915_private *dev_priv = ggtt->base.i915;
3083 struct pci_dev *pdev = dev_priv->drm.pdev;
3087 ggtt->mappable_base = pci_resource_start(pdev, 2);
3088 ggtt->mappable_end = pci_resource_len(pdev, 2);
3090 /* 64/512MB is the current min/max we actually know of, but this is just
3091 * a coarse sanity check.
3093 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3094 DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
3098 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(40)))
3099 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
3100 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3102 ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
3104 size = gen6_get_total_gtt_size(snb_gmch_ctl);
3105 ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
3107 ggtt->base.clear_range = gen6_ggtt_clear_range;
3108 ggtt->base.insert_page = gen6_ggtt_insert_page;
3109 ggtt->base.insert_entries = gen6_ggtt_insert_entries;
3110 ggtt->base.bind_vma = ggtt_bind_vma;
3111 ggtt->base.unbind_vma = ggtt_unbind_vma;
3112 ggtt->base.cleanup = gen6_gmch_remove;
3114 if (HAS_EDRAM(dev_priv))
3115 ggtt->base.pte_encode = iris_pte_encode;
3116 else if (IS_HASWELL(dev_priv))
3117 ggtt->base.pte_encode = hsw_pte_encode;
3118 else if (IS_VALLEYVIEW(dev_priv))
3119 ggtt->base.pte_encode = byt_pte_encode;
3120 else if (INTEL_GEN(dev_priv) >= 7)
3121 ggtt->base.pte_encode = ivb_pte_encode;
3123 ggtt->base.pte_encode = snb_pte_encode;
3125 return ggtt_probe_common(ggtt, size);
3128 static void i915_gmch_remove(struct i915_address_space *vm)
3130 intel_gmch_remove();
3133 static int i915_gmch_probe(struct i915_ggtt *ggtt)
3135 struct drm_i915_private *dev_priv = ggtt->base.i915;
3138 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3140 DRM_ERROR("failed to set up gmch\n");
3144 intel_gtt_get(&ggtt->base.total, &ggtt->stolen_size,
3145 &ggtt->mappable_base, &ggtt->mappable_end);
3147 ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3148 ggtt->base.insert_page = i915_ggtt_insert_page;
3149 ggtt->base.insert_entries = i915_ggtt_insert_entries;
3150 ggtt->base.clear_range = i915_ggtt_clear_range;
3151 ggtt->base.bind_vma = ggtt_bind_vma;
3152 ggtt->base.unbind_vma = ggtt_unbind_vma;
3153 ggtt->base.cleanup = i915_gmch_remove;
3155 if (unlikely(ggtt->do_idle_maps))
3156 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3162 * i915_ggtt_probe_hw - Probe GGTT hardware location
3163 * @dev_priv: i915 device
3165 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3167 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3170 ggtt->base.i915 = dev_priv;
3172 if (INTEL_GEN(dev_priv) <= 5)
3173 ret = i915_gmch_probe(ggtt);
3174 else if (INTEL_GEN(dev_priv) < 8)
3175 ret = gen6_gmch_probe(ggtt);
3177 ret = gen8_gmch_probe(ggtt);
3181 if ((ggtt->base.total - 1) >> 32) {
3182 DRM_ERROR("We never expected a Global GTT with more than 32bits"
3183 " of address space! Found %lldM!\n",
3184 ggtt->base.total >> 20);
3185 ggtt->base.total = 1ULL << 32;
3186 ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
3189 if (ggtt->mappable_end > ggtt->base.total) {
3190 DRM_ERROR("mappable aperture extends past end of GGTT,"
3191 " aperture=%llx, total=%llx\n",
3192 ggtt->mappable_end, ggtt->base.total);
3193 ggtt->mappable_end = ggtt->base.total;
3196 /* GMADR is the PCI mmio aperture into the global GTT. */
3197 DRM_INFO("Memory usable by graphics device = %lluM\n",
3198 ggtt->base.total >> 20);
3199 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
3200 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", ggtt->stolen_size >> 20);
3201 #ifdef CONFIG_INTEL_IOMMU
3202 if (intel_iommu_gfx_mapped)
3203 DRM_INFO("VT-d active for gfx access\n");
3210 * i915_ggtt_init_hw - Initialize GGTT hardware
3211 * @dev_priv: i915 device
3213 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3215 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3218 INIT_LIST_HEAD(&dev_priv->vm_list);
3220 /* Subtract the guard page before address space initialization to
3221 * shrink the range used by drm_mm.
3223 mutex_lock(&dev_priv->drm.struct_mutex);
3224 ggtt->base.total -= PAGE_SIZE;
3225 i915_address_space_init(&ggtt->base, dev_priv, "[global]");
3226 ggtt->base.total += PAGE_SIZE;
3227 if (!HAS_LLC(dev_priv))
3228 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
3229 mutex_unlock(&dev_priv->drm.struct_mutex);
3231 if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
3232 dev_priv->ggtt.mappable_base,
3233 dev_priv->ggtt.mappable_end)) {
3235 goto out_gtt_cleanup;
3238 ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
3241 * Initialise stolen early so that we may reserve preallocated
3242 * objects for the BIOS to KMS transition.
3244 ret = i915_gem_init_stolen(dev_priv);
3246 goto out_gtt_cleanup;
3251 ggtt->base.cleanup(&ggtt->base);
3255 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3257 if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3263 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3265 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3266 struct drm_i915_gem_object *obj, *on;
3268 i915_check_and_clear_faults(dev_priv);
3270 /* First fill our portion of the GTT with scratch pages */
3271 ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
3273 ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
3275 /* clflush objects bound into the GGTT and rebind them. */
3276 list_for_each_entry_safe(obj, on,
3277 &dev_priv->mm.bound_list, global_link) {
3278 bool ggtt_bound = false;
3279 struct i915_vma *vma;
3281 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3282 if (vma->vm != &ggtt->base)
3285 if (!i915_vma_unbind(vma))
3288 WARN_ON(i915_vma_bind(vma, obj->cache_level,
3294 WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3297 ggtt->base.closed = false;
3299 if (INTEL_GEN(dev_priv) >= 8) {
3300 if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3301 chv_setup_private_ppat(dev_priv);
3303 bdw_setup_private_ppat(dev_priv);
3308 if (USES_PPGTT(dev_priv)) {
3309 struct i915_address_space *vm;
3311 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3312 /* TODO: Perhaps it shouldn't be gen6 specific */
3314 struct i915_hw_ppgtt *ppgtt;
3316 if (i915_is_ggtt(vm))
3317 ppgtt = dev_priv->mm.aliasing_ppgtt;
3319 ppgtt = i915_vm_to_ppgtt(vm);
3321 gen6_write_page_range(dev_priv, &ppgtt->pd,
3322 0, ppgtt->base.total);
3326 i915_ggtt_flush(dev_priv);
3330 i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
3331 struct i915_address_space *vm,
3332 const struct i915_ggtt_view *view)
3336 rb = obj->vma_tree.rb_node;
3338 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
3341 cmp = i915_vma_compare(vma, vm, view);
3355 i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
3356 struct i915_address_space *vm,
3357 const struct i915_ggtt_view *view)
3359 struct i915_vma *vma;
3361 lockdep_assert_held(&obj->base.dev->struct_mutex);
3362 GEM_BUG_ON(view && !i915_is_ggtt(vm));
3364 vma = i915_gem_obj_to_vma(obj, vm, view);
3366 vma = i915_vma_create(obj, vm, view);
3367 GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view));
3370 GEM_BUG_ON(i915_vma_is_closed(vma));
3374 static struct scatterlist *
3375 rotate_pages(const dma_addr_t *in, unsigned int offset,
3376 unsigned int width, unsigned int height,
3377 unsigned int stride,
3378 struct sg_table *st, struct scatterlist *sg)
3380 unsigned int column, row;
3381 unsigned int src_idx;
3383 for (column = 0; column < width; column++) {
3384 src_idx = stride * (height - 1) + column;
3385 for (row = 0; row < height; row++) {
3387 /* We don't need the pages, but need to initialize
3388 * the entries so the sg list can be happily traversed.
3389 * The only thing we need are DMA addresses.
3391 sg_set_page(sg, NULL, PAGE_SIZE, 0);
3392 sg_dma_address(sg) = in[offset + src_idx];
3393 sg_dma_len(sg) = PAGE_SIZE;
3402 static struct sg_table *
3403 intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
3404 struct drm_i915_gem_object *obj)
3406 const size_t n_pages = obj->base.size / PAGE_SIZE;
3407 unsigned int size = intel_rotation_info_size(rot_info);
3408 struct sgt_iter sgt_iter;
3409 dma_addr_t dma_addr;
3411 dma_addr_t *page_addr_list;
3412 struct sg_table *st;
3413 struct scatterlist *sg;
3416 /* Allocate a temporary list of source pages for random access. */
3417 page_addr_list = drm_malloc_gfp(n_pages,
3420 if (!page_addr_list)
3421 return ERR_PTR(ret);
3423 /* Allocate target SG list. */
3424 st = kmalloc(sizeof(*st), GFP_KERNEL);
3428 ret = sg_alloc_table(st, size, GFP_KERNEL);
3432 /* Populate source page list from the object. */
3434 for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
3435 page_addr_list[i++] = dma_addr;
3437 GEM_BUG_ON(i != n_pages);
3441 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3442 sg = rotate_pages(page_addr_list, rot_info->plane[i].offset,
3443 rot_info->plane[i].width, rot_info->plane[i].height,
3444 rot_info->plane[i].stride, st, sg);
3447 DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
3448 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3450 drm_free_large(page_addr_list);
3457 drm_free_large(page_addr_list);
3459 DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
3460 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3462 return ERR_PTR(ret);
3465 static struct sg_table *
3466 intel_partial_pages(const struct i915_ggtt_view *view,
3467 struct drm_i915_gem_object *obj)
3469 struct sg_table *st;
3470 struct scatterlist *sg, *iter;
3471 unsigned int count = view->params.partial.size;
3472 unsigned int offset;
3475 st = kmalloc(sizeof(*st), GFP_KERNEL);
3479 ret = sg_alloc_table(st, count, GFP_KERNEL);
3483 iter = i915_gem_object_get_sg(obj,
3484 view->params.partial.offset,
3493 len = min(iter->length - (offset << PAGE_SHIFT),
3494 count << PAGE_SHIFT);
3495 sg_set_page(sg, NULL, len, 0);
3496 sg_dma_address(sg) =
3497 sg_dma_address(iter) + (offset << PAGE_SHIFT);
3498 sg_dma_len(sg) = len;
3501 count -= len >> PAGE_SHIFT;
3508 iter = __sg_next(iter);
3515 return ERR_PTR(ret);
3519 i915_get_ggtt_vma_pages(struct i915_vma *vma)
3523 /* The vma->pages are only valid within the lifespan of the borrowed
3524 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
3525 * must be the vma->pages. A simple rule is that vma->pages must only
3526 * be accessed when the obj->mm.pages are pinned.
3528 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
3533 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
3534 vma->pages = vma->obj->mm.pages;
3535 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
3537 intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
3538 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
3539 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3541 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3542 vma->ggtt_view.type);
3545 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
3546 vma->ggtt_view.type);
3548 } else if (IS_ERR(vma->pages)) {
3549 ret = PTR_ERR(vma->pages);
3551 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3552 vma->ggtt_view.type, ret);