drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <drm/drmP.h>
  26 #include <drm/i915_drm.h>
  27 #include "i915_drv.h"
  28 #include "i915_trace.h"
  29 #include "intel_drv.h"
  30
  31 /* PPGTT support for Sandybdrige/Gen6 and later */
  32 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  33                                    unsigned first_entry,
  34                                    unsigned num_entries)
  35 {
  36         uint32_t *pt_vaddr;
  37         uint32_t scratch_pte;
  38         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  39         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  40         unsigned last_pte, i;
  41
  42         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  43         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  44
  45         while (num_entries) {
  46                 last_pte = first_pte + num_entries;
  47                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  48                         last_pte = I915_PPGTT_PT_ENTRIES;
  49
  50                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  51
  52                 for (i = first_pte; i < last_pte; i++)
  53                         pt_vaddr[i] = scratch_pte;
  54
  55                 kunmap_atomic(pt_vaddr);
  56
  57                 num_entries -= last_pte - first_pte;
  58                 first_pte = 0;
  59                 act_pd++;
  60         }
  61 }
  62
  63 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  64 {
  65         struct drm_i915_private *dev_priv = dev->dev_private;
  66         struct i915_hw_ppgtt *ppgtt;
  67         unsigned first_pd_entry_in_global_pt;
  68         int i;
  69         int ret = -ENOMEM;
  70
  71         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  72          * entries. For aliasing ppgtt support we just steal them at the end for
  73          * now. */
  74         first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
  75
  76         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  77         if (!ppgtt)
  78                 return ret;
  79
  80         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  81         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  82                                   GFP_KERNEL);
  83         if (!ppgtt->pt_pages)
  84                 goto err_ppgtt;
  85
  86         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  87                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  88                 if (!ppgtt->pt_pages[i])
  89                         goto err_pt_alloc;
  90         }
  91
  92         if (dev_priv->mm.gtt->needs_dmar) {
  93                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
  94                                                 *ppgtt->num_pd_entries,
  95                                              GFP_KERNEL);
  96                 if (!ppgtt->pt_dma_addr)
  97                         goto err_pt_alloc;
  98
  99                 for (i = 0; i < ppgtt->num_pd_entries; i++) {
 100                         dma_addr_t pt_addr;
 101
 102                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 103                                                0, 4096,
 104                                                PCI_DMA_BIDIRECTIONAL);
 105
 106                         if (pci_dma_mapping_error(dev->pdev,
 107                                                   pt_addr)) {
 108                                 ret = -EIO;
 109                                 goto err_pd_pin;
 110
 111                         }
 112                         ppgtt->pt_dma_addr[i] = pt_addr;
 113                 }
 114         }
 115
 116         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 117
 118         i915_ppgtt_clear_range(ppgtt, 0,
 119                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 120
 121         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
 122
 123         dev_priv->mm.aliasing_ppgtt = ppgtt;
 124
 125         return 0;
 126
 127 err_pd_pin:
 128         if (ppgtt->pt_dma_addr) {
 129                 for (i--; i >= 0; i--)
 130                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 131                                        4096, PCI_DMA_BIDIRECTIONAL);
 132         }
 133 err_pt_alloc:
 134         kfree(ppgtt->pt_dma_addr);
 135         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 136                 if (ppgtt->pt_pages[i])
 137                         __free_page(ppgtt->pt_pages[i]);
 138         }
 139         kfree(ppgtt->pt_pages);
 140 err_ppgtt:
 141         kfree(ppgtt);
 142
 143         return ret;
 144 }
 145
 146 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 147 {
 148         struct drm_i915_private *dev_priv = dev->dev_private;
 149         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 150         int i;
 151
 152         if (!ppgtt)
 153                 return;
 154
 155         if (ppgtt->pt_dma_addr) {
 156                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 157                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 158                                        4096, PCI_DMA_BIDIRECTIONAL);
 159         }
 160
 161         kfree(ppgtt->pt_dma_addr);
 162         for (i = 0; i < ppgtt->num_pd_entries; i++)
 163                 __free_page(ppgtt->pt_pages[i]);
 164         kfree(ppgtt->pt_pages);
 165         kfree(ppgtt);
 166 }
 167
 168 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 169                                          const struct sg_table *pages,
 170                                          unsigned first_entry,
 171                                          uint32_t pte_flags)
 172 {
 173         uint32_t *pt_vaddr, pte;
 174         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 175         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 176         unsigned i, j, m, segment_len;
 177         dma_addr_t page_addr;
 178         struct scatterlist *sg;
 179
 180         /* init sg walking */
 181         sg = pages->sgl;
 182         i = 0;
 183         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 184         m = 0;
 185
 186         while (i < pages->nents) {
 187                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 188
 189                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 190                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 191                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 192                         pt_vaddr[j] = pte | pte_flags;
 193
 194                         /* grab the next page */
 195                         if (++m == segment_len) {
 196                                 if (++i == pages->nents)
 197                                         break;
 198
 199                                 sg = sg_next(sg);
 200                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 201                                 m = 0;
 202                         }
 203                 }
 204
 205                 kunmap_atomic(pt_vaddr);
 206
 207                 first_pte = 0;
 208                 act_pd++;
 209         }
 210 }
 211
 212 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 213                             struct drm_i915_gem_object *obj,
 214                             enum i915_cache_level cache_level)
 215 {
 216         uint32_t pte_flags = GEN6_PTE_VALID;
 217
 218         switch (cache_level) {
 219         case I915_CACHE_LLC_MLC:
 220                 /* Haswell doesn't set L3 this way */
 221                 if (IS_HASWELL(obj->base.dev))
 222                         pte_flags |= GEN6_PTE_CACHE_LLC;
 223                 else
 224                         pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 225                 break;
 226         case I915_CACHE_LLC:
 227                 pte_flags |= GEN6_PTE_CACHE_LLC;
 228                 break;
 229         case I915_CACHE_NONE:
 230                 if (IS_HASWELL(obj->base.dev))
 231                         pte_flags |= HSW_PTE_UNCACHED;
 232                 else
 233                         pte_flags |= GEN6_PTE_UNCACHED;
 234                 break;
 235         default:
 236                 BUG();
 237         }
 238
 239         i915_ppgtt_insert_sg_entries(ppgtt,
 240                                      obj->pages,
 241                                      obj->gtt_space->start >> PAGE_SHIFT,
 242                                      pte_flags);
 243 }
 244
 245 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 246                               struct drm_i915_gem_object *obj)
 247 {
 248         i915_ppgtt_clear_range(ppgtt,
 249                                obj->gtt_space->start >> PAGE_SHIFT,
 250                                obj->base.size >> PAGE_SHIFT);
 251 }
 252
 253 /* XXX kill agp_type! */
 254 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
 255                                             enum i915_cache_level cache_level)
 256 {
 257         switch (cache_level) {
 258         case I915_CACHE_LLC_MLC:
 259                 /* Older chipsets do not have this extra level of CPU
 260                  * cacheing, so fallthrough and request the PTE simply
 261                  * as cached.
 262                  */
 263                 if (INTEL_INFO(dev)->gen >= 6 && !IS_HASWELL(dev))
 264                         return AGP_USER_CACHED_MEMORY_LLC_MLC;
 265         case I915_CACHE_LLC:
 266                 return AGP_USER_CACHED_MEMORY;
 267         default:
 268         case I915_CACHE_NONE:
 269                 return AGP_USER_MEMORY;
 270         }
 271 }
 272
 273 static bool do_idling(struct drm_i915_private *dev_priv)
 274 {
 275         bool ret = dev_priv->mm.interruptible;
 276
 277         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 278                 dev_priv->mm.interruptible = false;
 279                 if (i915_gpu_idle(dev_priv->dev)) {
 280                         DRM_ERROR("Couldn't idle GPU\n");
 281                         /* Wait a bit, in hopes it avoids the hang */
 282                         udelay(10);
 283                 }
 284         }
 285
 286         return ret;
 287 }
 288
 289 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 290 {
 291         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 292                 dev_priv->mm.interruptible = interruptible;
 293 }
 294
 295 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 296 {
 297         struct drm_i915_private *dev_priv = dev->dev_private;
 298         struct drm_i915_gem_object *obj;
 299
 300         /* First fill our portion of the GTT with scratch pages */
 301         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 302                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 303
 304         list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 305                 i915_gem_clflush_object(obj);
 306                 i915_gem_gtt_bind_object(obj, obj->cache_level);
 307         }
 308
 309         intel_gtt_chipset_flush();
 310 }
 311
 312 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 313 {
 314         if (obj->has_dma_mapping)
 315                 return 0;
 316
 317         if (!dma_map_sg(&obj->base.dev->pdev->dev,
 318                         obj->pages->sgl, obj->pages->nents,
 319                         PCI_DMA_BIDIRECTIONAL))
 320                 return -ENOSPC;
 321
 322         return 0;
 323 }
 324
 325 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 326                               enum i915_cache_level cache_level)
 327 {
 328         struct drm_device *dev = obj->base.dev;
 329         unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 330
 331         intel_gtt_insert_sg_entries(obj->pages,
 332                                     obj->gtt_space->start >> PAGE_SHIFT,
 333                                     agp_type);
 334         obj->has_global_gtt_mapping = 1;
 335 }
 336
 337 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 338 {
 339         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 340                               obj->base.size >> PAGE_SHIFT);
 341
 342         obj->has_global_gtt_mapping = 0;
 343 }
 344
 345 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 346 {
 347         struct drm_device *dev = obj->base.dev;
 348         struct drm_i915_private *dev_priv = dev->dev_private;
 349         bool interruptible;
 350
 351         interruptible = do_idling(dev_priv);
 352
 353         if (!obj->has_dma_mapping)
 354                 dma_unmap_sg(&dev->pdev->dev,
 355                              obj->pages->sgl, obj->pages->nents,
 356                              PCI_DMA_BIDIRECTIONAL);
 357
 358         undo_idling(dev_priv, interruptible);
 359 }
 360
 361 static void i915_gtt_color_adjust(struct drm_mm_node *node,
 362                                   unsigned long color,
 363                                   unsigned long *start,
 364                                   unsigned long *end)
 365 {
 366         if (node->color != color)
 367                 *start += 4096;
 368
 369         if (!list_empty(&node->node_list)) {
 370                 node = list_entry(node->node_list.next,
 371                                   struct drm_mm_node,
 372                                   node_list);
 373                 if (node->allocated && node->color != color)
 374                         *end -= 4096;
 375         }
 376 }
 377
 378 void i915_gem_init_global_gtt(struct drm_device *dev,
 379                               unsigned long start,
 380                               unsigned long mappable_end,
 381                               unsigned long end)
 382 {
 383         drm_i915_private_t *dev_priv = dev->dev_private;
 384
 385         /* Substract the guard page ... */
 386         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
 387         if (!HAS_LLC(dev))
 388                 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
 389
 390         dev_priv->mm.gtt_start = start;
 391         dev_priv->mm.gtt_mappable_end = mappable_end;
 392         dev_priv->mm.gtt_end = end;
 393         dev_priv->mm.gtt_total = end - start;
 394         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 395
 396         /* ... but ensure that we clear the entire range. */
 397         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 398 }