drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38 #include <linux/dma-buf.h>
  39
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  42 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  43                                                     unsigned alignment,
  44                                                     bool map_and_fenceable,
  45                                                     bool nonblocking);
  46 static int i915_gem_phys_pwrite(struct drm_device *dev,
  47                                 struct drm_i915_gem_object *obj,
  48                                 struct drm_i915_gem_pwrite *args,
  49                                 struct drm_file *file);
  50
  51 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  52                                  struct drm_i915_gem_object *obj);
  53 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  54                                          struct drm_i915_fence_reg *fence,
  55                                          bool enable);
  56
  57 static unsigned long i915_gem_inactive_count(struct shrinker *shrinker,
  58                                              struct shrink_control *sc);
  59 static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker,
  60                                             struct shrink_control *sc);
  61 static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
  62 static long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
  63 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  64
  65 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  66 {
  67         if (obj->tiling_mode)
  68                 i915_gem_release_mmap(obj);
  69
  70         /* As we do not have an associated fence register, we will force
  71          * a tiling change if we ever need to acquire one.
  72          */
  73         obj->fence_dirty = false;
  74         obj->fence_reg = I915_FENCE_REG_NONE;
  75 }
  76
  77 /* some bookkeeping */
  78 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  79                                   size_t size)
  80 {
  81         spin_lock(&dev_priv->mm.object_stat_lock);
  82         dev_priv->mm.object_count++;
  83         dev_priv->mm.object_memory += size;
  84         spin_unlock(&dev_priv->mm.object_stat_lock);
  85 }
  86
  87 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  88                                      size_t size)
  89 {
  90         spin_lock(&dev_priv->mm.object_stat_lock);
  91         dev_priv->mm.object_count--;
  92         dev_priv->mm.object_memory -= size;
  93         spin_unlock(&dev_priv->mm.object_stat_lock);
  94 }
  95
  96 static int
  97 i915_gem_wait_for_error(struct i915_gpu_error *error)
  98 {
  99         int ret;
 100
 101 #define EXIT_COND (!i915_reset_in_progress(error) || \
 102                    i915_terminally_wedged(error))
 103         if (EXIT_COND)
 104                 return 0;
 105
 106         /*
 107          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 108          * userspace. If it takes that long something really bad is going on and
 109          * we should simply try to bail out and fail as gracefully as possible.
 110          */
 111         ret = wait_event_interruptible_timeout(error->reset_queue,
 112                                                EXIT_COND,
 113                                                10*HZ);
 114         if (ret == 0) {
 115                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 116                 return -EIO;
 117         } else if (ret < 0) {
 118                 return ret;
 119         }
 120 #undef EXIT_COND
 121
 122         return 0;
 123 }
 124
 125 int i915_mutex_lock_interruptible(struct drm_device *dev)
 126 {
 127         struct drm_i915_private *dev_priv = dev->dev_private;
 128         int ret;
 129
 130         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 131         if (ret)
 132                 return ret;
 133
 134         ret = mutex_lock_interruptible(&dev->struct_mutex);
 135         if (ret)
 136                 return ret;
 137
 138         WARN_ON(i915_verify_lists(dev));
 139         return 0;
 140 }
 141
 142 static inline bool
 143 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 144 {
 145         return i915_gem_obj_ggtt_bound(obj) && !obj->active;
 146 }
 147
 148 int
 149 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 150                     struct drm_file *file)
 151 {
 152         struct drm_i915_private *dev_priv = dev->dev_private;
 153         struct drm_i915_gem_init *args = data;
 154
 155         if (drm_core_check_feature(dev, DRIVER_MODESET))
 156                 return -ENODEV;
 157
 158         if (args->gtt_start >= args->gtt_end ||
 159             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 160                 return -EINVAL;
 161
 162         /* GEM with user mode setting was never supported on ilk and later. */
 163         if (INTEL_INFO(dev)->gen >= 5)
 164                 return -ENODEV;
 165
 166         mutex_lock(&dev->struct_mutex);
 167         i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
 168                                   args->gtt_end);
 169         dev_priv->gtt.mappable_end = args->gtt_end;
 170         mutex_unlock(&dev->struct_mutex);
 171
 172         return 0;
 173 }
 174
 175 int
 176 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 177                             struct drm_file *file)
 178 {
 179         struct drm_i915_private *dev_priv = dev->dev_private;
 180         struct drm_i915_gem_get_aperture *args = data;
 181         struct drm_i915_gem_object *obj;
 182         size_t pinned;
 183
 184         pinned = 0;
 185         mutex_lock(&dev->struct_mutex);
 186         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
 187                 if (obj->pin_count)
 188                         pinned += i915_gem_obj_ggtt_size(obj);
 189         mutex_unlock(&dev->struct_mutex);
 190
 191         args->aper_size = dev_priv->gtt.base.total;
 192         args->aper_available_size = args->aper_size - pinned;
 193
 194         return 0;
 195 }
 196
 197 void *i915_gem_object_alloc(struct drm_device *dev)
 198 {
 199         struct drm_i915_private *dev_priv = dev->dev_private;
 200         return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
 201 }
 202
 203 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 204 {
 205         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 206         kmem_cache_free(dev_priv->slab, obj);
 207 }
 208
 209 static int
 210 i915_gem_create(struct drm_file *file,
 211                 struct drm_device *dev,
 212                 uint64_t size,
 213                 uint32_t *handle_p)
 214 {
 215         struct drm_i915_gem_object *obj;
 216         int ret;
 217         u32 handle;
 218
 219         size = roundup(size, PAGE_SIZE);
 220         if (size == 0)
 221                 return -EINVAL;
 222
 223         /* Allocate the new object */
 224         obj = i915_gem_alloc_object(dev, size);
 225         if (obj == NULL)
 226                 return -ENOMEM;
 227
 228         ret = drm_gem_handle_create(file, &obj->base, &handle);
 229         /* drop reference from allocate - handle holds it now */
 230         drm_gem_object_unreference_unlocked(&obj->base);
 231         if (ret)
 232                 return ret;
 233
 234         *handle_p = handle;
 235         return 0;
 236 }
 237
 238 int
 239 i915_gem_dumb_create(struct drm_file *file,
 240                      struct drm_device *dev,
 241                      struct drm_mode_create_dumb *args)
 242 {
 243         /* have to work out size/pitch and return them */
 244         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
 245         args->size = args->pitch * args->height;
 246         return i915_gem_create(file, dev,
 247                                args->size, &args->handle);
 248 }
 249
 250 int i915_gem_dumb_destroy(struct drm_file *file,
 251                           struct drm_device *dev,
 252                           uint32_t handle)
 253 {
 254         return drm_gem_handle_delete(file, handle);
 255 }
 256
 257 /**
 258  * Creates a new mm object and returns a handle to it.
 259  */
 260 int
 261 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 262                       struct drm_file *file)
 263 {
 264         struct drm_i915_gem_create *args = data;
 265
 266         return i915_gem_create(file, dev,
 267                                args->size, &args->handle);
 268 }
 269
 270 static inline int
 271 __copy_to_user_swizzled(char __user *cpu_vaddr,
 272                         const char *gpu_vaddr, int gpu_offset,
 273                         int length)
 274 {
 275         int ret, cpu_offset = 0;
 276
 277         while (length > 0) {
 278                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 279                 int this_length = min(cacheline_end - gpu_offset, length);
 280                 int swizzled_gpu_offset = gpu_offset ^ 64;
 281
 282                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 283                                      gpu_vaddr + swizzled_gpu_offset,
 284                                      this_length);
 285                 if (ret)
 286                         return ret + length;
 287
 288                 cpu_offset += this_length;
 289                 gpu_offset += this_length;
 290                 length -= this_length;
 291         }
 292
 293         return 0;
 294 }
 295
 296 static inline int
 297 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 298                           const char __user *cpu_vaddr,
 299                           int length)
 300 {
 301         int ret, cpu_offset = 0;
 302
 303         while (length > 0) {
 304                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 305                 int this_length = min(cacheline_end - gpu_offset, length);
 306                 int swizzled_gpu_offset = gpu_offset ^ 64;
 307
 308                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 309                                        cpu_vaddr + cpu_offset,
 310                                        this_length);
 311                 if (ret)
 312                         return ret + length;
 313
 314                 cpu_offset += this_length;
 315                 gpu_offset += this_length;
 316                 length -= this_length;
 317         }
 318
 319         return 0;
 320 }
 321
 322 /* Per-page copy function for the shmem pread fastpath.
 323  * Flushes invalid cachelines before reading the target if
 324  * needs_clflush is set. */
 325 static int
 326 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 327                  char __user *user_data,
 328                  bool page_do_bit17_swizzling, bool needs_clflush)
 329 {
 330         char *vaddr;
 331         int ret;
 332
 333         if (unlikely(page_do_bit17_swizzling))
 334                 return -EINVAL;
 335
 336         vaddr = kmap_atomic(page);
 337         if (needs_clflush)
 338                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 339                                        page_length);
 340         ret = __copy_to_user_inatomic(user_data,
 341                                       vaddr + shmem_page_offset,
 342                                       page_length);
 343         kunmap_atomic(vaddr);
 344
 345         return ret ? -EFAULT : 0;
 346 }
 347
 348 static void
 349 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 350                              bool swizzled)
 351 {
 352         if (unlikely(swizzled)) {
 353                 unsigned long start = (unsigned long) addr;
 354                 unsigned long end = (unsigned long) addr + length;
 355
 356                 /* For swizzling simply ensure that we always flush both
 357                  * channels. Lame, but simple and it works. Swizzled
 358                  * pwrite/pread is far from a hotpath - current userspace
 359                  * doesn't use it at all. */
 360                 start = round_down(start, 128);
 361                 end = round_up(end, 128);
 362
 363                 drm_clflush_virt_range((void *)start, end - start);
 364         } else {
 365                 drm_clflush_virt_range(addr, length);
 366         }
 367
 368 }
 369
 370 /* Only difference to the fast-path function is that this can handle bit17
 371  * and uses non-atomic copy and kmap functions. */
 372 static int
 373 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 374                  char __user *user_data,
 375                  bool page_do_bit17_swizzling, bool needs_clflush)
 376 {
 377         char *vaddr;
 378         int ret;
 379
 380         vaddr = kmap(page);
 381         if (needs_clflush)
 382                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 383                                              page_length,
 384                                              page_do_bit17_swizzling);
 385
 386         if (page_do_bit17_swizzling)
 387                 ret = __copy_to_user_swizzled(user_data,
 388                                               vaddr, shmem_page_offset,
 389                                               page_length);
 390         else
 391                 ret = __copy_to_user(user_data,
 392                                      vaddr + shmem_page_offset,
 393                                      page_length);
 394         kunmap(page);
 395
 396         return ret ? - EFAULT : 0;
 397 }
 398
 399 static int
 400 i915_gem_shmem_pread(struct drm_device *dev,
 401                      struct drm_i915_gem_object *obj,
 402                      struct drm_i915_gem_pread *args,
 403                      struct drm_file *file)
 404 {
 405         char __user *user_data;
 406         ssize_t remain;
 407         loff_t offset;
 408         int shmem_page_offset, page_length, ret = 0;
 409         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 410         int prefaulted = 0;
 411         int needs_clflush = 0;
 412         struct sg_page_iter sg_iter;
 413
 414         user_data = to_user_ptr(args->data_ptr);
 415         remain = args->size;
 416
 417         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 418
 419         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 420                 /* If we're not in the cpu read domain, set ourself into the gtt
 421                  * read domain and manually flush cachelines (if required). This
 422                  * optimizes for the case when the gpu will dirty the data
 423                  * anyway again before the next pread happens. */
 424                 if (obj->cache_level == I915_CACHE_NONE)
 425                         needs_clflush = 1;
 426                 if (i915_gem_obj_ggtt_bound(obj)) {
 427                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 428                         if (ret)
 429                                 return ret;
 430                 }
 431         }
 432
 433         ret = i915_gem_object_get_pages(obj);
 434         if (ret)
 435                 return ret;
 436
 437         i915_gem_object_pin_pages(obj);
 438
 439         offset = args->offset;
 440
 441         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 442                          offset >> PAGE_SHIFT) {
 443                 struct page *page = sg_page_iter_page(&sg_iter);
 444
 445                 if (remain <= 0)
 446                         break;
 447
 448                 /* Operation in this page
 449                  *
 450                  * shmem_page_offset = offset within page in shmem file
 451                  * page_length = bytes to copy for this page
 452                  */
 453                 shmem_page_offset = offset_in_page(offset);
 454                 page_length = remain;
 455                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 456                         page_length = PAGE_SIZE - shmem_page_offset;
 457
 458                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 459                         (page_to_phys(page) & (1 << 17)) != 0;
 460
 461                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 462                                        user_data, page_do_bit17_swizzling,
 463                                        needs_clflush);
 464                 if (ret == 0)
 465                         goto next_page;
 466
 467                 mutex_unlock(&dev->struct_mutex);
 468
 469                 if (likely(!i915_prefault_disable) && !prefaulted) {
 470                         ret = fault_in_multipages_writeable(user_data, remain);
 471                         /* Userspace is tricking us, but we've already clobbered
 472                          * its pages with the prefault and promised to write the
 473                          * data up to the first fault. Hence ignore any errors
 474                          * and just continue. */
 475                         (void)ret;
 476                         prefaulted = 1;
 477                 }
 478
 479                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 480                                        user_data, page_do_bit17_swizzling,
 481                                        needs_clflush);
 482
 483                 mutex_lock(&dev->struct_mutex);
 484
 485 next_page:
 486                 mark_page_accessed(page);
 487
 488                 if (ret)
 489                         goto out;
 490
 491                 remain -= page_length;
 492                 user_data += page_length;
 493                 offset += page_length;
 494         }
 495
 496 out:
 497         i915_gem_object_unpin_pages(obj);
 498
 499         return ret;
 500 }
 501
 502 /**
 503  * Reads data from the object referenced by handle.
 504  *
 505  * On error, the contents of *data are undefined.
 506  */
 507 int
 508 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 509                      struct drm_file *file)
 510 {
 511         struct drm_i915_gem_pread *args = data;
 512         struct drm_i915_gem_object *obj;
 513         int ret = 0;
 514
 515         if (args->size == 0)
 516                 return 0;
 517
 518         if (!access_ok(VERIFY_WRITE,
 519                        to_user_ptr(args->data_ptr),
 520                        args->size))
 521                 return -EFAULT;
 522
 523         ret = i915_mutex_lock_interruptible(dev);
 524         if (ret)
 525                 return ret;
 526
 527         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 528         if (&obj->base == NULL) {
 529                 ret = -ENOENT;
 530                 goto unlock;
 531         }
 532
 533         /* Bounds check source.  */
 534         if (args->offset > obj->base.size ||
 535             args->size > obj->base.size - args->offset) {
 536                 ret = -EINVAL;
 537                 goto out;
 538         }
 539
 540         /* prime objects have no backing filp to GEM pread/pwrite
 541          * pages from.
 542          */
 543         if (!obj->base.filp) {
 544                 ret = -EINVAL;
 545                 goto out;
 546         }
 547
 548         trace_i915_gem_object_pread(obj, args->offset, args->size);
 549
 550         ret = i915_gem_shmem_pread(dev, obj, args, file);
 551
 552 out:
 553         drm_gem_object_unreference(&obj->base);
 554 unlock:
 555         mutex_unlock(&dev->struct_mutex);
 556         return ret;
 557 }
 558
 559 /* This is the fast write path which cannot handle
 560  * page faults in the source data
 561  */
 562
 563 static inline int
 564 fast_user_write(struct io_mapping *mapping,
 565                 loff_t page_base, int page_offset,
 566                 char __user *user_data,
 567                 int length)
 568 {
 569         void __iomem *vaddr_atomic;
 570         void *vaddr;
 571         unsigned long unwritten;
 572
 573         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 574         /* We can use the cpu mem copy function because this is X86. */
 575         vaddr = (void __force*)vaddr_atomic + page_offset;
 576         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 577                                                       user_data, length);
 578         io_mapping_unmap_atomic(vaddr_atomic);
 579         return unwritten;
 580 }
 581
 582 /**
 583  * This is the fast pwrite path, where we copy the data directly from the
 584  * user into the GTT, uncached.
 585  */
 586 static int
 587 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 588                          struct drm_i915_gem_object *obj,
 589                          struct drm_i915_gem_pwrite *args,
 590                          struct drm_file *file)
 591 {
 592         drm_i915_private_t *dev_priv = dev->dev_private;
 593         ssize_t remain;
 594         loff_t offset, page_base;
 595         char __user *user_data;
 596         int page_offset, page_length, ret;
 597
 598         ret = i915_gem_object_pin(obj, 0, true, true);
 599         if (ret)
 600                 goto out;
 601
 602         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 603         if (ret)
 604                 goto out_unpin;
 605
 606         ret = i915_gem_object_put_fence(obj);
 607         if (ret)
 608                 goto out_unpin;
 609
 610         user_data = to_user_ptr(args->data_ptr);
 611         remain = args->size;
 612
 613         offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
 614
 615         while (remain > 0) {
 616                 /* Operation in this page
 617                  *
 618                  * page_base = page offset within aperture
 619                  * page_offset = offset within page
 620                  * page_length = bytes to copy for this page
 621                  */
 622                 page_base = offset & PAGE_MASK;
 623                 page_offset = offset_in_page(offset);
 624                 page_length = remain;
 625                 if ((page_offset + remain) > PAGE_SIZE)
 626                         page_length = PAGE_SIZE - page_offset;
 627
 628                 /* If we get a fault while copying data, then (presumably) our
 629                  * source page isn't available.  Return the error and we'll
 630                  * retry in the slow path.
 631                  */
 632                 if (fast_user_write(dev_priv->gtt.mappable, page_base,
 633                                     page_offset, user_data, page_length)) {
 634                         ret = -EFAULT;
 635                         goto out_unpin;
 636                 }
 637
 638                 remain -= page_length;
 639                 user_data += page_length;
 640                 offset += page_length;
 641         }
 642
 643 out_unpin:
 644         i915_gem_object_unpin(obj);
 645 out:
 646         return ret;
 647 }
 648
 649 /* Per-page copy function for the shmem pwrite fastpath.
 650  * Flushes invalid cachelines before writing to the target if
 651  * needs_clflush_before is set and flushes out any written cachelines after
 652  * writing if needs_clflush is set. */
 653 static int
 654 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 655                   char __user *user_data,
 656                   bool page_do_bit17_swizzling,
 657                   bool needs_clflush_before,
 658                   bool needs_clflush_after)
 659 {
 660         char *vaddr;
 661         int ret;
 662
 663         if (unlikely(page_do_bit17_swizzling))
 664                 return -EINVAL;
 665
 666         vaddr = kmap_atomic(page);
 667         if (needs_clflush_before)
 668                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 669                                        page_length);
 670         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 671                                                 user_data,
 672                                                 page_length);
 673         if (needs_clflush_after)
 674                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 675                                        page_length);
 676         kunmap_atomic(vaddr);
 677
 678         return ret ? -EFAULT : 0;
 679 }
 680
 681 /* Only difference to the fast-path function is that this can handle bit17
 682  * and uses non-atomic copy and kmap functions. */
 683 static int
 684 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 685                   char __user *user_data,
 686                   bool page_do_bit17_swizzling,
 687                   bool needs_clflush_before,
 688                   bool needs_clflush_after)
 689 {
 690         char *vaddr;
 691         int ret;
 692
 693         vaddr = kmap(page);
 694         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 695                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 696                                              page_length,
 697                                              page_do_bit17_swizzling);
 698         if (page_do_bit17_swizzling)
 699                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 700                                                 user_data,
 701                                                 page_length);
 702         else
 703                 ret = __copy_from_user(vaddr + shmem_page_offset,
 704                                        user_data,
 705                                        page_length);
 706         if (needs_clflush_after)
 707                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 708                                              page_length,
 709                                              page_do_bit17_swizzling);
 710         kunmap(page);
 711
 712         return ret ? -EFAULT : 0;
 713 }
 714
 715 static int
 716 i915_gem_shmem_pwrite(struct drm_device *dev,
 717                       struct drm_i915_gem_object *obj,
 718                       struct drm_i915_gem_pwrite *args,
 719                       struct drm_file *file)
 720 {
 721         ssize_t remain;
 722         loff_t offset;
 723         char __user *user_data;
 724         int shmem_page_offset, page_length, ret = 0;
 725         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 726         int hit_slowpath = 0;
 727         int needs_clflush_after = 0;
 728         int needs_clflush_before = 0;
 729         struct sg_page_iter sg_iter;
 730
 731         user_data = to_user_ptr(args->data_ptr);
 732         remain = args->size;
 733
 734         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 735
 736         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 737                 /* If we're not in the cpu write domain, set ourself into the gtt
 738                  * write domain and manually flush cachelines (if required). This
 739                  * optimizes for the case when the gpu will use the data
 740                  * right away and we therefore have to clflush anyway. */
 741                 if (obj->cache_level == I915_CACHE_NONE)
 742                         needs_clflush_after = 1;
 743                 if (i915_gem_obj_ggtt_bound(obj)) {
 744                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 745                         if (ret)
 746                                 return ret;
 747                 }
 748         }
 749         /* Same trick applies for invalidate partially written cachelines before
 750          * writing.  */
 751         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
 752             && obj->cache_level == I915_CACHE_NONE)
 753                 needs_clflush_before = 1;
 754
 755         ret = i915_gem_object_get_pages(obj);
 756         if (ret)
 757                 return ret;
 758
 759         i915_gem_object_pin_pages(obj);
 760
 761         offset = args->offset;
 762         obj->dirty = 1;
 763
 764         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 765                          offset >> PAGE_SHIFT) {
 766                 struct page *page = sg_page_iter_page(&sg_iter);
 767                 int partial_cacheline_write;
 768
 769                 if (remain <= 0)
 770                         break;
 771
 772                 /* Operation in this page
 773                  *
 774                  * shmem_page_offset = offset within page in shmem file
 775                  * page_length = bytes to copy for this page
 776                  */
 777                 shmem_page_offset = offset_in_page(offset);
 778
 779                 page_length = remain;
 780                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 781                         page_length = PAGE_SIZE - shmem_page_offset;
 782
 783                 /* If we don't overwrite a cacheline completely we need to be
 784                  * careful to have up-to-date data by first clflushing. Don't
 785                  * overcomplicate things and flush the entire patch. */
 786                 partial_cacheline_write = needs_clflush_before &&
 787                         ((shmem_page_offset | page_length)
 788                                 & (boot_cpu_data.x86_clflush_size - 1));
 789
 790                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 791                         (page_to_phys(page) & (1 << 17)) != 0;
 792
 793                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 794                                         user_data, page_do_bit17_swizzling,
 795                                         partial_cacheline_write,
 796                                         needs_clflush_after);
 797                 if (ret == 0)
 798                         goto next_page;
 799
 800                 hit_slowpath = 1;
 801                 mutex_unlock(&dev->struct_mutex);
 802                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 803                                         user_data, page_do_bit17_swizzling,
 804                                         partial_cacheline_write,
 805                                         needs_clflush_after);
 806
 807                 mutex_lock(&dev->struct_mutex);
 808
 809 next_page:
 810                 set_page_dirty(page);
 811                 mark_page_accessed(page);
 812
 813                 if (ret)
 814                         goto out;
 815
 816                 remain -= page_length;
 817                 user_data += page_length;
 818                 offset += page_length;
 819         }
 820
 821 out:
 822         i915_gem_object_unpin_pages(obj);
 823
 824         if (hit_slowpath) {
 825                 /*
 826                  * Fixup: Flush cpu caches in case we didn't flush the dirty
 827                  * cachelines in-line while writing and the object moved
 828                  * out of the cpu write domain while we've dropped the lock.
 829                  */
 830                 if (!needs_clflush_after &&
 831                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 832                         i915_gem_clflush_object(obj);
 833                         i915_gem_chipset_flush(dev);
 834                 }
 835         }
 836
 837         if (needs_clflush_after)
 838                 i915_gem_chipset_flush(dev);
 839
 840         return ret;
 841 }
 842
 843 /**
 844  * Writes data to the object referenced by handle.
 845  *
 846  * On error, the contents of the buffer that were to be modified are undefined.
 847  */
 848 int
 849 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 850                       struct drm_file *file)
 851 {
 852         struct drm_i915_gem_pwrite *args = data;
 853         struct drm_i915_gem_object *obj;
 854         int ret;
 855
 856         if (args->size == 0)
 857                 return 0;
 858
 859         if (!access_ok(VERIFY_READ,
 860                        to_user_ptr(args->data_ptr),
 861                        args->size))
 862                 return -EFAULT;
 863
 864         if (likely(!i915_prefault_disable)) {
 865                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
 866                                                    args->size);
 867                 if (ret)
 868                         return -EFAULT;
 869         }
 870
 871         ret = i915_mutex_lock_interruptible(dev);
 872         if (ret)
 873                 return ret;
 874
 875         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 876         if (&obj->base == NULL) {
 877                 ret = -ENOENT;
 878                 goto unlock;
 879         }
 880
 881         /* Bounds check destination. */
 882         if (args->offset > obj->base.size ||
 883             args->size > obj->base.size - args->offset) {
 884                 ret = -EINVAL;
 885                 goto out;
 886         }
 887
 888         /* prime objects have no backing filp to GEM pread/pwrite
 889          * pages from.
 890          */
 891         if (!obj->base.filp) {
 892                 ret = -EINVAL;
 893                 goto out;
 894         }
 895
 896         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 897
 898         ret = -EFAULT;
 899         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 900          * it would end up going through the fenced access, and we'll get
 901          * different detiling behavior between reading and writing.
 902          * pread/pwrite currently are reading and writing from the CPU
 903          * perspective, requiring manual detiling by the client.
 904          */
 905         if (obj->phys_obj) {
 906                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 907                 goto out;
 908         }
 909
 910         if (obj->cache_level == I915_CACHE_NONE &&
 911             obj->tiling_mode == I915_TILING_NONE &&
 912             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 913                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 914                 /* Note that the gtt paths might fail with non-page-backed user
 915                  * pointers (e.g. gtt mappings when moving data between
 916                  * textures). Fallback to the shmem path in that case. */
 917         }
 918
 919         if (ret == -EFAULT || ret == -ENOSPC)
 920                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 921
 922 out:
 923         drm_gem_object_unreference(&obj->base);
 924 unlock:
 925         mutex_unlock(&dev->struct_mutex);
 926         return ret;
 927 }
 928
 929 int
 930 i915_gem_check_wedge(struct i915_gpu_error *error,
 931                      bool interruptible)
 932 {
 933         if (i915_reset_in_progress(error)) {
 934                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 935                  * -EIO unconditionally for these. */
 936                 if (!interruptible)
 937                         return -EIO;
 938
 939                 /* Recovery complete, but the reset failed ... */
 940                 if (i915_terminally_wedged(error))
 941                         return -EIO;
 942
 943                 return -EAGAIN;
 944         }
 945
 946         return 0;
 947 }
 948
 949 /*
 950  * Compare seqno against outstanding lazy request. Emit a request if they are
 951  * equal.
 952  */
 953 static int
 954 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
 955 {
 956         int ret;
 957
 958         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
 959
 960         ret = 0;
 961         if (seqno == ring->outstanding_lazy_request)
 962                 ret = i915_add_request(ring, NULL);
 963
 964         return ret;
 965 }
 966
 967 /**
 968  * __wait_seqno - wait until execution of seqno has finished
 969  * @ring: the ring expected to report seqno
 970  * @seqno: duh!
 971  * @reset_counter: reset sequence associated with the given seqno
 972  * @interruptible: do an interruptible wait (normally yes)
 973  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 974  *
 975  * Note: It is of utmost importance that the passed in seqno and reset_counter
 976  * values have been read by the caller in an smp safe manner. Where read-side
 977  * locks are involved, it is sufficient to read the reset_counter before
 978  * unlocking the lock that protects the seqno. For lockless tricks, the
 979  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
 980  * inserted.
 981  *
 982  * Returns 0 if the seqno was found within the alloted time. Else returns the
 983  * errno with remaining time filled in timeout argument.
 984  */
 985 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
 986                         unsigned reset_counter,
 987                         bool interruptible, struct timespec *timeout)
 988 {
 989         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 990         struct timespec before, now, wait_time={1,0};
 991         unsigned long timeout_jiffies;
 992         long end;
 993         bool wait_forever = true;
 994         int ret;
 995
 996         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
 997                 return 0;
 998
 999         trace_i915_gem_request_wait_begin(ring, seqno);
1000
1001         if (timeout != NULL) {
1002                 wait_time = *timeout;
1003                 wait_forever = false;
1004         }
1005
1006         timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
1007
1008         if (WARN_ON(!ring->irq_get(ring)))
1009                 return -ENODEV;
1010
1011         /* Record current time in case interrupted by signal, or wedged * */
1012         getrawmonotonic(&before);
1013
1014 #define EXIT_COND \
1015         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1016          i915_reset_in_progress(&dev_priv->gpu_error) || \
1017          reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1018         do {
1019                 if (interruptible)
1020                         end = wait_event_interruptible_timeout(ring->irq_queue,
1021                                                                EXIT_COND,
1022                                                                timeout_jiffies);
1023                 else
1024                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
1025                                                  timeout_jiffies);
1026
1027                 /* We need to check whether any gpu reset happened in between
1028                  * the caller grabbing the seqno and now ... */
1029                 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1030                         end = -EAGAIN;
1031
1032                 /* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
1033                  * gone. */
1034                 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1035                 if (ret)
1036                         end = ret;
1037         } while (end == 0 && wait_forever);
1038
1039         getrawmonotonic(&now);
1040
1041         ring->irq_put(ring);
1042         trace_i915_gem_request_wait_end(ring, seqno);
1043 #undef EXIT_COND
1044
1045         if (timeout) {
1046                 struct timespec sleep_time = timespec_sub(now, before);
1047                 *timeout = timespec_sub(*timeout, sleep_time);
1048                 if (!timespec_valid(timeout)) /* i.e. negative time remains */
1049                         set_normalized_timespec(timeout, 0, 0);
1050         }
1051
1052         switch (end) {
1053         case -EIO:
1054         case -EAGAIN: /* Wedged */
1055         case -ERESTARTSYS: /* Signal */
1056                 return (int)end;
1057         case 0: /* Timeout */
1058                 return -ETIME;
1059         default: /* Completed */
1060                 WARN_ON(end < 0); /* We're not aware of other errors */
1061                 return 0;
1062         }
1063 }
1064
1065 /**
1066  * Waits for a sequence number to be signaled, and cleans up the
1067  * request and object lists appropriately for that event.
1068  */
1069 int
1070 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1071 {
1072         struct drm_device *dev = ring->dev;
1073         struct drm_i915_private *dev_priv = dev->dev_private;
1074         bool interruptible = dev_priv->mm.interruptible;
1075         int ret;
1076
1077         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1078         BUG_ON(seqno == 0);
1079
1080         ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1081         if (ret)
1082                 return ret;
1083
1084         ret = i915_gem_check_olr(ring, seqno);
1085         if (ret)
1086                 return ret;
1087
1088         return __wait_seqno(ring, seqno,
1089                             atomic_read(&dev_priv->gpu_error.reset_counter),
1090                             interruptible, NULL);
1091 }
1092
1093 static int
1094 i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
1095                                      struct intel_ring_buffer *ring)
1096 {
1097         i915_gem_retire_requests_ring(ring);
1098
1099         /* Manually manage the write flush as we may have not yet
1100          * retired the buffer.
1101          *
1102          * Note that the last_write_seqno is always the earlier of
1103          * the two (read/write) seqno, so if we haved successfully waited,
1104          * we know we have passed the last write.
1105          */
1106         obj->last_write_seqno = 0;
1107         obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1108
1109         return 0;
1110 }
1111
1112 /**
1113  * Ensures that all rendering to the object has completed and the object is
1114  * safe to unbind from the GTT or access from the CPU.
1115  */
1116 static __must_check int
1117 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1118                                bool readonly)
1119 {
1120         struct intel_ring_buffer *ring = obj->ring;
1121         u32 seqno;
1122         int ret;
1123
1124         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1125         if (seqno == 0)
1126                 return 0;
1127
1128         ret = i915_wait_seqno(ring, seqno);
1129         if (ret)
1130                 return ret;
1131
1132         return i915_gem_object_wait_rendering__tail(obj, ring);
1133 }
1134
1135 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1136  * as the object state may change during this call.
1137  */
1138 static __must_check int
1139 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1140                                             bool readonly)
1141 {
1142         struct drm_device *dev = obj->base.dev;
1143         struct drm_i915_private *dev_priv = dev->dev_private;
1144         struct intel_ring_buffer *ring = obj->ring;
1145         unsigned reset_counter;
1146         u32 seqno;
1147         int ret;
1148
1149         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1150         BUG_ON(!dev_priv->mm.interruptible);
1151
1152         seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1153         if (seqno == 0)
1154                 return 0;
1155
1156         ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1157         if (ret)
1158                 return ret;
1159
1160         ret = i915_gem_check_olr(ring, seqno);
1161         if (ret)
1162                 return ret;
1163
1164         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1165         mutex_unlock(&dev->struct_mutex);
1166         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
1167         mutex_lock(&dev->struct_mutex);
1168         if (ret)
1169                 return ret;
1170
1171         return i915_gem_object_wait_rendering__tail(obj, ring);
1172 }
1173
1174 /**
1175  * Called when user space prepares to use an object with the CPU, either
1176  * through the mmap ioctl's mapping or a GTT mapping.
1177  */
1178 int
1179 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1180                           struct drm_file *file)
1181 {
1182         struct drm_i915_gem_set_domain *args = data;
1183         struct drm_i915_gem_object *obj;
1184         uint32_t read_domains = args->read_domains;
1185         uint32_t write_domain = args->write_domain;
1186         int ret;
1187
1188         /* Only handle setting domains to types used by the CPU. */
1189         if (write_domain & I915_GEM_GPU_DOMAINS)
1190                 return -EINVAL;
1191
1192         if (read_domains & I915_GEM_GPU_DOMAINS)
1193                 return -EINVAL;
1194
1195         /* Having something in the write domain implies it's in the read
1196          * domain, and only that read domain.  Enforce that in the request.
1197          */
1198         if (write_domain != 0 && read_domains != write_domain)
1199                 return -EINVAL;
1200
1201         ret = i915_mutex_lock_interruptible(dev);
1202         if (ret)
1203                 return ret;
1204
1205         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1206         if (&obj->base == NULL) {
1207                 ret = -ENOENT;
1208                 goto unlock;
1209         }
1210
1211         /* Try to flush the object off the GPU without holding the lock.
1212          * We will repeat the flush holding the lock in the normal manner
1213          * to catch cases where we are gazumped.
1214          */
1215         ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
1216         if (ret)
1217                 goto unref;
1218
1219         if (read_domains & I915_GEM_DOMAIN_GTT) {
1220                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1221
1222                 /* Silently promote "you're not bound, there was nothing to do"
1223                  * to success, since the client was just asking us to
1224                  * make sure everything was done.
1225                  */
1226                 if (ret == -EINVAL)
1227                         ret = 0;
1228         } else {
1229                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1230         }
1231
1232 unref:
1233         drm_gem_object_unreference(&obj->base);
1234 unlock:
1235         mutex_unlock(&dev->struct_mutex);
1236         return ret;
1237 }
1238
1239 /**
1240  * Called when user space has done writes to this buffer
1241  */
1242 int
1243 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1244                          struct drm_file *file)
1245 {
1246         struct drm_i915_gem_sw_finish *args = data;
1247         struct drm_i915_gem_object *obj;
1248         int ret = 0;
1249
1250         ret = i915_mutex_lock_interruptible(dev);
1251         if (ret)
1252                 return ret;
1253
1254         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1255         if (&obj->base == NULL) {
1256                 ret = -ENOENT;
1257                 goto unlock;
1258         }
1259
1260         /* Pinned buffers may be scanout, so flush the cache */
1261         if (obj->pin_count)
1262                 i915_gem_object_flush_cpu_write_domain(obj);
1263
1264         drm_gem_object_unreference(&obj->base);
1265 unlock:
1266         mutex_unlock(&dev->struct_mutex);
1267         return ret;
1268 }
1269
1270 /**
1271  * Maps the contents of an object, returning the address it is mapped
1272  * into.
1273  *
1274  * While the mapping holds a reference on the contents of the object, it doesn't
1275  * imply a ref on the object itself.
1276  */
1277 int
1278 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1279                     struct drm_file *file)
1280 {
1281         struct drm_i915_gem_mmap *args = data;
1282         struct drm_gem_object *obj;
1283         unsigned long addr;
1284
1285         obj = drm_gem_object_lookup(dev, file, args->handle);
1286         if (obj == NULL)
1287                 return -ENOENT;
1288
1289         /* prime objects have no backing filp to GEM mmap
1290          * pages from.
1291          */
1292         if (!obj->filp) {
1293                 drm_gem_object_unreference_unlocked(obj);
1294                 return -EINVAL;
1295         }
1296
1297         addr = vm_mmap(obj->filp, 0, args->size,
1298                        PROT_READ | PROT_WRITE, MAP_SHARED,
1299                        args->offset);
1300         drm_gem_object_unreference_unlocked(obj);
1301         if (IS_ERR((void *)addr))
1302                 return addr;
1303
1304         args->addr_ptr = (uint64_t) addr;
1305
1306         return 0;
1307 }
1308
1309 /**
1310  * i915_gem_fault - fault a page into the GTT
1311  * vma: VMA in question
1312  * vmf: fault info
1313  *
1314  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1315  * from userspace.  The fault handler takes care of binding the object to
1316  * the GTT (if needed), allocating and programming a fence register (again,
1317  * only if needed based on whether the old reg is still valid or the object
1318  * is tiled) and inserting a new PTE into the faulting process.
1319  *
1320  * Note that the faulting process may involve evicting existing objects
1321  * from the GTT and/or fence registers to make room.  So performance may
1322  * suffer if the GTT working set is large or there are few fence registers
1323  * left.
1324  */
1325 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1326 {
1327         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1328         struct drm_device *dev = obj->base.dev;
1329         drm_i915_private_t *dev_priv = dev->dev_private;
1330         pgoff_t page_offset;
1331         unsigned long pfn;
1332         int ret = 0;
1333         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1334
1335         /* We don't use vmf->pgoff since that has the fake offset */
1336         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1337                 PAGE_SHIFT;
1338
1339         ret = i915_mutex_lock_interruptible(dev);
1340         if (ret)
1341                 goto out;
1342
1343         trace_i915_gem_object_fault(obj, page_offset, true, write);
1344
1345         /* Access to snoopable pages through the GTT is incoherent. */
1346         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1347                 ret = -EINVAL;
1348                 goto unlock;
1349         }
1350
1351         /* Now bind it into the GTT if needed */
1352         ret = i915_gem_object_pin(obj, 0, true, false);
1353         if (ret)
1354                 goto unlock;
1355
1356         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1357         if (ret)
1358                 goto unpin;
1359
1360         ret = i915_gem_object_get_fence(obj);
1361         if (ret)
1362                 goto unpin;
1363
1364         obj->fault_mappable = true;
1365
1366         pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
1367         pfn >>= PAGE_SHIFT;
1368         pfn += page_offset;
1369
1370         /* Finally, remap it using the new GTT offset */
1371         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1372 unpin:
1373         i915_gem_object_unpin(obj);
1374 unlock:
1375         mutex_unlock(&dev->struct_mutex);
1376 out:
1377         switch (ret) {
1378         case -EIO:
1379                 /* If this -EIO is due to a gpu hang, give the reset code a
1380                  * chance to clean up the mess. Otherwise return the proper
1381                  * SIGBUS. */
1382                 if (i915_terminally_wedged(&dev_priv->gpu_error))
1383                         return VM_FAULT_SIGBUS;
1384         case -EAGAIN:
1385                 /* Give the error handler a chance to run and move the
1386                  * objects off the GPU active list. Next time we service the
1387                  * fault, we should be able to transition the page into the
1388                  * GTT without touching the GPU (and so avoid further
1389                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1390                  * with coherency, just lost writes.
1391                  */
1392                 set_need_resched();
1393         case 0:
1394         case -ERESTARTSYS:
1395         case -EINTR:
1396         case -EBUSY:
1397                 /*
1398                  * EBUSY is ok: this just means that another thread
1399                  * already did the job.
1400                  */
1401                 return VM_FAULT_NOPAGE;
1402         case -ENOMEM:
1403                 return VM_FAULT_OOM;
1404         case -ENOSPC:
1405                 return VM_FAULT_SIGBUS;
1406         default:
1407                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1408                 return VM_FAULT_SIGBUS;
1409         }
1410 }
1411
1412 /**
1413  * i915_gem_release_mmap - remove physical page mappings
1414  * @obj: obj in question
1415  *
1416  * Preserve the reservation of the mmapping with the DRM core code, but
1417  * relinquish ownership of the pages back to the system.
1418  *
1419  * It is vital that we remove the page mapping if we have mapped a tiled
1420  * object through the GTT and then lose the fence register due to
1421  * resource pressure. Similarly if the object has been moved out of the
1422  * aperture, than pages mapped into userspace must be revoked. Removing the
1423  * mapping will then trigger a page fault on the next user access, allowing
1424  * fixup by i915_gem_fault().
1425  */
1426 void
1427 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1428 {
1429         if (!obj->fault_mappable)
1430                 return;
1431
1432         drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
1433         obj->fault_mappable = false;
1434 }
1435
1436 uint32_t
1437 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1438 {
1439         uint32_t gtt_size;
1440
1441         if (INTEL_INFO(dev)->gen >= 4 ||
1442             tiling_mode == I915_TILING_NONE)
1443                 return size;
1444
1445         /* Previous chips need a power-of-two fence region when tiling */
1446         if (INTEL_INFO(dev)->gen == 3)
1447                 gtt_size = 1024*1024;
1448         else
1449                 gtt_size = 512*1024;
1450
1451         while (gtt_size < size)
1452                 gtt_size <<= 1;
1453
1454         return gtt_size;
1455 }
1456
1457 /**
1458  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1459  * @obj: object to check
1460  *
1461  * Return the required GTT alignment for an object, taking into account
1462  * potential fence register mapping.
1463  */
1464 uint32_t
1465 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1466                            int tiling_mode, bool fenced)
1467 {
1468         /*
1469          * Minimum alignment is 4k (GTT page size), but might be greater
1470          * if a fence register is needed for the object.
1471          */
1472         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1473             tiling_mode == I915_TILING_NONE)
1474                 return 4096;
1475
1476         /*
1477          * Previous chips need to be aligned to the size of the smallest
1478          * fence register that can contain the object.
1479          */
1480         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1481 }
1482
1483 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1484 {
1485         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1486         int ret;
1487
1488         if (drm_vma_node_has_offset(&obj->base.vma_node))
1489                 return 0;
1490
1491         dev_priv->mm.shrinker_no_lock_stealing = true;
1492
1493         ret = drm_gem_create_mmap_offset(&obj->base);
1494         if (ret != -ENOSPC)
1495                 goto out;
1496
1497         /* Badly fragmented mmap space? The only way we can recover
1498          * space is by destroying unwanted objects. We can't randomly release
1499          * mmap_offsets as userspace expects them to be persistent for the
1500          * lifetime of the objects. The closest we can is to release the
1501          * offsets on purgeable objects by truncating it and marking it purged,
1502          * which prevents userspace from ever using that object again.
1503          */
1504         i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1505         ret = drm_gem_create_mmap_offset(&obj->base);
1506         if (ret != -ENOSPC)
1507                 goto out;
1508
1509         i915_gem_shrink_all(dev_priv);
1510         ret = drm_gem_create_mmap_offset(&obj->base);
1511 out:
1512         dev_priv->mm.shrinker_no_lock_stealing = false;
1513
1514         return ret;
1515 }
1516
1517 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1518 {
1519         drm_gem_free_mmap_offset(&obj->base);
1520 }
1521
1522 int
1523 i915_gem_mmap_gtt(struct drm_file *file,
1524                   struct drm_device *dev,
1525                   uint32_t handle,
1526                   uint64_t *offset)
1527 {
1528         struct drm_i915_private *dev_priv = dev->dev_private;
1529         struct drm_i915_gem_object *obj;
1530         int ret;
1531
1532         ret = i915_mutex_lock_interruptible(dev);
1533         if (ret)
1534                 return ret;
1535
1536         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1537         if (&obj->base == NULL) {
1538                 ret = -ENOENT;
1539                 goto unlock;
1540         }
1541
1542         if (obj->base.size > dev_priv->gtt.mappable_end) {
1543                 ret = -E2BIG;
1544                 goto out;
1545         }
1546
1547         if (obj->madv != I915_MADV_WILLNEED) {
1548                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1549                 ret = -EINVAL;
1550                 goto out;
1551         }
1552
1553         ret = i915_gem_object_create_mmap_offset(obj);
1554         if (ret)
1555                 goto out;
1556
1557         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1558
1559 out:
1560         drm_gem_object_unreference(&obj->base);
1561 unlock:
1562         mutex_unlock(&dev->struct_mutex);
1563         return ret;
1564 }
1565
1566 /**
1567  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1568  * @dev: DRM device
1569  * @data: GTT mapping ioctl data
1570  * @file: GEM object info
1571  *
1572  * Simply returns the fake offset to userspace so it can mmap it.
1573  * The mmap call will end up in drm_gem_mmap(), which will set things
1574  * up so we can get faults in the handler above.
1575  *
1576  * The fault handler will take care of binding the object into the GTT
1577  * (since it may have been evicted to make room for something), allocating
1578  * a fence register, and mapping the appropriate aperture address into
1579  * userspace.
1580  */
1581 int
1582 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1583                         struct drm_file *file)
1584 {
1585         struct drm_i915_gem_mmap_gtt *args = data;
1586
1587         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1588 }
1589
1590 /* Immediately discard the backing storage */
1591 static void
1592 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1593 {
1594         struct inode *inode;
1595
1596         i915_gem_object_free_mmap_offset(obj);
1597
1598         if (obj->base.filp == NULL)
1599                 return;
1600
1601         /* Our goal here is to return as much of the memory as
1602          * is possible back to the system as we are called from OOM.
1603          * To do this we must instruct the shmfs to drop all of its
1604          * backing pages, *now*.
1605          */
1606         inode = file_inode(obj->base.filp);
1607         shmem_truncate_range(inode, 0, (loff_t)-1);
1608
1609         obj->madv = __I915_MADV_PURGED;
1610 }
1611
1612 static inline int
1613 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1614 {
1615         return obj->madv == I915_MADV_DONTNEED;
1616 }
1617
1618 static void
1619 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1620 {
1621         struct sg_page_iter sg_iter;
1622         int ret;
1623
1624         BUG_ON(obj->madv == __I915_MADV_PURGED);
1625
1626         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1627         if (ret) {
1628                 /* In the event of a disaster, abandon all caches and
1629                  * hope for the best.
1630                  */
1631                 WARN_ON(ret != -EIO);
1632                 i915_gem_clflush_object(obj);
1633                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1634         }
1635
1636         if (i915_gem_object_needs_bit17_swizzle(obj))
1637                 i915_gem_object_save_bit_17_swizzle(obj);
1638
1639         if (obj->madv == I915_MADV_DONTNEED)
1640                 obj->dirty = 0;
1641
1642         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1643                 struct page *page = sg_page_iter_page(&sg_iter);
1644
1645                 if (obj->dirty)
1646                         set_page_dirty(page);
1647
1648                 if (obj->madv == I915_MADV_WILLNEED)
1649                         mark_page_accessed(page);
1650
1651                 page_cache_release(page);
1652         }
1653         obj->dirty = 0;
1654
1655         sg_free_table(obj->pages);
1656         kfree(obj->pages);
1657 }
1658
1659 int
1660 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1661 {
1662         const struct drm_i915_gem_object_ops *ops = obj->ops;
1663
1664         if (obj->pages == NULL)
1665                 return 0;
1666
1667         BUG_ON(i915_gem_obj_ggtt_bound(obj));
1668
1669         if (obj->pages_pin_count)
1670                 return -EBUSY;
1671
1672         /* ->put_pages might need to allocate memory for the bit17 swizzle
1673          * array, hence protect them from being reaped by removing them from gtt
1674          * lists early. */
1675         list_del(&obj->global_list);
1676
1677         ops->put_pages(obj);
1678         obj->pages = NULL;
1679
1680         if (i915_gem_object_is_purgeable(obj))
1681                 i915_gem_object_truncate(obj);
1682
1683         return 0;
1684 }
1685
1686 static long
1687 __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
1688                   bool purgeable_only)
1689 {
1690         struct drm_i915_gem_object *obj, *next;
1691         struct i915_address_space *vm = &dev_priv->gtt.base;
1692         long count = 0;
1693
1694         list_for_each_entry_safe(obj, next,
1695                                  &dev_priv->mm.unbound_list,
1696                                  global_list) {
1697                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1698                     i915_gem_object_put_pages(obj) == 0) {
1699                         count += obj->base.size >> PAGE_SHIFT;
1700                         if (count >= target)
1701                                 return count;
1702                 }
1703         }
1704
1705         list_for_each_entry_safe(obj, next, &vm->inactive_list, mm_list) {
1706                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1707                     i915_gem_object_unbind(obj) == 0 &&
1708                     i915_gem_object_put_pages(obj) == 0) {
1709                         count += obj->base.size >> PAGE_SHIFT;
1710                         if (count >= target)
1711                                 return count;
1712                 }
1713         }
1714
1715         return count;
1716 }
1717
1718 static long
1719 i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1720 {
1721         return __i915_gem_shrink(dev_priv, target, true);
1722 }
1723
1724 static long
1725 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1726 {
1727         struct drm_i915_gem_object *obj, *next;
1728         long freed = 0;
1729
1730         i915_gem_evict_everything(dev_priv->dev);
1731
1732         list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
1733                                  global_list) {
1734                 if (obj->pages_pin_count == 0)
1735                         freed += obj->base.size >> PAGE_SHIFT;
1736                 i915_gem_object_put_pages(obj);
1737         }
1738         return freed;
1739 }
1740
1741 static int
1742 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1743 {
1744         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1745         int page_count, i;
1746         struct address_space *mapping;
1747         struct sg_table *st;
1748         struct scatterlist *sg;
1749         struct sg_page_iter sg_iter;
1750         struct page *page;
1751         unsigned long last_pfn = 0;     /* suppress gcc warning */
1752         gfp_t gfp;
1753
1754         /* Assert that the object is not currently in any GPU domain. As it
1755          * wasn't in the GTT, there shouldn't be any way it could have been in
1756          * a GPU cache
1757          */
1758         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1759         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1760
1761         st = kmalloc(sizeof(*st), GFP_KERNEL);
1762         if (st == NULL)
1763                 return -ENOMEM;
1764
1765         page_count = obj->base.size / PAGE_SIZE;
1766         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1767                 sg_free_table(st);
1768                 kfree(st);
1769                 return -ENOMEM;
1770         }
1771
1772         /* Get the list of pages out of our struct file.  They'll be pinned
1773          * at this point until we release them.
1774          *
1775          * Fail silently without starting the shrinker
1776          */
1777         mapping = file_inode(obj->base.filp)->i_mapping;
1778         gfp = mapping_gfp_mask(mapping);
1779         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1780         gfp &= ~(__GFP_IO | __GFP_WAIT);
1781         sg = st->sgl;
1782         st->nents = 0;
1783         for (i = 0; i < page_count; i++) {
1784                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1785                 if (IS_ERR(page)) {
1786                         i915_gem_purge(dev_priv, page_count);
1787                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1788                 }
1789                 if (IS_ERR(page)) {
1790                         /* We've tried hard to allocate the memory by reaping
1791                          * our own buffer, now let the real VM do its job and
1792                          * go down in flames if truly OOM.
1793                          */
1794                         gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
1795                         gfp |= __GFP_IO | __GFP_WAIT;
1796
1797                         i915_gem_shrink_all(dev_priv);
1798                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1799                         if (IS_ERR(page))
1800                                 goto err_pages;
1801
1802                         gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
1803                         gfp &= ~(__GFP_IO | __GFP_WAIT);
1804                 }
1805 #ifdef CONFIG_SWIOTLB
1806                 if (swiotlb_nr_tbl()) {
1807                         st->nents++;
1808                         sg_set_page(sg, page, PAGE_SIZE, 0);
1809                         sg = sg_next(sg);
1810                         continue;
1811                 }
1812 #endif
1813                 if (!i || page_to_pfn(page) != last_pfn + 1) {
1814                         if (i)
1815                                 sg = sg_next(sg);
1816                         st->nents++;
1817                         sg_set_page(sg, page, PAGE_SIZE, 0);
1818                 } else {
1819                         sg->length += PAGE_SIZE;
1820                 }
1821                 last_pfn = page_to_pfn(page);
1822         }
1823 #ifdef CONFIG_SWIOTLB
1824         if (!swiotlb_nr_tbl())
1825 #endif
1826                 sg_mark_end(sg);
1827         obj->pages = st;
1828
1829         if (i915_gem_object_needs_bit17_swizzle(obj))
1830                 i915_gem_object_do_bit_17_swizzle(obj);
1831
1832         return 0;
1833
1834 err_pages:
1835         sg_mark_end(sg);
1836         for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1837                 page_cache_release(sg_page_iter_page(&sg_iter));
1838         sg_free_table(st);
1839         kfree(st);
1840         return PTR_ERR(page);
1841 }
1842
1843 /* Ensure that the associated pages are gathered from the backing storage
1844  * and pinned into our object. i915_gem_object_get_pages() may be called
1845  * multiple times before they are released by a single call to
1846  * i915_gem_object_put_pages() - once the pages are no longer referenced
1847  * either as a result of memory pressure (reaping pages under the shrinker)
1848  * or as the object is itself released.
1849  */
1850 int
1851 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1852 {
1853         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1854         const struct drm_i915_gem_object_ops *ops = obj->ops;
1855         int ret;
1856
1857         if (obj->pages)
1858                 return 0;
1859
1860         if (obj->madv != I915_MADV_WILLNEED) {
1861                 DRM_ERROR("Attempting to obtain a purgeable object\n");
1862                 return -EINVAL;
1863         }
1864
1865         BUG_ON(obj->pages_pin_count);
1866
1867         ret = ops->get_pages(obj);
1868         if (ret)
1869                 return ret;
1870
1871         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1872         return 0;
1873 }
1874
1875 void
1876 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1877                                struct intel_ring_buffer *ring)
1878 {
1879         struct drm_device *dev = obj->base.dev;
1880         struct drm_i915_private *dev_priv = dev->dev_private;
1881         struct i915_address_space *vm = &dev_priv->gtt.base;
1882         u32 seqno = intel_ring_get_seqno(ring);
1883
1884         BUG_ON(ring == NULL);
1885         if (obj->ring != ring && obj->last_write_seqno) {
1886                 /* Keep the seqno relative to the current ring */
1887                 obj->last_write_seqno = seqno;
1888         }
1889         obj->ring = ring;
1890
1891         /* Add a reference if we're newly entering the active list. */
1892         if (!obj->active) {
1893                 drm_gem_object_reference(&obj->base);
1894                 obj->active = 1;
1895         }
1896
1897         /* Move from whatever list we were on to the tail of execution. */
1898         list_move_tail(&obj->mm_list, &vm->active_list);
1899         list_move_tail(&obj->ring_list, &ring->active_list);
1900
1901         obj->last_read_seqno = seqno;
1902
1903         if (obj->fenced_gpu_access) {
1904                 obj->last_fenced_seqno = seqno;
1905
1906                 /* Bump MRU to take account of the delayed flush */
1907                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1908                         struct drm_i915_fence_reg *reg;
1909
1910                         reg = &dev_priv->fence_regs[obj->fence_reg];
1911                         list_move_tail(&reg->lru_list,
1912                                        &dev_priv->mm.fence_list);
1913                 }
1914         }
1915 }
1916
1917 static void
1918 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1919 {
1920         struct drm_device *dev = obj->base.dev;
1921         struct drm_i915_private *dev_priv = dev->dev_private;
1922         struct i915_address_space *vm = &dev_priv->gtt.base;
1923
1924         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1925         BUG_ON(!obj->active);
1926
1927         list_move_tail(&obj->mm_list, &vm->inactive_list);
1928
1929         list_del_init(&obj->ring_list);
1930         obj->ring = NULL;
1931
1932         obj->last_read_seqno = 0;
1933         obj->last_write_seqno = 0;
1934         obj->base.write_domain = 0;
1935
1936         obj->last_fenced_seqno = 0;
1937         obj->fenced_gpu_access = false;
1938
1939         obj->active = 0;
1940         drm_gem_object_unreference(&obj->base);
1941
1942         WARN_ON(i915_verify_lists(dev));
1943 }
1944
1945 static int
1946 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
1947 {
1948         struct drm_i915_private *dev_priv = dev->dev_private;
1949         struct intel_ring_buffer *ring;
1950         int ret, i, j;
1951
1952         /* Carefully retire all requests without writing to the rings */
1953         for_each_ring(ring, dev_priv, i) {
1954                 ret = intel_ring_idle(ring);
1955                 if (ret)
1956                         return ret;
1957         }
1958         i915_gem_retire_requests(dev);
1959
1960         /* Finally reset hw state */
1961         for_each_ring(ring, dev_priv, i) {
1962                 intel_ring_init_seqno(ring, seqno);
1963
1964                 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
1965                         ring->sync_seqno[j] = 0;
1966         }
1967
1968         return 0;
1969 }
1970
1971 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
1972 {
1973         struct drm_i915_private *dev_priv = dev->dev_private;
1974         int ret;
1975
1976         if (seqno == 0)
1977                 return -EINVAL;
1978
1979         /* HWS page needs to be set less than what we
1980          * will inject to ring
1981          */
1982         ret = i915_gem_init_seqno(dev, seqno - 1);
1983         if (ret)
1984                 return ret;
1985
1986         /* Carefully set the last_seqno value so that wrap
1987          * detection still works
1988          */
1989         dev_priv->next_seqno = seqno;
1990         dev_priv->last_seqno = seqno - 1;
1991         if (dev_priv->last_seqno == 0)
1992                 dev_priv->last_seqno--;
1993
1994         return 0;
1995 }
1996
1997 int
1998 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
1999 {
2000         struct drm_i915_private *dev_priv = dev->dev_private;
2001
2002         /* reserve 0 for non-seqno */
2003         if (dev_priv->next_seqno == 0) {
2004                 int ret = i915_gem_init_seqno(dev, 0);
2005                 if (ret)
2006                         return ret;
2007
2008                 dev_priv->next_seqno = 1;
2009         }
2010
2011         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2012         return 0;
2013 }
2014
2015 int __i915_add_request(struct intel_ring_buffer *ring,
2016                        struct drm_file *file,
2017                        struct drm_i915_gem_object *obj,
2018                        u32 *out_seqno)
2019 {
2020         drm_i915_private_t *dev_priv = ring->dev->dev_private;
2021         struct drm_i915_gem_request *request;
2022         u32 request_ring_position, request_start;
2023         int was_empty;
2024         int ret;
2025
2026         request_start = intel_ring_get_tail(ring);
2027         /*
2028          * Emit any outstanding flushes - execbuf can fail to emit the flush
2029          * after having emitted the batchbuffer command. Hence we need to fix
2030          * things up similar to emitting the lazy request. The difference here
2031          * is that the flush _must_ happen before the next request, no matter
2032          * what.
2033          */
2034         ret = intel_ring_flush_all_caches(ring);
2035         if (ret)
2036                 return ret;
2037
2038         request = kmalloc(sizeof(*request), GFP_KERNEL);
2039         if (request == NULL)
2040                 return -ENOMEM;
2041
2042
2043         /* Record the position of the start of the request so that
2044          * should we detect the updated seqno part-way through the
2045          * GPU processing the request, we never over-estimate the
2046          * position of the head.
2047          */
2048         request_ring_position = intel_ring_get_tail(ring);
2049
2050         ret = ring->add_request(ring);
2051         if (ret) {
2052                 kfree(request);
2053                 return ret;
2054         }
2055
2056         request->seqno = intel_ring_get_seqno(ring);
2057         request->ring = ring;
2058         request->head = request_start;
2059         request->tail = request_ring_position;
2060         request->ctx = ring->last_context;
2061         request->batch_obj = obj;
2062
2063         /* Whilst this request exists, batch_obj will be on the
2064          * active_list, and so will hold the active reference. Only when this
2065          * request is retired will the the batch_obj be moved onto the
2066          * inactive_list and lose its active reference. Hence we do not need
2067          * to explicitly hold another reference here.
2068          */
2069
2070         if (request->ctx)
2071                 i915_gem_context_reference(request->ctx);
2072
2073         request->emitted_jiffies = jiffies;
2074         was_empty = list_empty(&ring->request_list);
2075         list_add_tail(&request->list, &ring->request_list);
2076         request->file_priv = NULL;
2077
2078         if (file) {
2079                 struct drm_i915_file_private *file_priv = file->driver_priv;
2080
2081                 spin_lock(&file_priv->mm.lock);
2082                 request->file_priv = file_priv;
2083                 list_add_tail(&request->client_list,
2084                               &file_priv->mm.request_list);
2085                 spin_unlock(&file_priv->mm.lock);
2086         }
2087
2088         trace_i915_gem_request_add(ring, request->seqno);
2089         ring->outstanding_lazy_request = 0;
2090
2091         if (!dev_priv->ums.mm_suspended) {
2092                 i915_queue_hangcheck(ring->dev);
2093
2094                 if (was_empty) {
2095                         queue_delayed_work(dev_priv->wq,
2096                                            &dev_priv->mm.retire_work,
2097                                            round_jiffies_up_relative(HZ));
2098                         intel_mark_busy(dev_priv->dev);
2099                 }
2100         }
2101
2102         if (out_seqno)
2103                 *out_seqno = request->seqno;
2104         return 0;
2105 }
2106
2107 static inline void
2108 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2109 {
2110         struct drm_i915_file_private *file_priv = request->file_priv;
2111
2112         if (!file_priv)
2113                 return;
2114
2115         spin_lock(&file_priv->mm.lock);
2116         if (request->file_priv) {
2117                 list_del(&request->client_list);
2118                 request->file_priv = NULL;
2119         }
2120         spin_unlock(&file_priv->mm.lock);
2121 }
2122
2123 static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
2124 {
2125         if (acthd >= i915_gem_obj_ggtt_offset(obj) &&
2126             acthd < i915_gem_obj_ggtt_offset(obj) + obj->base.size)
2127                 return true;
2128
2129         return false;
2130 }
2131
2132 static bool i915_head_inside_request(const u32 acthd_unmasked,
2133                                      const u32 request_start,
2134                                      const u32 request_end)
2135 {
2136         const u32 acthd = acthd_unmasked & HEAD_ADDR;
2137
2138         if (request_start < request_end) {
2139                 if (acthd >= request_start && acthd < request_end)
2140                         return true;
2141         } else if (request_start > request_end) {
2142                 if (acthd >= request_start || acthd < request_end)
2143                         return true;
2144         }
2145
2146         return false;
2147 }
2148
2149 static bool i915_request_guilty(struct drm_i915_gem_request *request,
2150                                 const u32 acthd, bool *inside)
2151 {
2152         /* There is a possibility that unmasked head address
2153          * pointing inside the ring, matches the batch_obj address range.
2154          * However this is extremely unlikely.
2155          */
2156
2157         if (request->batch_obj) {
2158                 if (i915_head_inside_object(acthd, request->batch_obj)) {
2159                         *inside = true;
2160                         return true;
2161                 }
2162         }
2163
2164         if (i915_head_inside_request(acthd, request->head, request->tail)) {
2165                 *inside = false;
2166                 return true;
2167         }
2168
2169         return false;
2170 }
2171
2172 static void i915_set_reset_status(struct intel_ring_buffer *ring,
2173                                   struct drm_i915_gem_request *request,
2174                                   u32 acthd)
2175 {
2176         struct i915_ctx_hang_stats *hs = NULL;
2177         bool inside, guilty;
2178
2179         /* Innocent until proven guilty */
2180         guilty = false;
2181
2182         if (ring->hangcheck.action != wait &&
2183             i915_request_guilty(request, acthd, &inside)) {
2184                 DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
2185                           ring->name,
2186                           inside ? "inside" : "flushing",
2187                           request->batch_obj ?
2188                           i915_gem_obj_ggtt_offset(request->batch_obj) : 0,
2189                           request->ctx ? request->ctx->id : 0,
2190                           acthd);
2191
2192                 guilty = true;
2193         }
2194
2195         /* If contexts are disabled or this is the default context, use
2196          * file_priv->reset_state
2197          */
2198         if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
2199                 hs = &request->ctx->hang_stats;
2200         else if (request->file_priv)
2201                 hs = &request->file_priv->hang_stats;
2202
2203         if (hs) {
2204                 if (guilty)
2205                         hs->batch_active++;
2206                 else
2207                         hs->batch_pending++;
2208         }
2209 }
2210
2211 static void i915_gem_free_request(struct drm_i915_gem_request *request)
2212 {
2213         list_del(&request->list);
2214         i915_gem_request_remove_from_client(request);
2215
2216         if (request->ctx)
2217                 i915_gem_context_unreference(request->ctx);
2218
2219         kfree(request);
2220 }
2221
2222 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2223                                       struct intel_ring_buffer *ring)
2224 {
2225         u32 completed_seqno;
2226         u32 acthd;
2227
2228         acthd = intel_ring_get_active_head(ring);
2229         completed_seqno = ring->get_seqno(ring, false);
2230
2231         while (!list_empty(&ring->request_list)) {
2232                 struct drm_i915_gem_request *request;
2233
2234                 request = list_first_entry(&ring->request_list,
2235                                            struct drm_i915_gem_request,
2236                                            list);
2237
2238                 if (request->seqno > completed_seqno)
2239                         i915_set_reset_status(ring, request, acthd);
2240
2241                 i915_gem_free_request(request);
2242         }
2243
2244         while (!list_empty(&ring->active_list)) {
2245                 struct drm_i915_gem_object *obj;
2246
2247                 obj = list_first_entry(&ring->active_list,
2248                                        struct drm_i915_gem_object,
2249                                        ring_list);
2250
2251                 i915_gem_object_move_to_inactive(obj);
2252         }
2253 }
2254
2255 void i915_gem_restore_fences(struct drm_device *dev)
2256 {
2257         struct drm_i915_private *dev_priv = dev->dev_private;
2258         int i;
2259
2260         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2261                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2262
2263                 /*
2264                  * Commit delayed tiling changes if we have an object still
2265                  * attached to the fence, otherwise just clear the fence.
2266                  */
2267                 if (reg->obj) {
2268                         i915_gem_object_update_fence(reg->obj, reg,
2269                                                      reg->obj->tiling_mode);
2270                 } else {
2271                         i915_gem_write_fence(dev, i, NULL);
2272                 }
2273         }
2274 }
2275
2276 void i915_gem_reset(struct drm_device *dev)
2277 {
2278         struct drm_i915_private *dev_priv = dev->dev_private;
2279         struct i915_address_space *vm = &dev_priv->gtt.base;
2280         struct drm_i915_gem_object *obj;
2281         struct intel_ring_buffer *ring;
2282         int i;
2283
2284         for_each_ring(ring, dev_priv, i)
2285                 i915_gem_reset_ring_lists(dev_priv, ring);
2286
2287         /* Move everything out of the GPU domains to ensure we do any
2288          * necessary invalidation upon reuse.
2289          */
2290         list_for_each_entry(obj, &vm->inactive_list, mm_list)
2291                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2292
2293         i915_gem_restore_fences(dev);
2294 }
2295
2296 /**
2297  * This function clears the request list as sequence numbers are passed.
2298  */
2299 void
2300 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2301 {
2302         uint32_t seqno;
2303
2304         if (list_empty(&ring->request_list))
2305                 return;
2306
2307         WARN_ON(i915_verify_lists(ring->dev));
2308
2309         seqno = ring->get_seqno(ring, true);
2310
2311         while (!list_empty(&ring->request_list)) {
2312                 struct drm_i915_gem_request *request;
2313
2314                 request = list_first_entry(&ring->request_list,
2315                                            struct drm_i915_gem_request,
2316                                            list);
2317
2318                 if (!i915_seqno_passed(seqno, request->seqno))
2319                         break;
2320
2321                 trace_i915_gem_request_retire(ring, request->seqno);
2322                 /* We know the GPU must have read the request to have
2323                  * sent us the seqno + interrupt, so use the position
2324                  * of tail of the request to update the last known position
2325                  * of the GPU head.
2326                  */
2327                 ring->last_retired_head = request->tail;
2328
2329                 i915_gem_free_request(request);
2330         }
2331
2332         /* Move any buffers on the active list that are no longer referenced
2333          * by the ringbuffer to the flushing/inactive lists as appropriate.
2334          */
2335         while (!list_empty(&ring->active_list)) {
2336                 struct drm_i915_gem_object *obj;
2337
2338                 obj = list_first_entry(&ring->active_list,
2339                                       struct drm_i915_gem_object,
2340                                       ring_list);
2341
2342                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2343                         break;
2344
2345                 i915_gem_object_move_to_inactive(obj);
2346         }
2347
2348         if (unlikely(ring->trace_irq_seqno &&
2349                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2350                 ring->irq_put(ring);
2351                 ring->trace_irq_seqno = 0;
2352         }
2353
2354         WARN_ON(i915_verify_lists(ring->dev));
2355 }
2356
2357 void
2358 i915_gem_retire_requests(struct drm_device *dev)
2359 {
2360         drm_i915_private_t *dev_priv = dev->dev_private;
2361         struct intel_ring_buffer *ring;
2362         int i;
2363
2364         for_each_ring(ring, dev_priv, i)
2365                 i915_gem_retire_requests_ring(ring);
2366 }
2367
2368 static void
2369 i915_gem_retire_work_handler(struct work_struct *work)
2370 {
2371         drm_i915_private_t *dev_priv;
2372         struct drm_device *dev;
2373         struct intel_ring_buffer *ring;
2374         bool idle;
2375         int i;
2376
2377         dev_priv = container_of(work, drm_i915_private_t,
2378                                 mm.retire_work.work);
2379         dev = dev_priv->dev;
2380
2381         /* Come back later if the device is busy... */
2382         if (!mutex_trylock(&dev->struct_mutex)) {
2383                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2384                                    round_jiffies_up_relative(HZ));
2385                 return;
2386         }
2387
2388         i915_gem_retire_requests(dev);
2389
2390         /* Send a periodic flush down the ring so we don't hold onto GEM
2391          * objects indefinitely.
2392          */
2393         idle = true;
2394         for_each_ring(ring, dev_priv, i) {
2395                 if (ring->gpu_caches_dirty)
2396                         i915_add_request(ring, NULL);
2397
2398                 idle &= list_empty(&ring->request_list);
2399         }
2400
2401         if (!dev_priv->ums.mm_suspended && !idle)
2402                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2403                                    round_jiffies_up_relative(HZ));
2404         if (idle)
2405                 intel_mark_idle(dev);
2406
2407         mutex_unlock(&dev->struct_mutex);
2408 }
2409
2410 /**
2411  * Ensures that an object will eventually get non-busy by flushing any required
2412  * write domains, emitting any outstanding lazy request and retiring and
2413  * completed requests.
2414  */
2415 static int
2416 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2417 {
2418         int ret;
2419
2420         if (obj->active) {
2421                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2422                 if (ret)
2423                         return ret;
2424
2425                 i915_gem_retire_requests_ring(obj->ring);
2426         }
2427
2428         return 0;
2429 }
2430
2431 /**
2432  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2433  * @DRM_IOCTL_ARGS: standard ioctl arguments
2434  *
2435  * Returns 0 if successful, else an error is returned with the remaining time in
2436  * the timeout parameter.
2437  *  -ETIME: object is still busy after timeout
2438  *  -ERESTARTSYS: signal interrupted the wait
2439  *  -ENONENT: object doesn't exist
2440  * Also possible, but rare:
2441  *  -EAGAIN: GPU wedged
2442  *  -ENOMEM: damn
2443  *  -ENODEV: Internal IRQ fail
2444  *  -E?: The add request failed
2445  *
2446  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2447  * non-zero timeout parameter the wait ioctl will wait for the given number of
2448  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2449  * without holding struct_mutex the object may become re-busied before this
2450  * function completes. A similar but shorter * race condition exists in the busy
2451  * ioctl
2452  */
2453 int
2454 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2455 {
2456         drm_i915_private_t *dev_priv = dev->dev_private;
2457         struct drm_i915_gem_wait *args = data;
2458         struct drm_i915_gem_object *obj;
2459         struct intel_ring_buffer *ring = NULL;
2460         struct timespec timeout_stack, *timeout = NULL;
2461         unsigned reset_counter;
2462         u32 seqno = 0;
2463         int ret = 0;
2464
2465         if (args->timeout_ns >= 0) {
2466                 timeout_stack = ns_to_timespec(args->timeout_ns);
2467                 timeout = &timeout_stack;
2468         }
2469
2470         ret = i915_mutex_lock_interruptible(dev);
2471         if (ret)
2472                 return ret;
2473
2474         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2475         if (&obj->base == NULL) {
2476                 mutex_unlock(&dev->struct_mutex);
2477                 return -ENOENT;
2478         }
2479
2480         /* Need to make sure the object gets inactive eventually. */
2481         ret = i915_gem_object_flush_active(obj);
2482         if (ret)
2483                 goto out;
2484
2485         if (obj->active) {
2486                 seqno = obj->last_read_seqno;
2487                 ring = obj->ring;
2488         }
2489
2490         if (seqno == 0)
2491                  goto out;
2492
2493         /* Do this after OLR check to make sure we make forward progress polling
2494          * on this IOCTL with a 0 timeout (like busy ioctl)
2495          */
2496         if (!args->timeout_ns) {
2497                 ret = -ETIME;
2498                 goto out;
2499         }
2500
2501         drm_gem_object_unreference(&obj->base);
2502         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2503         mutex_unlock(&dev->struct_mutex);
2504
2505         ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
2506         if (timeout)
2507                 args->timeout_ns = timespec_to_ns(timeout);
2508         return ret;
2509
2510 out:
2511         drm_gem_object_unreference(&obj->base);
2512         mutex_unlock(&dev->struct_mutex);
2513         return ret;
2514 }
2515
2516 /**
2517  * i915_gem_object_sync - sync an object to a ring.
2518  *
2519  * @obj: object which may be in use on another ring.
2520  * @to: ring we wish to use the object on. May be NULL.
2521  *
2522  * This code is meant to abstract object synchronization with the GPU.
2523  * Calling with NULL implies synchronizing the object with the CPU
2524  * rather than a particular GPU ring.
2525  *
2526  * Returns 0 if successful, else propagates up the lower layer error.
2527  */
2528 int
2529 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2530                      struct intel_ring_buffer *to)
2531 {
2532         struct intel_ring_buffer *from = obj->ring;
2533         u32 seqno;
2534         int ret, idx;
2535
2536         if (from == NULL || to == from)
2537                 return 0;
2538
2539         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2540                 return i915_gem_object_wait_rendering(obj, false);
2541
2542         idx = intel_ring_sync_index(from, to);
2543
2544         seqno = obj->last_read_seqno;
2545         if (seqno <= from->sync_seqno[idx])
2546                 return 0;
2547
2548         ret = i915_gem_check_olr(obj->ring, seqno);
2549         if (ret)
2550                 return ret;
2551
2552         ret = to->sync_to(to, from, seqno);
2553         if (!ret)
2554                 /* We use last_read_seqno because sync_to()
2555                  * might have just caused seqno wrap under
2556                  * the radar.
2557                  */
2558                 from->sync_seqno[idx] = obj->last_read_seqno;
2559
2560         return ret;
2561 }
2562
2563 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2564 {
2565         u32 old_write_domain, old_read_domains;
2566
2567         /* Force a pagefault for domain tracking on next user access */
2568         i915_gem_release_mmap(obj);
2569
2570         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2571                 return;
2572
2573         /* Wait for any direct GTT access to complete */
2574         mb();
2575
2576         old_read_domains = obj->base.read_domains;
2577         old_write_domain = obj->base.write_domain;
2578
2579         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2580         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2581
2582         trace_i915_gem_object_change_domain(obj,
2583                                             old_read_domains,
2584                                             old_write_domain);
2585 }
2586
2587 /**
2588  * Unbinds an object from the GTT aperture.
2589  */
2590 int
2591 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2592 {
2593         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2594         struct i915_vma *vma;
2595         int ret;
2596
2597         if (!i915_gem_obj_ggtt_bound(obj))
2598                 return 0;
2599
2600         if (obj->pin_count)
2601                 return -EBUSY;
2602
2603         BUG_ON(obj->pages == NULL);
2604
2605         ret = i915_gem_object_finish_gpu(obj);
2606         if (ret)
2607                 return ret;
2608         /* Continue on if we fail due to EIO, the GPU is hung so we
2609          * should be safe and we need to cleanup or else we might
2610          * cause memory corruption through use-after-free.
2611          */
2612
2613         i915_gem_object_finish_gtt(obj);
2614
2615         /* release the fence reg _after_ flushing */
2616         ret = i915_gem_object_put_fence(obj);
2617         if (ret)
2618                 return ret;
2619
2620         trace_i915_gem_object_unbind(obj);
2621
2622         if (obj->has_global_gtt_mapping)
2623                 i915_gem_gtt_unbind_object(obj);
2624         if (obj->has_aliasing_ppgtt_mapping) {
2625                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2626                 obj->has_aliasing_ppgtt_mapping = 0;
2627         }
2628         i915_gem_gtt_finish_object(obj);
2629         i915_gem_object_unpin_pages(obj);
2630
2631         list_del(&obj->mm_list);
2632         /* Avoid an unnecessary call to unbind on rebind. */
2633         obj->map_and_fenceable = true;
2634
2635         vma = __i915_gem_obj_to_vma(obj);
2636         list_del(&vma->vma_link);
2637         drm_mm_remove_node(&vma->node);
2638         i915_gem_vma_destroy(vma);
2639
2640         /* Since the unbound list is global, only move to that list if
2641          * no more VMAs exist.
2642          * NB: Until we have real VMAs there will only ever be one */
2643         WARN_ON(!list_empty(&obj->vma_list));
2644         if (list_empty(&obj->vma_list))
2645                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2646
2647         return 0;
2648 }
2649
2650 int i915_gpu_idle(struct drm_device *dev)
2651 {
2652         drm_i915_private_t *dev_priv = dev->dev_private;
2653         struct intel_ring_buffer *ring;
2654         int ret, i;
2655
2656         /* Flush everything onto the inactive list. */
2657         for_each_ring(ring, dev_priv, i) {
2658                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2659                 if (ret)
2660                         return ret;
2661
2662                 ret = intel_ring_idle(ring);
2663                 if (ret)
2664                         return ret;
2665         }
2666
2667         return 0;
2668 }
2669
2670 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2671                                  struct drm_i915_gem_object *obj)
2672 {
2673         drm_i915_private_t *dev_priv = dev->dev_private;
2674         int fence_reg;
2675         int fence_pitch_shift;
2676
2677         if (INTEL_INFO(dev)->gen >= 6) {
2678                 fence_reg = FENCE_REG_SANDYBRIDGE_0;
2679                 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
2680         } else {
2681                 fence_reg = FENCE_REG_965_0;
2682                 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
2683         }
2684
2685         fence_reg += reg * 8;
2686
2687         /* To w/a incoherency with non-atomic 64-bit register updates,
2688          * we split the 64-bit update into two 32-bit writes. In order
2689          * for a partial fence not to be evaluated between writes, we
2690          * precede the update with write to turn off the fence register,
2691          * and only enable the fence as the last step.
2692          *
2693          * For extra levels of paranoia, we make sure each step lands
2694          * before applying the next step.
2695          */
2696         I915_WRITE(fence_reg, 0);
2697         POSTING_READ(fence_reg);
2698
2699         if (obj) {
2700                 u32 size = i915_gem_obj_ggtt_size(obj);
2701                 uint64_t val;
2702
2703                 val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2704                                  0xfffff000) << 32;
2705                 val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2706                 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2707                 if (obj->tiling_mode == I915_TILING_Y)
2708                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2709                 val |= I965_FENCE_REG_VALID;
2710
2711                 I915_WRITE(fence_reg + 4, val >> 32);
2712                 POSTING_READ(fence_reg + 4);
2713
2714                 I915_WRITE(fence_reg + 0, val);
2715                 POSTING_READ(fence_reg);
2716         } else {
2717                 I915_WRITE(fence_reg + 4, 0);
2718                 POSTING_READ(fence_reg + 4);
2719         }
2720 }
2721
2722 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2723                                  struct drm_i915_gem_object *obj)
2724 {
2725         drm_i915_private_t *dev_priv = dev->dev_private;
2726         u32 val;
2727
2728         if (obj) {
2729                 u32 size = i915_gem_obj_ggtt_size(obj);
2730                 int pitch_val;
2731                 int tile_width;
2732
2733                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2734                      (size & -size) != size ||
2735                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2736                      "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2737                      i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2738
2739                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2740                         tile_width = 128;
2741                 else
2742                         tile_width = 512;
2743
2744                 /* Note: pitch better be a power of two tile widths */
2745                 pitch_val = obj->stride / tile_width;
2746                 pitch_val = ffs(pitch_val) - 1;
2747
2748                 val = i915_gem_obj_ggtt_offset(obj);
2749                 if (obj->tiling_mode == I915_TILING_Y)
2750                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2751                 val |= I915_FENCE_SIZE_BITS(size);
2752                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2753                 val |= I830_FENCE_REG_VALID;
2754         } else
2755                 val = 0;
2756
2757         if (reg < 8)
2758                 reg = FENCE_REG_830_0 + reg * 4;
2759         else
2760                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2761
2762         I915_WRITE(reg, val);
2763         POSTING_READ(reg);
2764 }
2765
2766 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2767                                 struct drm_i915_gem_object *obj)
2768 {
2769         drm_i915_private_t *dev_priv = dev->dev_private;
2770         uint32_t val;
2771
2772         if (obj) {
2773                 u32 size = i915_gem_obj_ggtt_size(obj);
2774                 uint32_t pitch_val;
2775
2776                 WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2777                      (size & -size) != size ||
2778                      (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
2779                      "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
2780                      i915_gem_obj_ggtt_offset(obj), size);
2781
2782                 pitch_val = obj->stride / 128;
2783                 pitch_val = ffs(pitch_val) - 1;
2784
2785                 val = i915_gem_obj_ggtt_offset(obj);
2786                 if (obj->tiling_mode == I915_TILING_Y)
2787                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2788                 val |= I830_FENCE_SIZE_BITS(size);
2789                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2790                 val |= I830_FENCE_REG_VALID;
2791         } else
2792                 val = 0;
2793
2794         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2795         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2796 }
2797
2798 inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
2799 {
2800         return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
2801 }
2802
2803 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2804                                  struct drm_i915_gem_object *obj)
2805 {
2806         struct drm_i915_private *dev_priv = dev->dev_private;
2807
2808         /* Ensure that all CPU reads are completed before installing a fence
2809          * and all writes before removing the fence.
2810          */
2811         if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
2812                 mb();
2813
2814         WARN(obj && (!obj->stride || !obj->tiling_mode),
2815              "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
2816              obj->stride, obj->tiling_mode);
2817
2818         switch (INTEL_INFO(dev)->gen) {
2819         case 7:
2820         case 6:
2821         case 5:
2822         case 4: i965_write_fence_reg(dev, reg, obj); break;
2823         case 3: i915_write_fence_reg(dev, reg, obj); break;
2824         case 2: i830_write_fence_reg(dev, reg, obj); break;
2825         default: BUG();
2826         }
2827
2828         /* And similarly be paranoid that no direct access to this region
2829          * is reordered to before the fence is installed.
2830          */
2831         if (i915_gem_object_needs_mb(obj))
2832                 mb();
2833 }
2834
2835 static inline int fence_number(struct drm_i915_private *dev_priv,
2836                                struct drm_i915_fence_reg *fence)
2837 {
2838         return fence - dev_priv->fence_regs;
2839 }
2840
2841 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2842                                          struct drm_i915_fence_reg *fence,
2843                                          bool enable)
2844 {
2845         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2846         int reg = fence_number(dev_priv, fence);
2847
2848         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2849
2850         if (enable) {
2851                 obj->fence_reg = reg;
2852                 fence->obj = obj;
2853                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2854         } else {
2855                 obj->fence_reg = I915_FENCE_REG_NONE;
2856                 fence->obj = NULL;
2857                 list_del_init(&fence->lru_list);
2858         }
2859         obj->fence_dirty = false;
2860 }
2861
2862 static int
2863 i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
2864 {
2865         if (obj->last_fenced_seqno) {
2866                 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2867                 if (ret)
2868                         return ret;
2869
2870                 obj->last_fenced_seqno = 0;
2871         }
2872
2873         obj->fenced_gpu_access = false;
2874         return 0;
2875 }
2876
2877 int
2878 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2879 {
2880         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2881         struct drm_i915_fence_reg *fence;
2882         int ret;
2883
2884         ret = i915_gem_object_wait_fence(obj);
2885         if (ret)
2886                 return ret;
2887
2888         if (obj->fence_reg == I915_FENCE_REG_NONE)
2889                 return 0;
2890
2891         fence = &dev_priv->fence_regs[obj->fence_reg];
2892
2893         i915_gem_object_fence_lost(obj);
2894         i915_gem_object_update_fence(obj, fence, false);
2895
2896         return 0;
2897 }
2898
2899 static struct drm_i915_fence_reg *
2900 i915_find_fence_reg(struct drm_device *dev)
2901 {
2902         struct drm_i915_private *dev_priv = dev->dev_private;
2903         struct drm_i915_fence_reg *reg, *avail;
2904         int i;
2905
2906         /* First try to find a free reg */
2907         avail = NULL;
2908         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2909                 reg = &dev_priv->fence_regs[i];
2910                 if (!reg->obj)
2911                         return reg;
2912
2913                 if (!reg->pin_count)
2914                         avail = reg;
2915         }
2916
2917         if (avail == NULL)
2918                 return NULL;
2919
2920         /* None available, try to steal one or wait for a user to finish */
2921         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2922                 if (reg->pin_count)
2923                         continue;
2924
2925                 return reg;
2926         }
2927
2928         return NULL;
2929 }
2930
2931 /**
2932  * i915_gem_object_get_fence - set up fencing for an object
2933  * @obj: object to map through a fence reg
2934  *
2935  * When mapping objects through the GTT, userspace wants to be able to write
2936  * to them without having to worry about swizzling if the object is tiled.
2937  * This function walks the fence regs looking for a free one for @obj,
2938  * stealing one if it can't find any.
2939  *
2940  * It then sets up the reg based on the object's properties: address, pitch
2941  * and tiling format.
2942  *
2943  * For an untiled surface, this removes any existing fence.
2944  */
2945 int
2946 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2947 {
2948         struct drm_device *dev = obj->base.dev;
2949         struct drm_i915_private *dev_priv = dev->dev_private;
2950         bool enable = obj->tiling_mode != I915_TILING_NONE;
2951         struct drm_i915_fence_reg *reg;
2952         int ret;
2953
2954         /* Have we updated the tiling parameters upon the object and so
2955          * will need to serialise the write to the associated fence register?
2956          */
2957         if (obj->fence_dirty) {
2958                 ret = i915_gem_object_wait_fence(obj);
2959                 if (ret)
2960                         return ret;
2961         }
2962
2963         /* Just update our place in the LRU if our fence is getting reused. */
2964         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2965                 reg = &dev_priv->fence_regs[obj->fence_reg];
2966                 if (!obj->fence_dirty) {
2967                         list_move_tail(&reg->lru_list,
2968                                        &dev_priv->mm.fence_list);
2969                         return 0;
2970                 }
2971         } else if (enable) {
2972                 reg = i915_find_fence_reg(dev);
2973                 if (reg == NULL)
2974                         return -EDEADLK;
2975
2976                 if (reg->obj) {
2977                         struct drm_i915_gem_object *old = reg->obj;
2978
2979                         ret = i915_gem_object_wait_fence(old);
2980                         if (ret)
2981                                 return ret;
2982
2983                         i915_gem_object_fence_lost(old);
2984                 }
2985         } else
2986                 return 0;
2987
2988         i915_gem_object_update_fence(obj, reg, enable);
2989
2990         return 0;
2991 }
2992
2993 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
2994                                      struct drm_mm_node *gtt_space,
2995                                      unsigned long cache_level)
2996 {
2997         struct drm_mm_node *other;
2998
2999         /* On non-LLC machines we have to be careful when putting differing
3000          * types of snoopable memory together to avoid the prefetcher
3001          * crossing memory domains and dying.
3002          */
3003         if (HAS_LLC(dev))
3004                 return true;
3005
3006         if (!drm_mm_node_allocated(gtt_space))
3007                 return true;
3008
3009         if (list_empty(&gtt_space->node_list))
3010                 return true;
3011
3012         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3013         if (other->allocated && !other->hole_follows && other->color != cache_level)
3014                 return false;
3015
3016         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3017         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3018                 return false;
3019
3020         return true;
3021 }
3022
3023 static void i915_gem_verify_gtt(struct drm_device *dev)
3024 {
3025 #if WATCH_GTT
3026         struct drm_i915_private *dev_priv = dev->dev_private;
3027         struct drm_i915_gem_object *obj;
3028         int err = 0;
3029
3030         list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031                 if (obj->gtt_space == NULL) {
3032                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
3033                         err++;
3034                         continue;
3035                 }
3036
3037                 if (obj->cache_level != obj->gtt_space->color) {
3038                         printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3039                                i915_gem_obj_ggtt_offset(obj),
3040                                i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3041                                obj->cache_level,
3042                                obj->gtt_space->color);
3043                         err++;
3044                         continue;
3045                 }
3046
3047                 if (!i915_gem_valid_gtt_space(dev,
3048                                               obj->gtt_space,
3049                                               obj->cache_level)) {
3050                         printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3051                                i915_gem_obj_ggtt_offset(obj),
3052                                i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3053                                obj->cache_level);
3054                         err++;
3055                         continue;
3056                 }
3057         }
3058
3059         WARN_ON(err);
3060 #endif
3061 }
3062
3063 /**
3064  * Finds free space in the GTT aperture and binds the object there.
3065  */
3066 static int
3067 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
3068                             unsigned alignment,
3069                             bool map_and_fenceable,
3070                             bool nonblocking)
3071 {
3072         struct drm_device *dev = obj->base.dev;
3073         drm_i915_private_t *dev_priv = dev->dev_private;
3074         struct i915_address_space *vm = &dev_priv->gtt.base;
3075         u32 size, fence_size, fence_alignment, unfenced_alignment;
3076         bool mappable, fenceable;
3077         size_t gtt_max = map_and_fenceable ?
3078                 dev_priv->gtt.mappable_end : dev_priv->gtt.base.total;
3079         struct i915_vma *vma;
3080         int ret;
3081
3082         if (WARN_ON(!list_empty(&obj->vma_list)))
3083                 return -EBUSY;
3084
3085         fence_size = i915_gem_get_gtt_size(dev,
3086                                            obj->base.size,
3087                                            obj->tiling_mode);
3088         fence_alignment = i915_gem_get_gtt_alignment(dev,
3089                                                      obj->base.size,
3090                                                      obj->tiling_mode, true);
3091         unfenced_alignment =
3092                 i915_gem_get_gtt_alignment(dev,
3093                                                     obj->base.size,
3094                                                     obj->tiling_mode, false);
3095
3096         if (alignment == 0)
3097                 alignment = map_and_fenceable ? fence_alignment :
3098                                                 unfenced_alignment;
3099         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3100                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
3101                 return -EINVAL;
3102         }
3103
3104         size = map_and_fenceable ? fence_size : obj->base.size;
3105
3106         /* If the object is bigger than the entire aperture, reject it early
3107          * before evicting everything in a vain attempt to find space.
3108          */
3109         if (obj->base.size > gtt_max) {
3110                 DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3111                           obj->base.size,
3112                           map_and_fenceable ? "mappable" : "total",
3113                           gtt_max);
3114                 return -E2BIG;
3115         }
3116
3117         ret = i915_gem_object_get_pages(obj);
3118         if (ret)
3119                 return ret;
3120
3121         i915_gem_object_pin_pages(obj);
3122
3123         vma = i915_gem_vma_create(obj, &dev_priv->gtt.base);
3124         if (IS_ERR(vma)) {
3125                 ret = PTR_ERR(vma);
3126                 goto err_unpin;
3127         }
3128
3129 search_free:
3130         ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
3131                                                   &vma->node,
3132                                                   size, alignment,
3133                                                   obj->cache_level, 0, gtt_max);
3134         if (ret) {
3135                 ret = i915_gem_evict_something(dev, size, alignment,
3136                                                obj->cache_level,
3137                                                map_and_fenceable,
3138                                                nonblocking);
3139                 if (ret == 0)
3140                         goto search_free;
3141
3142                 goto err_free_vma;
3143         }
3144         if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
3145                                               obj->cache_level))) {
3146                 ret = -EINVAL;
3147                 goto err_remove_node;
3148         }
3149
3150         ret = i915_gem_gtt_prepare_object(obj);
3151         if (ret)
3152                 goto err_remove_node;
3153
3154         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3155         list_add_tail(&obj->mm_list, &vm->inactive_list);
3156         list_add(&vma->vma_link, &obj->vma_list);
3157
3158         fenceable =
3159                 i915_gem_obj_ggtt_size(obj) == fence_size &&
3160                 (i915_gem_obj_ggtt_offset(obj) & (fence_alignment - 1)) == 0;
3161
3162         mappable = i915_gem_obj_ggtt_offset(obj) + obj->base.size <=
3163                 dev_priv->gtt.mappable_end;
3164
3165         obj->map_and_fenceable = mappable && fenceable;
3166
3167         trace_i915_gem_object_bind(obj, map_and_fenceable);
3168         i915_gem_verify_gtt(dev);
3169         return 0;
3170
3171 err_remove_node:
3172         drm_mm_remove_node(&vma->node);
3173 err_free_vma:
3174         i915_gem_vma_destroy(vma);
3175 err_unpin:
3176         i915_gem_object_unpin_pages(obj);
3177         return ret;
3178 }
3179
3180 void
3181 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
3182 {
3183         /* If we don't have a page list set up, then we're not pinned
3184          * to GPU, and we can ignore the cache flush because it'll happen
3185          * again at bind time.
3186          */
3187         if (obj->pages == NULL)
3188                 return;
3189
3190         /*
3191          * Stolen memory is always coherent with the GPU as it is explicitly
3192          * marked as wc by the system, or the system is cache-coherent.
3193          */
3194         if (obj->stolen)
3195                 return;
3196
3197         /* If the GPU is snooping the contents of the CPU cache,
3198          * we do not need to manually clear the CPU cache lines.  However,
3199          * the caches are only snooped when the render cache is
3200          * flushed/invalidated.  As we always have to emit invalidations
3201          * and flushes when moving into and out of the RENDER domain, correct
3202          * snooping behaviour occurs naturally as the result of our domain
3203          * tracking.
3204          */
3205         if (obj->cache_level != I915_CACHE_NONE)
3206                 return;
3207
3208         trace_i915_gem_object_clflush(obj);
3209
3210         drm_clflush_sg(obj->pages);
3211 }
3212
3213 /** Flushes the GTT write domain for the object if it's dirty. */
3214 static void
3215 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3216 {
3217         uint32_t old_write_domain;
3218
3219         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3220                 return;
3221
3222         /* No actual flushing is required for the GTT write domain.  Writes
3223          * to it immediately go to main memory as far as we know, so there's
3224          * no chipset flush.  It also doesn't land in render cache.
3225          *
3226          * However, we do have to enforce the order so that all writes through
3227          * the GTT land before any writes to the device, such as updates to
3228          * the GATT itself.
3229          */
3230         wmb();
3231
3232         old_write_domain = obj->base.write_domain;
3233         obj->base.write_domain = 0;
3234
3235         trace_i915_gem_object_change_domain(obj,
3236                                             obj->base.read_domains,
3237                                             old_write_domain);
3238 }
3239
3240 /** Flushes the CPU write domain for the object if it's dirty. */
3241 static void
3242 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3243 {
3244         uint32_t old_write_domain;
3245
3246         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3247                 return;
3248
3249         i915_gem_clflush_object(obj);
3250         i915_gem_chipset_flush(obj->base.dev);
3251         old_write_domain = obj->base.write_domain;
3252         obj->base.write_domain = 0;
3253
3254         trace_i915_gem_object_change_domain(obj,
3255                                             obj->base.read_domains,
3256                                             old_write_domain);
3257 }
3258
3259 /**
3260  * Moves a single object to the GTT read, and possibly write domain.
3261  *
3262  * This function returns when the move is complete, including waiting on
3263  * flushes to occur.
3264  */
3265 int
3266 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3267 {
3268         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3269         uint32_t old_write_domain, old_read_domains;
3270         int ret;
3271
3272         /* Not valid to be called on unbound objects. */
3273         if (!i915_gem_obj_ggtt_bound(obj))
3274                 return -EINVAL;
3275
3276         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3277                 return 0;
3278
3279         ret = i915_gem_object_wait_rendering(obj, !write);
3280         if (ret)
3281                 return ret;
3282
3283         i915_gem_object_flush_cpu_write_domain(obj);
3284
3285         /* Serialise direct access to this object with the barriers for
3286          * coherent writes from the GPU, by effectively invalidating the
3287          * GTT domain upon first access.
3288          */
3289         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3290                 mb();
3291
3292         old_write_domain = obj->base.write_domain;
3293         old_read_domains = obj->base.read_domains;
3294
3295         /* It should now be out of any other write domains, and we can update
3296          * the domain values for our changes.
3297          */
3298         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3299         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3300         if (write) {
3301                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3302                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3303                 obj->dirty = 1;
3304         }
3305
3306         trace_i915_gem_object_change_domain(obj,
3307                                             old_read_domains,
3308                                             old_write_domain);
3309
3310         /* And bump the LRU for this access */
3311         if (i915_gem_object_is_inactive(obj))
3312                 list_move_tail(&obj->mm_list,
3313                                &dev_priv->gtt.base.inactive_list);
3314
3315         return 0;
3316 }
3317
3318 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3319                                     enum i915_cache_level cache_level)
3320 {
3321         struct drm_device *dev = obj->base.dev;
3322         drm_i915_private_t *dev_priv = dev->dev_private;
3323         struct i915_vma *vma = __i915_gem_obj_to_vma(obj);
3324         int ret;
3325
3326         if (obj->cache_level == cache_level)
3327                 return 0;
3328
3329         if (obj->pin_count) {
3330                 DRM_DEBUG("can not change the cache level of pinned objects\n");
3331                 return -EBUSY;
3332         }
3333
3334         if (vma && !i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
3335                 ret = i915_gem_object_unbind(obj);
3336                 if (ret)
3337                         return ret;
3338         }
3339
3340         if (i915_gem_obj_ggtt_bound(obj)) {
3341                 ret = i915_gem_object_finish_gpu(obj);
3342                 if (ret)
3343                         return ret;
3344
3345                 i915_gem_object_finish_gtt(obj);
3346
3347                 /* Before SandyBridge, you could not use tiling or fence
3348                  * registers with snooped memory, so relinquish any fences
3349                  * currently pointing to our region in the aperture.
3350                  */
3351                 if (INTEL_INFO(dev)->gen < 6) {
3352                         ret = i915_gem_object_put_fence(obj);
3353                         if (ret)
3354                                 return ret;
3355                 }
3356
3357                 if (obj->has_global_gtt_mapping)
3358                         i915_gem_gtt_bind_object(obj, cache_level);
3359                 if (obj->has_aliasing_ppgtt_mapping)
3360                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3361                                                obj, cache_level);
3362
3363                 i915_gem_obj_ggtt_set_color(obj, cache_level);
3364         }
3365
3366         if (cache_level == I915_CACHE_NONE) {
3367                 u32 old_read_domains, old_write_domain;
3368
3369                 /* If we're coming from LLC cached, then we haven't
3370                  * actually been tracking whether the data is in the
3371                  * CPU cache or not, since we only allow one bit set
3372                  * in obj->write_domain and have been skipping the clflushes.
3373                  * Just set it to the CPU cache for now.
3374                  */
3375                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3376                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3377
3378                 old_read_domains = obj->base.read_domains;
3379                 old_write_domain = obj->base.write_domain;
3380
3381                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3382                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3383
3384                 trace_i915_gem_object_change_domain(obj,
3385                                                     old_read_domains,
3386                                                     old_write_domain);
3387         }
3388
3389         obj->cache_level = cache_level;
3390         i915_gem_verify_gtt(dev);
3391         return 0;
3392 }
3393
3394 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3395                                struct drm_file *file)
3396 {
3397         struct drm_i915_gem_caching *args = data;
3398         struct drm_i915_gem_object *obj;
3399         int ret;
3400
3401         ret = i915_mutex_lock_interruptible(dev);
3402         if (ret)
3403                 return ret;
3404
3405         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3406         if (&obj->base == NULL) {
3407                 ret = -ENOENT;
3408                 goto unlock;
3409         }
3410
3411         args->caching = obj->cache_level != I915_CACHE_NONE;
3412
3413         drm_gem_object_unreference(&obj->base);
3414 unlock:
3415         mutex_unlock(&dev->struct_mutex);
3416         return ret;
3417 }
3418
3419 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3420                                struct drm_file *file)
3421 {
3422         struct drm_i915_gem_caching *args = data;
3423         struct drm_i915_gem_object *obj;
3424         enum i915_cache_level level;
3425         int ret;
3426
3427         switch (args->caching) {
3428         case I915_CACHING_NONE:
3429                 level = I915_CACHE_NONE;
3430                 break;
3431         case I915_CACHING_CACHED:
3432                 level = I915_CACHE_LLC;
3433                 break;
3434         default:
3435                 return -EINVAL;
3436         }
3437
3438         ret = i915_mutex_lock_interruptible(dev);
3439         if (ret)
3440                 return ret;
3441
3442         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3443         if (&obj->base == NULL) {
3444                 ret = -ENOENT;
3445                 goto unlock;
3446         }
3447
3448         ret = i915_gem_object_set_cache_level(obj, level);
3449
3450         drm_gem_object_unreference(&obj->base);
3451 unlock:
3452         mutex_unlock(&dev->struct_mutex);
3453         return ret;
3454 }
3455
3456 /*
3457  * Prepare buffer for display plane (scanout, cursors, etc).
3458  * Can be called from an uninterruptible phase (modesetting) and allows
3459  * any flushes to be pipelined (for pageflips).
3460  */
3461 int
3462 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3463                                      u32 alignment,
3464                                      struct intel_ring_buffer *pipelined)
3465 {
3466         u32 old_read_domains, old_write_domain;
3467         int ret;
3468
3469         if (pipelined != obj->ring) {
3470                 ret = i915_gem_object_sync(obj, pipelined);
3471                 if (ret)
3472                         return ret;
3473         }
3474
3475         /* The display engine is not coherent with the LLC cache on gen6.  As
3476          * a result, we make sure that the pinning that is about to occur is
3477          * done with uncached PTEs. This is lowest common denominator for all
3478          * chipsets.
3479          *
3480          * However for gen6+, we could do better by using the GFDT bit instead
3481          * of uncaching, which would allow us to flush all the LLC-cached data
3482          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3483          */
3484         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3485         if (ret)
3486                 return ret;
3487
3488         /* As the user may map the buffer once pinned in the display plane
3489          * (e.g. libkms for the bootup splash), we have to ensure that we
3490          * always use map_and_fenceable for all scanout buffers.
3491          */
3492         ret = i915_gem_object_pin(obj, alignment, true, false);
3493         if (ret)
3494                 return ret;
3495
3496         i915_gem_object_flush_cpu_write_domain(obj);
3497
3498         old_write_domain = obj->base.write_domain;
3499         old_read_domains = obj->base.read_domains;
3500
3501         /* It should now be out of any other write domains, and we can update
3502          * the domain values for our changes.
3503          */
3504         obj->base.write_domain = 0;
3505         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3506
3507         trace_i915_gem_object_change_domain(obj,
3508                                             old_read_domains,
3509                                             old_write_domain);
3510
3511         return 0;
3512 }
3513
3514 int
3515 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3516 {
3517         int ret;
3518
3519         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3520                 return 0;
3521
3522         ret = i915_gem_object_wait_rendering(obj, false);
3523         if (ret)
3524                 return ret;
3525
3526         /* Ensure that we invalidate the GPU's caches and TLBs. */
3527         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3528         return 0;
3529 }
3530
3531 /**
3532  * Moves a single object to the CPU read, and possibly write domain.
3533  *
3534  * This function returns when the move is complete, including waiting on
3535  * flushes to occur.
3536  */
3537 int
3538 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3539 {
3540         uint32_t old_write_domain, old_read_domains;
3541         int ret;
3542
3543         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3544                 return 0;
3545
3546         ret = i915_gem_object_wait_rendering(obj, !write);
3547         if (ret)
3548                 return ret;
3549
3550         i915_gem_object_flush_gtt_write_domain(obj);
3551
3552         old_write_domain = obj->base.write_domain;
3553         old_read_domains = obj->base.read_domains;
3554
3555         /* Flush the CPU cache if it's still invalid. */
3556         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3557                 i915_gem_clflush_object(obj);
3558
3559                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3560         }
3561
3562         /* It should now be out of any other write domains, and we can update
3563          * the domain values for our changes.
3564          */
3565         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3566
3567         /* If we're writing through the CPU, then the GPU read domains will
3568          * need to be invalidated at next use.
3569          */
3570         if (write) {
3571                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3572                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3573         }
3574
3575         trace_i915_gem_object_change_domain(obj,
3576                                             old_read_domains,
3577                                             old_write_domain);
3578
3579         return 0;
3580 }
3581
3582 /* Throttle our rendering by waiting until the ring has completed our requests
3583  * emitted over 20 msec ago.
3584  *
3585  * Note that if we were to use the current jiffies each time around the loop,
3586  * we wouldn't escape the function with any frames outstanding if the time to
3587  * render a frame was over 20ms.
3588  *
3589  * This should get us reasonable parallelism between CPU and GPU but also
3590  * relatively low latency when blocking on a particular request to finish.
3591  */
3592 static int
3593 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3594 {
3595         struct drm_i915_private *dev_priv = dev->dev_private;
3596         struct drm_i915_file_private *file_priv = file->driver_priv;
3597         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3598         struct drm_i915_gem_request *request;
3599         struct intel_ring_buffer *ring = NULL;
3600         unsigned reset_counter;
3601         u32 seqno = 0;
3602         int ret;
3603
3604         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3605         if (ret)
3606                 return ret;
3607
3608         ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
3609         if (ret)
3610                 return ret;
3611
3612         spin_lock(&file_priv->mm.lock);
3613         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3614                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3615                         break;
3616
3617                 ring = request->ring;
3618                 seqno = request->seqno;
3619         }
3620         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3621         spin_unlock(&file_priv->mm.lock);
3622
3623         if (seqno == 0)
3624                 return 0;
3625
3626         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3627         if (ret == 0)
3628                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3629
3630         return ret;
3631 }
3632
3633 int
3634 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3635                     uint32_t alignment,
3636                     bool map_and_fenceable,
3637                     bool nonblocking)
3638 {
3639         int ret;
3640
3641         if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3642                 return -EBUSY;
3643
3644         if (i915_gem_obj_ggtt_bound(obj)) {
3645                 if ((alignment && i915_gem_obj_ggtt_offset(obj) & (alignment - 1)) ||
3646                     (map_and_fenceable && !obj->map_and_fenceable)) {
3647                         WARN(obj->pin_count,
3648                              "bo is already pinned with incorrect alignment:"
3649                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3650                              " obj->map_and_fenceable=%d\n",
3651                              i915_gem_obj_ggtt_offset(obj), alignment,
3652                              map_and_fenceable,
3653                              obj->map_and_fenceable);
3654                         ret = i915_gem_object_unbind(obj);
3655                         if (ret)
3656                                 return ret;
3657                 }
3658         }
3659
3660         if (!i915_gem_obj_ggtt_bound(obj)) {
3661                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3662
3663                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3664                                                   map_and_fenceable,
3665                                                   nonblocking);
3666                 if (ret)
3667                         return ret;
3668
3669                 if (!dev_priv->mm.aliasing_ppgtt)
3670                         i915_gem_gtt_bind_object(obj, obj->cache_level);
3671         }
3672
3673         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3674                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3675
3676         obj->pin_count++;
3677         obj->pin_mappable |= map_and_fenceable;
3678
3679         return 0;
3680 }
3681
3682 void
3683 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3684 {
3685         BUG_ON(obj->pin_count == 0);
3686         BUG_ON(!i915_gem_obj_ggtt_bound(obj));
3687
3688         if (--obj->pin_count == 0)
3689                 obj->pin_mappable = false;
3690 }
3691
3692 int
3693 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3694                    struct drm_file *file)
3695 {
3696         struct drm_i915_gem_pin *args = data;
3697         struct drm_i915_gem_object *obj;
3698         int ret;
3699
3700         ret = i915_mutex_lock_interruptible(dev);
3701         if (ret)
3702                 return ret;
3703
3704         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3705         if (&obj->base == NULL) {
3706                 ret = -ENOENT;
3707                 goto unlock;
3708         }
3709
3710         if (obj->madv != I915_MADV_WILLNEED) {
3711                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3712                 ret = -EINVAL;
3713                 goto out;
3714         }
3715
3716         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3717                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3718                           args->handle);
3719                 ret = -EINVAL;
3720                 goto out;
3721         }
3722
3723         if (obj->user_pin_count == 0) {
3724                 ret = i915_gem_object_pin(obj, args->alignment, true, false);
3725                 if (ret)
3726                         goto out;
3727         }
3728
3729         obj->user_pin_count++;
3730         obj->pin_filp = file;
3731
3732         /* XXX - flush the CPU caches for pinned objects
3733          * as the X server doesn't manage domains yet
3734          */
3735         i915_gem_object_flush_cpu_write_domain(obj);
3736         args->offset = i915_gem_obj_ggtt_offset(obj);
3737 out:
3738         drm_gem_object_unreference(&obj->base);
3739 unlock:
3740         mutex_unlock(&dev->struct_mutex);
3741         return ret;
3742 }
3743
3744 int
3745 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3746                      struct drm_file *file)
3747 {
3748         struct drm_i915_gem_pin *args = data;
3749         struct drm_i915_gem_object *obj;
3750         int ret;
3751
3752         ret = i915_mutex_lock_interruptible(dev);
3753         if (ret)
3754                 return ret;
3755
3756         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3757         if (&obj->base == NULL) {
3758                 ret = -ENOENT;
3759                 goto unlock;
3760         }
3761
3762         if (obj->pin_filp != file) {
3763                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3764                           args->handle);
3765                 ret = -EINVAL;
3766                 goto out;
3767         }
3768         obj->user_pin_count--;
3769         if (obj->user_pin_count == 0) {
3770                 obj->pin_filp = NULL;
3771                 i915_gem_object_unpin(obj);
3772         }
3773
3774 out:
3775         drm_gem_object_unreference(&obj->base);
3776 unlock:
3777         mutex_unlock(&dev->struct_mutex);
3778         return ret;
3779 }
3780
3781 int
3782 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3783                     struct drm_file *file)
3784 {
3785         struct drm_i915_gem_busy *args = data;
3786         struct drm_i915_gem_object *obj;
3787         int ret;
3788
3789         ret = i915_mutex_lock_interruptible(dev);
3790         if (ret)
3791                 return ret;
3792
3793         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3794         if (&obj->base == NULL) {
3795                 ret = -ENOENT;
3796                 goto unlock;
3797         }
3798
3799         /* Count all active objects as busy, even if they are currently not used
3800          * by the gpu. Users of this interface expect objects to eventually
3801          * become non-busy without any further actions, therefore emit any
3802          * necessary flushes here.
3803          */
3804         ret = i915_gem_object_flush_active(obj);
3805
3806         args->busy = obj->active;
3807         if (obj->ring) {
3808                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
3809                 args->busy |= intel_ring_flag(obj->ring) << 16;
3810         }
3811
3812         drm_gem_object_unreference(&obj->base);
3813 unlock:
3814         mutex_unlock(&dev->struct_mutex);
3815         return ret;
3816 }
3817
3818 int
3819 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3820                         struct drm_file *file_priv)
3821 {
3822         return i915_gem_ring_throttle(dev, file_priv);
3823 }
3824
3825 int
3826 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3827                        struct drm_file *file_priv)
3828 {
3829         struct drm_i915_gem_madvise *args = data;
3830         struct drm_i915_gem_object *obj;
3831         int ret;
3832
3833         switch (args->madv) {
3834         case I915_MADV_DONTNEED:
3835         case I915_MADV_WILLNEED:
3836             break;
3837         default:
3838             return -EINVAL;
3839         }
3840
3841         ret = i915_mutex_lock_interruptible(dev);
3842         if (ret)
3843                 return ret;
3844
3845         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3846         if (&obj->base == NULL) {
3847                 ret = -ENOENT;
3848                 goto unlock;
3849         }
3850
3851         if (obj->pin_count) {
3852                 ret = -EINVAL;
3853                 goto out;
3854         }
3855
3856         if (obj->madv != __I915_MADV_PURGED)
3857                 obj->madv = args->madv;
3858
3859         /* if the object is no longer attached, discard its backing storage */
3860         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3861                 i915_gem_object_truncate(obj);
3862
3863         args->retained = obj->madv != __I915_MADV_PURGED;
3864
3865 out:
3866         drm_gem_object_unreference(&obj->base);
3867 unlock:
3868         mutex_unlock(&dev->struct_mutex);
3869         return ret;
3870 }
3871
3872 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3873                           const struct drm_i915_gem_object_ops *ops)
3874 {
3875         INIT_LIST_HEAD(&obj->mm_list);
3876         INIT_LIST_HEAD(&obj->global_list);
3877         INIT_LIST_HEAD(&obj->ring_list);
3878         INIT_LIST_HEAD(&obj->exec_list);
3879         INIT_LIST_HEAD(&obj->vma_list);
3880
3881         obj->ops = ops;
3882
3883         obj->fence_reg = I915_FENCE_REG_NONE;
3884         obj->madv = I915_MADV_WILLNEED;
3885         /* Avoid an unnecessary call to unbind on the first bind. */
3886         obj->map_and_fenceable = true;
3887
3888         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
3889 }
3890
3891 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3892         .get_pages = i915_gem_object_get_pages_gtt,
3893         .put_pages = i915_gem_object_put_pages_gtt,
3894 };
3895
3896 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3897                                                   size_t size)
3898 {
3899         struct drm_i915_gem_object *obj;
3900         struct address_space *mapping;
3901         gfp_t mask;
3902
3903         obj = i915_gem_object_alloc(dev);
3904         if (obj == NULL)
3905                 return NULL;
3906
3907         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3908                 i915_gem_object_free(obj);
3909                 return NULL;
3910         }
3911
3912         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3913         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3914                 /* 965gm cannot relocate objects above 4GiB. */
3915                 mask &= ~__GFP_HIGHMEM;
3916                 mask |= __GFP_DMA32;
3917         }
3918
3919         mapping = file_inode(obj->base.filp)->i_mapping;
3920         mapping_set_gfp_mask(mapping, mask);
3921
3922         i915_gem_object_init(obj, &i915_gem_object_ops);
3923
3924         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3925         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3926
3927         if (HAS_LLC(dev)) {
3928                 /* On some devices, we can have the GPU use the LLC (the CPU
3929                  * cache) for about a 10% performance improvement
3930                  * compared to uncached.  Graphics requests other than
3931                  * display scanout are coherent with the CPU in
3932                  * accessing this cache.  This means in this mode we
3933                  * don't need to clflush on the CPU side, and on the
3934                  * GPU side we only need to flush internal caches to
3935                  * get data visible to the CPU.
3936                  *
3937                  * However, we maintain the display planes as UC, and so
3938                  * need to rebind when first used as such.
3939                  */
3940                 obj->cache_level = I915_CACHE_LLC;
3941         } else
3942                 obj->cache_level = I915_CACHE_NONE;
3943
3944         trace_i915_gem_object_create(obj);
3945
3946         return obj;
3947 }
3948
3949 int i915_gem_init_object(struct drm_gem_object *obj)
3950 {
3951         BUG();
3952
3953         return 0;
3954 }
3955
3956 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3957 {
3958         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3959         struct drm_device *dev = obj->base.dev;
3960         drm_i915_private_t *dev_priv = dev->dev_private;
3961
3962         trace_i915_gem_object_destroy(obj);
3963
3964         if (obj->phys_obj)
3965                 i915_gem_detach_phys_object(dev, obj);
3966
3967         obj->pin_count = 0;
3968         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3969                 bool was_interruptible;
3970
3971                 was_interruptible = dev_priv->mm.interruptible;
3972                 dev_priv->mm.interruptible = false;
3973
3974                 WARN_ON(i915_gem_object_unbind(obj));
3975
3976                 dev_priv->mm.interruptible = was_interruptible;
3977         }
3978
3979         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
3980          * before progressing. */
3981         if (obj->stolen)
3982                 i915_gem_object_unpin_pages(obj);
3983
3984         if (WARN_ON(obj->pages_pin_count))
3985                 obj->pages_pin_count = 0;
3986         i915_gem_object_put_pages(obj);
3987         i915_gem_object_free_mmap_offset(obj);
3988         i915_gem_object_release_stolen(obj);
3989
3990         BUG_ON(obj->pages);
3991
3992         if (obj->base.import_attach)
3993                 drm_prime_gem_destroy(&obj->base, NULL);
3994
3995         drm_gem_object_release(&obj->base);
3996         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3997
3998         kfree(obj->bit_17);
3999         i915_gem_object_free(obj);
4000 }
4001
4002 struct i915_vma *i915_gem_vma_create(struct drm_i915_gem_object *obj,
4003                                      struct i915_address_space *vm)
4004 {
4005         struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
4006         if (vma == NULL)
4007                 return ERR_PTR(-ENOMEM);
4008
4009         INIT_LIST_HEAD(&vma->vma_link);
4010         vma->vm = vm;
4011         vma->obj = obj;
4012
4013         return vma;
4014 }
4015
4016 void i915_gem_vma_destroy(struct i915_vma *vma)
4017 {
4018         WARN_ON(vma->node.allocated);
4019         kfree(vma);
4020 }
4021
4022 int
4023 i915_gem_idle(struct drm_device *dev)
4024 {
4025         drm_i915_private_t *dev_priv = dev->dev_private;
4026         int ret;
4027
4028         if (dev_priv->ums.mm_suspended) {
4029                 mutex_unlock(&dev->struct_mutex);
4030                 return 0;
4031         }
4032
4033         ret = i915_gpu_idle(dev);
4034         if (ret) {
4035                 mutex_unlock(&dev->struct_mutex);
4036                 return ret;
4037         }
4038         i915_gem_retire_requests(dev);
4039
4040         /* Under UMS, be paranoid and evict. */
4041         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4042                 i915_gem_evict_everything(dev);
4043
4044         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4045
4046         i915_kernel_lost_context(dev);
4047         i915_gem_cleanup_ringbuffer(dev);
4048
4049         /* Cancel the retire work handler, which should be idle now. */
4050         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4051
4052         return 0;
4053 }
4054
4055 void i915_gem_l3_remap(struct drm_device *dev)
4056 {
4057         drm_i915_private_t *dev_priv = dev->dev_private;
4058         u32 misccpctl;
4059         int i;
4060
4061         if (!HAS_L3_GPU_CACHE(dev))
4062                 return;
4063
4064         if (!dev_priv->l3_parity.remap_info)
4065                 return;
4066
4067         misccpctl = I915_READ(GEN7_MISCCPCTL);
4068         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
4069         POSTING_READ(GEN7_MISCCPCTL);
4070
4071         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4072                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
4073                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
4074                         DRM_DEBUG("0x%x was already programmed to %x\n",
4075                                   GEN7_L3LOG_BASE + i, remap);
4076                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
4077                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
4078                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
4079         }
4080
4081         /* Make sure all the writes land before disabling dop clock gating */
4082         POSTING_READ(GEN7_L3LOG_BASE);
4083
4084         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
4085 }
4086
4087 void i915_gem_init_swizzling(struct drm_device *dev)
4088 {
4089         drm_i915_private_t *dev_priv = dev->dev_private;
4090
4091         if (INTEL_INFO(dev)->gen < 5 ||
4092             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4093                 return;
4094
4095         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4096                                  DISP_TILE_SURFACE_SWIZZLING);
4097
4098         if (IS_GEN5(dev))
4099                 return;
4100
4101         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4102         if (IS_GEN6(dev))
4103                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4104         else if (IS_GEN7(dev))
4105                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4106         else
4107                 BUG();
4108 }
4109
4110 static bool
4111 intel_enable_blt(struct drm_device *dev)
4112 {
4113         if (!HAS_BLT(dev))
4114                 return false;
4115
4116         /* The blitter was dysfunctional on early prototypes */
4117         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
4118                 DRM_INFO("BLT not supported on this pre-production hardware;"
4119                          " graphics performance will be degraded.\n");
4120                 return false;
4121         }
4122
4123         return true;
4124 }
4125
4126 static int i915_gem_init_rings(struct drm_device *dev)
4127 {
4128         struct drm_i915_private *dev_priv = dev->dev_private;
4129         int ret;
4130
4131         ret = intel_init_render_ring_buffer(dev);
4132         if (ret)
4133                 return ret;
4134
4135         if (HAS_BSD(dev)) {
4136                 ret = intel_init_bsd_ring_buffer(dev);
4137                 if (ret)
4138                         goto cleanup_render_ring;
4139         }
4140
4141         if (intel_enable_blt(dev)) {
4142                 ret = intel_init_blt_ring_buffer(dev);
4143                 if (ret)
4144                         goto cleanup_bsd_ring;
4145         }
4146
4147         if (HAS_VEBOX(dev)) {
4148                 ret = intel_init_vebox_ring_buffer(dev);
4149                 if (ret)
4150                         goto cleanup_blt_ring;
4151         }
4152
4153
4154         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4155         if (ret)
4156                 goto cleanup_vebox_ring;
4157
4158         return 0;
4159
4160 cleanup_vebox_ring:
4161         intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4162 cleanup_blt_ring:
4163         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4164 cleanup_bsd_ring:
4165         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4166 cleanup_render_ring:
4167         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4168
4169         return ret;
4170 }
4171
4172 int
4173 i915_gem_init_hw(struct drm_device *dev)
4174 {
4175         drm_i915_private_t *dev_priv = dev->dev_private;
4176         int ret;
4177
4178         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4179                 return -EIO;
4180
4181         if (dev_priv->ellc_size)
4182                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4183
4184         if (HAS_PCH_NOP(dev)) {
4185                 u32 temp = I915_READ(GEN7_MSG_CTL);
4186                 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4187                 I915_WRITE(GEN7_MSG_CTL, temp);
4188         }
4189
4190         i915_gem_l3_remap(dev);
4191
4192         i915_gem_init_swizzling(dev);
4193
4194         ret = i915_gem_init_rings(dev);
4195         if (ret)
4196                 return ret;
4197
4198         /*
4199          * XXX: There was some w/a described somewhere suggesting loading
4200          * contexts before PPGTT.
4201          */
4202         i915_gem_context_init(dev);
4203         if (dev_priv->mm.aliasing_ppgtt) {
4204                 ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
4205                 if (ret) {
4206                         i915_gem_cleanup_aliasing_ppgtt(dev);
4207                         DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
4208                 }
4209         }
4210
4211         return 0;
4212 }
4213
4214 int i915_gem_init(struct drm_device *dev)
4215 {
4216         struct drm_i915_private *dev_priv = dev->dev_private;
4217         int ret;
4218
4219         mutex_lock(&dev->struct_mutex);
4220
4221         if (IS_VALLEYVIEW(dev)) {
4222                 /* VLVA0 (potential hack), BIOS isn't actually waking us */
4223                 I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
4224                 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
4225                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4226         }
4227
4228         i915_gem_init_global_gtt(dev);
4229
4230         ret = i915_gem_init_hw(dev);
4231         mutex_unlock(&dev->struct_mutex);
4232         if (ret) {
4233                 i915_gem_cleanup_aliasing_ppgtt(dev);
4234                 return ret;
4235         }
4236
4237         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4238         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4239                 dev_priv->dri1.allow_batchbuffer = 1;
4240         return 0;
4241 }
4242
4243 void
4244 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4245 {
4246         drm_i915_private_t *dev_priv = dev->dev_private;
4247         struct intel_ring_buffer *ring;
4248         int i;
4249
4250         for_each_ring(ring, dev_priv, i)
4251                 intel_cleanup_ring_buffer(ring);
4252 }
4253
4254 int
4255 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4256                        struct drm_file *file_priv)
4257 {
4258         struct drm_i915_private *dev_priv = dev->dev_private;
4259         int ret;
4260
4261         if (drm_core_check_feature(dev, DRIVER_MODESET))
4262                 return 0;
4263
4264         if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4265                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4266                 atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4267         }
4268
4269         mutex_lock(&dev->struct_mutex);
4270         dev_priv->ums.mm_suspended = 0;
4271
4272         ret = i915_gem_init_hw(dev);
4273         if (ret != 0) {
4274                 mutex_unlock(&dev->struct_mutex);
4275                 return ret;
4276         }
4277
4278         BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
4279         mutex_unlock(&dev->struct_mutex);
4280
4281         ret = drm_irq_install(dev);
4282         if (ret)
4283                 goto cleanup_ringbuffer;
4284
4285         return 0;
4286
4287 cleanup_ringbuffer:
4288         mutex_lock(&dev->struct_mutex);
4289         i915_gem_cleanup_ringbuffer(dev);
4290         dev_priv->ums.mm_suspended = 1;
4291         mutex_unlock(&dev->struct_mutex);
4292
4293         return ret;
4294 }
4295
4296 int
4297 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4298                        struct drm_file *file_priv)
4299 {
4300         struct drm_i915_private *dev_priv = dev->dev_private;
4301         int ret;
4302
4303         if (drm_core_check_feature(dev, DRIVER_MODESET))
4304                 return 0;
4305
4306         drm_irq_uninstall(dev);
4307
4308         mutex_lock(&dev->struct_mutex);
4309         ret =  i915_gem_idle(dev);
4310
4311         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4312          * We need to replace this with a semaphore, or something.
4313          * And not confound ums.mm_suspended!
4314          */
4315         if (ret != 0)
4316                 dev_priv->ums.mm_suspended = 1;
4317         mutex_unlock(&dev->struct_mutex);
4318
4319         return ret;
4320 }
4321
4322 void
4323 i915_gem_lastclose(struct drm_device *dev)
4324 {
4325         int ret;
4326
4327         if (drm_core_check_feature(dev, DRIVER_MODESET))
4328                 return;
4329
4330         mutex_lock(&dev->struct_mutex);
4331         ret = i915_gem_idle(dev);
4332         if (ret)
4333                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4334         mutex_unlock(&dev->struct_mutex);
4335 }
4336
4337 static void
4338 init_ring_lists(struct intel_ring_buffer *ring)
4339 {
4340         INIT_LIST_HEAD(&ring->active_list);
4341         INIT_LIST_HEAD(&ring->request_list);
4342 }
4343
4344 void
4345 i915_gem_load(struct drm_device *dev)
4346 {
4347         drm_i915_private_t *dev_priv = dev->dev_private;
4348         int i;
4349
4350         dev_priv->slab =
4351                 kmem_cache_create("i915_gem_object",
4352                                   sizeof(struct drm_i915_gem_object), 0,
4353                                   SLAB_HWCACHE_ALIGN,
4354                                   NULL);
4355
4356         INIT_LIST_HEAD(&dev_priv->gtt.base.active_list);
4357         INIT_LIST_HEAD(&dev_priv->gtt.base.inactive_list);
4358         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4359         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4360         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4361         for (i = 0; i < I915_NUM_RINGS; i++)
4362                 init_ring_lists(&dev_priv->ring[i]);
4363         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4364                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4365         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4366                           i915_gem_retire_work_handler);
4367         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4368
4369         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4370         if (IS_GEN3(dev)) {
4371                 I915_WRITE(MI_ARB_STATE,
4372                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4373         }
4374
4375         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4376
4377         /* Old X drivers will take 0-2 for front, back, depth buffers */
4378         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4379                 dev_priv->fence_reg_start = 3;
4380
4381         if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
4382                 dev_priv->num_fence_regs = 32;
4383         else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4384                 dev_priv->num_fence_regs = 16;
4385         else
4386                 dev_priv->num_fence_regs = 8;
4387
4388         /* Initialize fence registers to zero */
4389         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4390         i915_gem_restore_fences(dev);
4391
4392         i915_gem_detect_bit_6_swizzle(dev);
4393         init_waitqueue_head(&dev_priv->pending_flip_queue);
4394
4395         dev_priv->mm.interruptible = true;
4396
4397         dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan;
4398         dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count;
4399         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4400         register_shrinker(&dev_priv->mm.inactive_shrinker);
4401 }
4402
4403 /*
4404  * Create a physically contiguous memory object for this object
4405  * e.g. for cursor + overlay regs
4406  */
4407 static int i915_gem_init_phys_object(struct drm_device *dev,
4408                                      int id, int size, int align)
4409 {
4410         drm_i915_private_t *dev_priv = dev->dev_private;
4411         struct drm_i915_gem_phys_object *phys_obj;
4412         int ret;
4413
4414         if (dev_priv->mm.phys_objs[id - 1] || !size)
4415                 return 0;
4416
4417         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4418         if (!phys_obj)
4419                 return -ENOMEM;
4420
4421         phys_obj->id = id;
4422
4423         phys_obj->handle = drm_pci_alloc(dev, size, align);
4424         if (!phys_obj->handle) {
4425                 ret = -ENOMEM;
4426                 goto kfree_obj;
4427         }
4428 #ifdef CONFIG_X86
4429         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4430 #endif
4431
4432         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4433
4434         return 0;
4435 kfree_obj:
4436         kfree(phys_obj);
4437         return ret;
4438 }
4439
4440 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4441 {
4442         drm_i915_private_t *dev_priv = dev->dev_private;
4443         struct drm_i915_gem_phys_object *phys_obj;
4444
4445         if (!dev_priv->mm.phys_objs[id - 1])
4446                 return;
4447
4448         phys_obj = dev_priv->mm.phys_objs[id - 1];
4449         if (phys_obj->cur_obj) {
4450                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4451         }
4452
4453 #ifdef CONFIG_X86
4454         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4455 #endif
4456         drm_pci_free(dev, phys_obj->handle);
4457         kfree(phys_obj);
4458         dev_priv->mm.phys_objs[id - 1] = NULL;
4459 }
4460
4461 void i915_gem_free_all_phys_object(struct drm_device *dev)
4462 {
4463         int i;
4464
4465         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4466                 i915_gem_free_phys_object(dev, i);
4467 }
4468
4469 void i915_gem_detach_phys_object(struct drm_device *dev,
4470                                  struct drm_i915_gem_object *obj)
4471 {
4472         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4473         char *vaddr;
4474         int i;
4475         int page_count;
4476
4477         if (!obj->phys_obj)
4478                 return;
4479         vaddr = obj->phys_obj->handle->vaddr;
4480
4481         page_count = obj->base.size / PAGE_SIZE;
4482         for (i = 0; i < page_count; i++) {
4483                 struct page *page = shmem_read_mapping_page(mapping, i);
4484                 if (!IS_ERR(page)) {
4485                         char *dst = kmap_atomic(page);
4486                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4487                         kunmap_atomic(dst);
4488
4489                         drm_clflush_pages(&page, 1);
4490
4491                         set_page_dirty(page);
4492                         mark_page_accessed(page);
4493                         page_cache_release(page);
4494                 }
4495         }
4496         i915_gem_chipset_flush(dev);
4497
4498         obj->phys_obj->cur_obj = NULL;
4499         obj->phys_obj = NULL;
4500 }
4501
4502 int
4503 i915_gem_attach_phys_object(struct drm_device *dev,
4504                             struct drm_i915_gem_object *obj,
4505                             int id,
4506                             int align)
4507 {
4508         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4509         drm_i915_private_t *dev_priv = dev->dev_private;
4510         int ret = 0;
4511         int page_count;
4512         int i;
4513
4514         if (id > I915_MAX_PHYS_OBJECT)
4515                 return -EINVAL;
4516
4517         if (obj->phys_obj) {
4518                 if (obj->phys_obj->id == id)
4519                         return 0;
4520                 i915_gem_detach_phys_object(dev, obj);
4521         }
4522
4523         /* create a new object */
4524         if (!dev_priv->mm.phys_objs[id - 1]) {
4525                 ret = i915_gem_init_phys_object(dev, id,
4526                                                 obj->base.size, align);
4527                 if (ret) {
4528                         DRM_ERROR("failed to init phys object %d size: %zu\n",
4529                                   id, obj->base.size);
4530                         return ret;
4531                 }
4532         }
4533
4534         /* bind to the object */
4535         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4536         obj->phys_obj->cur_obj = obj;
4537
4538         page_count = obj->base.size / PAGE_SIZE;
4539
4540         for (i = 0; i < page_count; i++) {
4541                 struct page *page;
4542                 char *dst, *src;
4543
4544                 page = shmem_read_mapping_page(mapping, i);
4545                 if (IS_ERR(page))
4546                         return PTR_ERR(page);
4547
4548                 src = kmap_atomic(page);
4549                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4550                 memcpy(dst, src, PAGE_SIZE);
4551                 kunmap_atomic(src);
4552
4553                 mark_page_accessed(page);
4554                 page_cache_release(page);
4555         }
4556
4557         return 0;
4558 }
4559
4560 static int
4561 i915_gem_phys_pwrite(struct drm_device *dev,
4562                      struct drm_i915_gem_object *obj,
4563                      struct drm_i915_gem_pwrite *args,
4564                      struct drm_file *file_priv)
4565 {
4566         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4567         char __user *user_data = to_user_ptr(args->data_ptr);
4568
4569         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4570                 unsigned long unwritten;
4571
4572                 /* The physical object once assigned is fixed for the lifetime
4573                  * of the obj, so we can safely drop the lock and continue
4574                  * to access vaddr.
4575                  */
4576                 mutex_unlock(&dev->struct_mutex);
4577                 unwritten = copy_from_user(vaddr, user_data, args->size);
4578                 mutex_lock(&dev->struct_mutex);
4579                 if (unwritten)
4580                         return -EFAULT;
4581         }
4582
4583         i915_gem_chipset_flush(dev);
4584         return 0;
4585 }
4586
4587 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4588 {
4589         struct drm_i915_file_private *file_priv = file->driver_priv;
4590
4591         /* Clean up our request list when the client is going away, so that
4592          * later retire_requests won't dereference our soon-to-be-gone
4593          * file_priv.
4594          */
4595         spin_lock(&file_priv->mm.lock);
4596         while (!list_empty(&file_priv->mm.request_list)) {
4597                 struct drm_i915_gem_request *request;
4598
4599                 request = list_first_entry(&file_priv->mm.request_list,
4600                                            struct drm_i915_gem_request,
4601                                            client_list);
4602                 list_del(&request->client_list);
4603                 request->file_priv = NULL;
4604         }
4605         spin_unlock(&file_priv->mm.lock);
4606 }
4607
4608 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
4609 {
4610         if (!mutex_is_locked(mutex))
4611                 return false;
4612
4613 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
4614         return mutex->owner == task;
4615 #else
4616         /* Since UP may be pre-empted, we cannot assume that we own the lock */
4617         return false;
4618 #endif
4619 }
4620
4621 static unsigned long
4622 i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
4623 {
4624         struct drm_i915_private *dev_priv =
4625                 container_of(shrinker,
4626                              struct drm_i915_private,
4627                              mm.inactive_shrinker);
4628         struct drm_device *dev = dev_priv->dev;
4629         struct i915_address_space *vm = &dev_priv->gtt.base;
4630         struct drm_i915_gem_object *obj;
4631         bool unlock = true;
4632         unsigned long count;
4633
4634         if (!mutex_trylock(&dev->struct_mutex)) {
4635                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
4636                         return 0;
4637
4638                 if (dev_priv->mm.shrinker_no_lock_stealing)
4639                         return 0;
4640
4641                 unlock = false;
4642         }
4643
4644         count = 0;
4645         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4646                 if (obj->pages_pin_count == 0)
4647                         count += obj->base.size >> PAGE_SHIFT;
4648         list_for_each_entry(obj, &vm->inactive_list, mm_list)
4649                 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
4650                         count += obj->base.size >> PAGE_SHIFT;
4651
4652         if (unlock)
4653                 mutex_unlock(&dev->struct_mutex);
4654         return count;
4655 }
4656
4657 static unsigned long
4658 i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc)
4659 {
4660         struct drm_i915_private *dev_priv =
4661                 container_of(shrinker,
4662                              struct drm_i915_private,
4663                              mm.inactive_shrinker);
4664         struct drm_device *dev = dev_priv->dev;
4665         int nr_to_scan = sc->nr_to_scan;
4666         unsigned long freed;
4667         bool unlock = true;
4668
4669         if (!mutex_trylock(&dev->struct_mutex)) {
4670                 if (!mutex_is_locked_by(&dev->struct_mutex, current))
4671                         return 0;
4672
4673                 if (dev_priv->mm.shrinker_no_lock_stealing)
4674                         return 0;
4675
4676                 unlock = false;
4677         }
4678
4679         freed = i915_gem_purge(dev_priv, nr_to_scan);
4680         if (freed < nr_to_scan)
4681                 freed += __i915_gem_shrink(dev_priv, nr_to_scan,
4682                                                         false);
4683         if (freed < nr_to_scan)
4684                 freed += i915_gem_shrink_all(dev_priv);
4685
4686         if (unlock)
4687                 mutex_unlock(&dev->struct_mutex);
4688         return freed;
4689 }