drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40
  41 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  42                                                   bool pipelined);
  43 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  44 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  45 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  46                                              int write);
  47 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  48                                                      uint64_t offset,
  49                                                      uint64_t size);
  50 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  51 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  52                                           bool interruptible);
  53 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  54                                            unsigned alignment);
  55 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  56 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  57                                 struct drm_i915_gem_pwrite *args,
  58                                 struct drm_file *file_priv);
  59 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  60
  61 static int
  62 i915_gem_object_get_pages(struct drm_gem_object *obj,
  63                           gfp_t gfpmask);
  64
  65 static void
  66 i915_gem_object_put_pages(struct drm_gem_object *obj);
  67
  68 static LIST_HEAD(shrink_list);
  69 static DEFINE_SPINLOCK(shrink_list_lock);
  70
  71 /* some bookkeeping */
  72 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  73                                   size_t size)
  74 {
  75         dev_priv->mm.object_count++;
  76         dev_priv->mm.object_memory += size;
  77 }
  78
  79 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  80                                      size_t size)
  81 {
  82         dev_priv->mm.object_count--;
  83         dev_priv->mm.object_memory -= size;
  84 }
  85
  86 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  87                                   size_t size)
  88 {
  89         dev_priv->mm.gtt_count++;
  90         dev_priv->mm.gtt_memory += size;
  91 }
  92
  93 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
  94                                      size_t size)
  95 {
  96         dev_priv->mm.gtt_count--;
  97         dev_priv->mm.gtt_memory -= size;
  98 }
  99
 100 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 101                                   size_t size)
 102 {
 103         dev_priv->mm.pin_count++;
 104         dev_priv->mm.pin_memory += size;
 105 }
 106
 107 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 108                                      size_t size)
 109 {
 110         dev_priv->mm.pin_count--;
 111         dev_priv->mm.pin_memory -= size;
 112 }
 113
 114 int
 115 i915_gem_check_is_wedged(struct drm_device *dev)
 116 {
 117         struct drm_i915_private *dev_priv = dev->dev_private;
 118         struct completion *x = &dev_priv->error_completion;
 119         unsigned long flags;
 120         int ret;
 121
 122         if (!atomic_read(&dev_priv->mm.wedged))
 123                 return 0;
 124
 125         ret = wait_for_completion_interruptible(x);
 126         if (ret)
 127                 return ret;
 128
 129         /* Success, we reset the GPU! */
 130         if (!atomic_read(&dev_priv->mm.wedged))
 131                 return 0;
 132
 133         /* GPU is hung, bump the completion count to account for
 134          * the token we just consumed so that we never hit zero and
 135          * end up waiting upon a subsequent completion event that
 136          * will never happen.
 137          */
 138         spin_lock_irqsave(&x->wait.lock, flags);
 139         x->done++;
 140         spin_unlock_irqrestore(&x->wait.lock, flags);
 141         return -EIO;
 142 }
 143
 144 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 145 {
 146         struct drm_i915_private *dev_priv = dev->dev_private;
 147         int ret;
 148
 149         ret = i915_gem_check_is_wedged(dev);
 150         if (ret)
 151                 return ret;
 152
 153         ret = mutex_lock_interruptible(&dev->struct_mutex);
 154         if (ret)
 155                 return ret;
 156
 157         if (atomic_read(&dev_priv->mm.wedged)) {
 158                 mutex_unlock(&dev->struct_mutex);
 159                 return -EAGAIN;
 160         }
 161
 162         WARN_ON(i915_verify_lists(dev));
 163         return 0;
 164 }
 165
 166 static inline bool
 167 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 168 {
 169         return obj_priv->gtt_space &&
 170                 !obj_priv->active &&
 171                 obj_priv->pin_count == 0;
 172 }
 173
 174 int i915_gem_do_init(struct drm_device *dev,
 175                      unsigned long start,
 176                      unsigned long end)
 177 {
 178         drm_i915_private_t *dev_priv = dev->dev_private;
 179
 180         if (start >= end ||
 181             (start & (PAGE_SIZE - 1)) != 0 ||
 182             (end & (PAGE_SIZE - 1)) != 0) {
 183                 return -EINVAL;
 184         }
 185
 186         drm_mm_init(&dev_priv->mm.gtt_space, start,
 187                     end - start);
 188
 189         dev_priv->mm.gtt_total = end - start;
 190
 191         return 0;
 192 }
 193
 194 int
 195 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 196                     struct drm_file *file_priv)
 197 {
 198         struct drm_i915_gem_init *args = data;
 199         int ret;
 200
 201         mutex_lock(&dev->struct_mutex);
 202         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
 203         mutex_unlock(&dev->struct_mutex);
 204
 205         return ret;
 206 }
 207
 208 int
 209 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 210                             struct drm_file *file_priv)
 211 {
 212         struct drm_i915_private *dev_priv = dev->dev_private;
 213         struct drm_i915_gem_get_aperture *args = data;
 214
 215         if (!(dev->driver->driver_features & DRIVER_GEM))
 216                 return -ENODEV;
 217
 218         mutex_lock(&dev->struct_mutex);
 219         args->aper_size = dev_priv->mm.gtt_total;
 220         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 221         mutex_unlock(&dev->struct_mutex);
 222
 223         return 0;
 224 }
 225
 226
 227 /**
 228  * Creates a new mm object and returns a handle to it.
 229  */
 230 int
 231 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 232                       struct drm_file *file_priv)
 233 {
 234         struct drm_i915_gem_create *args = data;
 235         struct drm_gem_object *obj;
 236         int ret;
 237         u32 handle;
 238
 239         args->size = roundup(args->size, PAGE_SIZE);
 240
 241         /* Allocate the new object */
 242         obj = i915_gem_alloc_object(dev, args->size);
 243         if (obj == NULL)
 244                 return -ENOMEM;
 245
 246         ret = drm_gem_handle_create(file_priv, obj, &handle);
 247         /* drop reference from allocate - handle holds it now */
 248         drm_gem_object_unreference_unlocked(obj);
 249         if (ret) {
 250                 return ret;
 251         }
 252
 253         args->handle = handle;
 254         return 0;
 255 }
 256
 257 static inline int
 258 fast_shmem_read(struct page **pages,
 259                 loff_t page_base, int page_offset,
 260                 char __user *data,
 261                 int length)
 262 {
 263         char __iomem *vaddr;
 264         int unwritten;
 265
 266         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 267         if (vaddr == NULL)
 268                 return -ENOMEM;
 269         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 270         kunmap_atomic(vaddr, KM_USER0);
 271
 272         if (unwritten)
 273                 return -EFAULT;
 274
 275         return 0;
 276 }
 277
 278 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 279 {
 280         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 281         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 282
 283         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 284                 obj_priv->tiling_mode != I915_TILING_NONE;
 285 }
 286
 287 static inline void
 288 slow_shmem_copy(struct page *dst_page,
 289                 int dst_offset,
 290                 struct page *src_page,
 291                 int src_offset,
 292                 int length)
 293 {
 294         char *dst_vaddr, *src_vaddr;
 295
 296         dst_vaddr = kmap(dst_page);
 297         src_vaddr = kmap(src_page);
 298
 299         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 300
 301         kunmap(src_page);
 302         kunmap(dst_page);
 303 }
 304
 305 static inline void
 306 slow_shmem_bit17_copy(struct page *gpu_page,
 307                       int gpu_offset,
 308                       struct page *cpu_page,
 309                       int cpu_offset,
 310                       int length,
 311                       int is_read)
 312 {
 313         char *gpu_vaddr, *cpu_vaddr;
 314
 315         /* Use the unswizzled path if this page isn't affected. */
 316         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 317                 if (is_read)
 318                         return slow_shmem_copy(cpu_page, cpu_offset,
 319                                                gpu_page, gpu_offset, length);
 320                 else
 321                         return slow_shmem_copy(gpu_page, gpu_offset,
 322                                                cpu_page, cpu_offset, length);
 323         }
 324
 325         gpu_vaddr = kmap(gpu_page);
 326         cpu_vaddr = kmap(cpu_page);
 327
 328         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 329          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 330          */
 331         while (length > 0) {
 332                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 333                 int this_length = min(cacheline_end - gpu_offset, length);
 334                 int swizzled_gpu_offset = gpu_offset ^ 64;
 335
 336                 if (is_read) {
 337                         memcpy(cpu_vaddr + cpu_offset,
 338                                gpu_vaddr + swizzled_gpu_offset,
 339                                this_length);
 340                 } else {
 341                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 342                                cpu_vaddr + cpu_offset,
 343                                this_length);
 344                 }
 345                 cpu_offset += this_length;
 346                 gpu_offset += this_length;
 347                 length -= this_length;
 348         }
 349
 350         kunmap(cpu_page);
 351         kunmap(gpu_page);
 352 }
 353
 354 /**
 355  * This is the fast shmem pread path, which attempts to copy_from_user directly
 356  * from the backing pages of the object to the user's address space.  On a
 357  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 358  */
 359 static int
 360 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 361                           struct drm_i915_gem_pread *args,
 362                           struct drm_file *file_priv)
 363 {
 364         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 365         ssize_t remain;
 366         loff_t offset, page_base;
 367         char __user *user_data;
 368         int page_offset, page_length;
 369         int ret;
 370
 371         user_data = (char __user *) (uintptr_t) args->data_ptr;
 372         remain = args->size;
 373
 374         ret = i915_mutex_lock_interruptible(dev);
 375         if (ret)
 376                 return ret;
 377
 378         ret = i915_gem_object_get_pages(obj, 0);
 379         if (ret != 0)
 380                 goto fail_unlock;
 381
 382         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 383                                                         args->size);
 384         if (ret != 0)
 385                 goto fail_put_pages;
 386
 387         obj_priv = to_intel_bo(obj);
 388         offset = args->offset;
 389
 390         while (remain > 0) {
 391                 /* Operation in this page
 392                  *
 393                  * page_base = page offset within aperture
 394                  * page_offset = offset within page
 395                  * page_length = bytes to copy for this page
 396                  */
 397                 page_base = (offset & ~(PAGE_SIZE-1));
 398                 page_offset = offset & (PAGE_SIZE-1);
 399                 page_length = remain;
 400                 if ((page_offset + remain) > PAGE_SIZE)
 401                         page_length = PAGE_SIZE - page_offset;
 402
 403                 ret = fast_shmem_read(obj_priv->pages,
 404                                       page_base, page_offset,
 405                                       user_data, page_length);
 406                 if (ret)
 407                         goto fail_put_pages;
 408
 409                 remain -= page_length;
 410                 user_data += page_length;
 411                 offset += page_length;
 412         }
 413
 414 fail_put_pages:
 415         i915_gem_object_put_pages(obj);
 416 fail_unlock:
 417         mutex_unlock(&dev->struct_mutex);
 418
 419         return ret;
 420 }
 421
 422 static int
 423 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 424 {
 425         int ret;
 426
 427         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 428
 429         /* If we've insufficient memory to map in the pages, attempt
 430          * to make some space by throwing out some old buffers.
 431          */
 432         if (ret == -ENOMEM) {
 433                 struct drm_device *dev = obj->dev;
 434
 435                 ret = i915_gem_evict_something(dev, obj->size,
 436                                                i915_gem_get_gtt_alignment(obj));
 437                 if (ret)
 438                         return ret;
 439
 440                 ret = i915_gem_object_get_pages(obj, 0);
 441         }
 442
 443         return ret;
 444 }
 445
 446 /**
 447  * This is the fallback shmem pread path, which allocates temporary storage
 448  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 449  * can copy out of the object's backing pages while holding the struct mutex
 450  * and not take page faults.
 451  */
 452 static int
 453 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 454                           struct drm_i915_gem_pread *args,
 455                           struct drm_file *file_priv)
 456 {
 457         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 458         struct mm_struct *mm = current->mm;
 459         struct page **user_pages;
 460         ssize_t remain;
 461         loff_t offset, pinned_pages, i;
 462         loff_t first_data_page, last_data_page, num_pages;
 463         int shmem_page_index, shmem_page_offset;
 464         int data_page_index,  data_page_offset;
 465         int page_length;
 466         int ret;
 467         uint64_t data_ptr = args->data_ptr;
 468         int do_bit17_swizzling;
 469
 470         remain = args->size;
 471
 472         /* Pin the user pages containing the data.  We can't fault while
 473          * holding the struct mutex, yet we want to hold it while
 474          * dereferencing the user data.
 475          */
 476         first_data_page = data_ptr / PAGE_SIZE;
 477         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 478         num_pages = last_data_page - first_data_page + 1;
 479
 480         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 481         if (user_pages == NULL)
 482                 return -ENOMEM;
 483
 484         down_read(&mm->mmap_sem);
 485         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 486                                       num_pages, 1, 0, user_pages, NULL);
 487         up_read(&mm->mmap_sem);
 488         if (pinned_pages < num_pages) {
 489                 ret = -EFAULT;
 490                 goto fail_put_user_pages;
 491         }
 492
 493         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 494
 495         ret = i915_mutex_lock_interruptible(dev);
 496         if (ret)
 497                 goto fail_put_user_pages;
 498
 499         ret = i915_gem_object_get_pages_or_evict(obj);
 500         if (ret)
 501                 goto fail_unlock;
 502
 503         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 504                                                         args->size);
 505         if (ret != 0)
 506                 goto fail_put_pages;
 507
 508         obj_priv = to_intel_bo(obj);
 509         offset = args->offset;
 510
 511         while (remain > 0) {
 512                 /* Operation in this page
 513                  *
 514                  * shmem_page_index = page number within shmem file
 515                  * shmem_page_offset = offset within page in shmem file
 516                  * data_page_index = page number in get_user_pages return
 517                  * data_page_offset = offset with data_page_index page.
 518                  * page_length = bytes to copy for this page
 519                  */
 520                 shmem_page_index = offset / PAGE_SIZE;
 521                 shmem_page_offset = offset & ~PAGE_MASK;
 522                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 523                 data_page_offset = data_ptr & ~PAGE_MASK;
 524
 525                 page_length = remain;
 526                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 527                         page_length = PAGE_SIZE - shmem_page_offset;
 528                 if ((data_page_offset + page_length) > PAGE_SIZE)
 529                         page_length = PAGE_SIZE - data_page_offset;
 530
 531                 if (do_bit17_swizzling) {
 532                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 533                                               shmem_page_offset,
 534                                               user_pages[data_page_index],
 535                                               data_page_offset,
 536                                               page_length,
 537                                               1);
 538                 } else {
 539                         slow_shmem_copy(user_pages[data_page_index],
 540                                         data_page_offset,
 541                                         obj_priv->pages[shmem_page_index],
 542                                         shmem_page_offset,
 543                                         page_length);
 544                 }
 545
 546                 remain -= page_length;
 547                 data_ptr += page_length;
 548                 offset += page_length;
 549         }
 550
 551 fail_put_pages:
 552         i915_gem_object_put_pages(obj);
 553 fail_unlock:
 554         mutex_unlock(&dev->struct_mutex);
 555 fail_put_user_pages:
 556         for (i = 0; i < pinned_pages; i++) {
 557                 SetPageDirty(user_pages[i]);
 558                 page_cache_release(user_pages[i]);
 559         }
 560         drm_free_large(user_pages);
 561
 562         return ret;
 563 }
 564
 565 /**
 566  * Reads data from the object referenced by handle.
 567  *
 568  * On error, the contents of *data are undefined.
 569  */
 570 int
 571 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 572                      struct drm_file *file_priv)
 573 {
 574         struct drm_i915_gem_pread *args = data;
 575         struct drm_gem_object *obj;
 576         struct drm_i915_gem_object *obj_priv;
 577         int ret = 0;
 578
 579         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 580         if (obj == NULL)
 581                 return -ENOENT;
 582         obj_priv = to_intel_bo(obj);
 583
 584         /* Bounds check source.  */
 585         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 586                 ret = -EINVAL;
 587                 goto out;
 588         }
 589
 590         if (args->size == 0)
 591                 goto out;
 592
 593         if (!access_ok(VERIFY_WRITE,
 594                        (char __user *)(uintptr_t)args->data_ptr,
 595                        args->size)) {
 596                 ret = -EFAULT;
 597                 goto out;
 598         }
 599
 600         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 601                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 602         } else {
 603                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 604                 if (ret != 0)
 605                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 606                                                         file_priv);
 607         }
 608
 609 out:
 610         drm_gem_object_unreference_unlocked(obj);
 611         return ret;
 612 }
 613
 614 /* This is the fast write path which cannot handle
 615  * page faults in the source data
 616  */
 617
 618 static inline int
 619 fast_user_write(struct io_mapping *mapping,
 620                 loff_t page_base, int page_offset,
 621                 char __user *user_data,
 622                 int length)
 623 {
 624         char *vaddr_atomic;
 625         unsigned long unwritten;
 626
 627         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 628         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 629                                                       user_data, length);
 630         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 631         if (unwritten)
 632                 return -EFAULT;
 633         return 0;
 634 }
 635
 636 /* Here's the write path which can sleep for
 637  * page faults
 638  */
 639
 640 static inline void
 641 slow_kernel_write(struct io_mapping *mapping,
 642                   loff_t gtt_base, int gtt_offset,
 643                   struct page *user_page, int user_offset,
 644                   int length)
 645 {
 646         char __iomem *dst_vaddr;
 647         char *src_vaddr;
 648
 649         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 650         src_vaddr = kmap(user_page);
 651
 652         memcpy_toio(dst_vaddr + gtt_offset,
 653                     src_vaddr + user_offset,
 654                     length);
 655
 656         kunmap(user_page);
 657         io_mapping_unmap(dst_vaddr);
 658 }
 659
 660 static inline int
 661 fast_shmem_write(struct page **pages,
 662                  loff_t page_base, int page_offset,
 663                  char __user *data,
 664                  int length)
 665 {
 666         char __iomem *vaddr;
 667         unsigned long unwritten;
 668
 669         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 670         if (vaddr == NULL)
 671                 return -ENOMEM;
 672         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 673         kunmap_atomic(vaddr, KM_USER0);
 674
 675         if (unwritten)
 676                 return -EFAULT;
 677         return 0;
 678 }
 679
 680 /**
 681  * This is the fast pwrite path, where we copy the data directly from the
 682  * user into the GTT, uncached.
 683  */
 684 static int
 685 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 686                          struct drm_i915_gem_pwrite *args,
 687                          struct drm_file *file_priv)
 688 {
 689         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 690         drm_i915_private_t *dev_priv = dev->dev_private;
 691         ssize_t remain;
 692         loff_t offset, page_base;
 693         char __user *user_data;
 694         int page_offset, page_length;
 695         int ret;
 696
 697         user_data = (char __user *) (uintptr_t) args->data_ptr;
 698         remain = args->size;
 699
 700         ret = i915_mutex_lock_interruptible(dev);
 701         if (ret)
 702                 return ret;
 703
 704         ret = i915_gem_object_pin(obj, 0);
 705         if (ret) {
 706                 mutex_unlock(&dev->struct_mutex);
 707                 return ret;
 708         }
 709         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 710         if (ret)
 711                 goto fail;
 712
 713         obj_priv = to_intel_bo(obj);
 714         offset = obj_priv->gtt_offset + args->offset;
 715
 716         while (remain > 0) {
 717                 /* Operation in this page
 718                  *
 719                  * page_base = page offset within aperture
 720                  * page_offset = offset within page
 721                  * page_length = bytes to copy for this page
 722                  */
 723                 page_base = (offset & ~(PAGE_SIZE-1));
 724                 page_offset = offset & (PAGE_SIZE-1);
 725                 page_length = remain;
 726                 if ((page_offset + remain) > PAGE_SIZE)
 727                         page_length = PAGE_SIZE - page_offset;
 728
 729                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 730                                        page_offset, user_data, page_length);
 731
 732                 /* If we get a fault while copying data, then (presumably) our
 733                  * source page isn't available.  Return the error and we'll
 734                  * retry in the slow path.
 735                  */
 736                 if (ret)
 737                         goto fail;
 738
 739                 remain -= page_length;
 740                 user_data += page_length;
 741                 offset += page_length;
 742         }
 743
 744 fail:
 745         i915_gem_object_unpin(obj);
 746         mutex_unlock(&dev->struct_mutex);
 747
 748         return ret;
 749 }
 750
 751 /**
 752  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 753  * the memory and maps it using kmap_atomic for copying.
 754  *
 755  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 756  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 757  */
 758 static int
 759 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 760                          struct drm_i915_gem_pwrite *args,
 761                          struct drm_file *file_priv)
 762 {
 763         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 764         drm_i915_private_t *dev_priv = dev->dev_private;
 765         ssize_t remain;
 766         loff_t gtt_page_base, offset;
 767         loff_t first_data_page, last_data_page, num_pages;
 768         loff_t pinned_pages, i;
 769         struct page **user_pages;
 770         struct mm_struct *mm = current->mm;
 771         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 772         int ret;
 773         uint64_t data_ptr = args->data_ptr;
 774
 775         remain = args->size;
 776
 777         /* Pin the user pages containing the data.  We can't fault while
 778          * holding the struct mutex, and all of the pwrite implementations
 779          * want to hold it while dereferencing the user data.
 780          */
 781         first_data_page = data_ptr / PAGE_SIZE;
 782         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 783         num_pages = last_data_page - first_data_page + 1;
 784
 785         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 786         if (user_pages == NULL)
 787                 return -ENOMEM;
 788
 789         down_read(&mm->mmap_sem);
 790         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 791                                       num_pages, 0, 0, user_pages, NULL);
 792         up_read(&mm->mmap_sem);
 793         if (pinned_pages < num_pages) {
 794                 ret = -EFAULT;
 795                 goto out_unpin_pages;
 796         }
 797
 798         ret = i915_mutex_lock_interruptible(dev);
 799         if (ret)
 800                 goto out_unpin_pages;
 801
 802         ret = i915_gem_object_pin(obj, 0);
 803         if (ret)
 804                 goto out_unlock;
 805
 806         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 807         if (ret)
 808                 goto out_unpin_object;
 809
 810         obj_priv = to_intel_bo(obj);
 811         offset = obj_priv->gtt_offset + args->offset;
 812
 813         while (remain > 0) {
 814                 /* Operation in this page
 815                  *
 816                  * gtt_page_base = page offset within aperture
 817                  * gtt_page_offset = offset within page in aperture
 818                  * data_page_index = page number in get_user_pages return
 819                  * data_page_offset = offset with data_page_index page.
 820                  * page_length = bytes to copy for this page
 821                  */
 822                 gtt_page_base = offset & PAGE_MASK;
 823                 gtt_page_offset = offset & ~PAGE_MASK;
 824                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 825                 data_page_offset = data_ptr & ~PAGE_MASK;
 826
 827                 page_length = remain;
 828                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 829                         page_length = PAGE_SIZE - gtt_page_offset;
 830                 if ((data_page_offset + page_length) > PAGE_SIZE)
 831                         page_length = PAGE_SIZE - data_page_offset;
 832
 833                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 834                                   gtt_page_base, gtt_page_offset,
 835                                   user_pages[data_page_index],
 836                                   data_page_offset,
 837                                   page_length);
 838
 839                 remain -= page_length;
 840                 offset += page_length;
 841                 data_ptr += page_length;
 842         }
 843
 844 out_unpin_object:
 845         i915_gem_object_unpin(obj);
 846 out_unlock:
 847         mutex_unlock(&dev->struct_mutex);
 848 out_unpin_pages:
 849         for (i = 0; i < pinned_pages; i++)
 850                 page_cache_release(user_pages[i]);
 851         drm_free_large(user_pages);
 852
 853         return ret;
 854 }
 855
 856 /**
 857  * This is the fast shmem pwrite path, which attempts to directly
 858  * copy_from_user into the kmapped pages backing the object.
 859  */
 860 static int
 861 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 862                            struct drm_i915_gem_pwrite *args,
 863                            struct drm_file *file_priv)
 864 {
 865         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 866         ssize_t remain;
 867         loff_t offset, page_base;
 868         char __user *user_data;
 869         int page_offset, page_length;
 870         int ret;
 871
 872         user_data = (char __user *) (uintptr_t) args->data_ptr;
 873         remain = args->size;
 874
 875         ret = i915_mutex_lock_interruptible(dev);
 876         if (ret)
 877                 return ret;
 878
 879         ret = i915_gem_object_get_pages(obj, 0);
 880         if (ret != 0)
 881                 goto fail_unlock;
 882
 883         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 884         if (ret != 0)
 885                 goto fail_put_pages;
 886
 887         obj_priv = to_intel_bo(obj);
 888         offset = args->offset;
 889         obj_priv->dirty = 1;
 890
 891         while (remain > 0) {
 892                 /* Operation in this page
 893                  *
 894                  * page_base = page offset within aperture
 895                  * page_offset = offset within page
 896                  * page_length = bytes to copy for this page
 897                  */
 898                 page_base = (offset & ~(PAGE_SIZE-1));
 899                 page_offset = offset & (PAGE_SIZE-1);
 900                 page_length = remain;
 901                 if ((page_offset + remain) > PAGE_SIZE)
 902                         page_length = PAGE_SIZE - page_offset;
 903
 904                 ret = fast_shmem_write(obj_priv->pages,
 905                                        page_base, page_offset,
 906                                        user_data, page_length);
 907                 if (ret)
 908                         goto fail_put_pages;
 909
 910                 remain -= page_length;
 911                 user_data += page_length;
 912                 offset += page_length;
 913         }
 914
 915 fail_put_pages:
 916         i915_gem_object_put_pages(obj);
 917 fail_unlock:
 918         mutex_unlock(&dev->struct_mutex);
 919
 920         return ret;
 921 }
 922
 923 /**
 924  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 925  * the memory and maps it using kmap_atomic for copying.
 926  *
 927  * This avoids taking mmap_sem for faulting on the user's address while the
 928  * struct_mutex is held.
 929  */
 930 static int
 931 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 932                            struct drm_i915_gem_pwrite *args,
 933                            struct drm_file *file_priv)
 934 {
 935         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 936         struct mm_struct *mm = current->mm;
 937         struct page **user_pages;
 938         ssize_t remain;
 939         loff_t offset, pinned_pages, i;
 940         loff_t first_data_page, last_data_page, num_pages;
 941         int shmem_page_index, shmem_page_offset;
 942         int data_page_index,  data_page_offset;
 943         int page_length;
 944         int ret;
 945         uint64_t data_ptr = args->data_ptr;
 946         int do_bit17_swizzling;
 947
 948         remain = args->size;
 949
 950         /* Pin the user pages containing the data.  We can't fault while
 951          * holding the struct mutex, and all of the pwrite implementations
 952          * want to hold it while dereferencing the user data.
 953          */
 954         first_data_page = data_ptr / PAGE_SIZE;
 955         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 956         num_pages = last_data_page - first_data_page + 1;
 957
 958         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 959         if (user_pages == NULL)
 960                 return -ENOMEM;
 961
 962         down_read(&mm->mmap_sem);
 963         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 964                                       num_pages, 0, 0, user_pages, NULL);
 965         up_read(&mm->mmap_sem);
 966         if (pinned_pages < num_pages) {
 967                 ret = -EFAULT;
 968                 goto fail_put_user_pages;
 969         }
 970
 971         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 972
 973         ret = i915_mutex_lock_interruptible(dev);
 974         if (ret)
 975                 goto fail_put_user_pages;
 976
 977         ret = i915_gem_object_get_pages_or_evict(obj);
 978         if (ret)
 979                 goto fail_unlock;
 980
 981         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 982         if (ret != 0)
 983                 goto fail_put_pages;
 984
 985         obj_priv = to_intel_bo(obj);
 986         offset = args->offset;
 987         obj_priv->dirty = 1;
 988
 989         while (remain > 0) {
 990                 /* Operation in this page
 991                  *
 992                  * shmem_page_index = page number within shmem file
 993                  * shmem_page_offset = offset within page in shmem file
 994                  * data_page_index = page number in get_user_pages return
 995                  * data_page_offset = offset with data_page_index page.
 996                  * page_length = bytes to copy for this page
 997                  */
 998                 shmem_page_index = offset / PAGE_SIZE;
 999                 shmem_page_offset = offset & ~PAGE_MASK;
1000                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
1001                 data_page_offset = data_ptr & ~PAGE_MASK;
1002
1003                 page_length = remain;
1004                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1005                         page_length = PAGE_SIZE - shmem_page_offset;
1006                 if ((data_page_offset + page_length) > PAGE_SIZE)
1007                         page_length = PAGE_SIZE - data_page_offset;
1008
1009                 if (do_bit17_swizzling) {
1010                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
1011                                               shmem_page_offset,
1012                                               user_pages[data_page_index],
1013                                               data_page_offset,
1014                                               page_length,
1015                                               0);
1016                 } else {
1017                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
1018                                         shmem_page_offset,
1019                                         user_pages[data_page_index],
1020                                         data_page_offset,
1021                                         page_length);
1022                 }
1023
1024                 remain -= page_length;
1025                 data_ptr += page_length;
1026                 offset += page_length;
1027         }
1028
1029 fail_put_pages:
1030         i915_gem_object_put_pages(obj);
1031 fail_unlock:
1032         mutex_unlock(&dev->struct_mutex);
1033 fail_put_user_pages:
1034         for (i = 0; i < pinned_pages; i++)
1035                 page_cache_release(user_pages[i]);
1036         drm_free_large(user_pages);
1037
1038         return ret;
1039 }
1040
1041 /**
1042  * Writes data to the object referenced by handle.
1043  *
1044  * On error, the contents of the buffer that were to be modified are undefined.
1045  */
1046 int
1047 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1048                       struct drm_file *file_priv)
1049 {
1050         struct drm_i915_gem_pwrite *args = data;
1051         struct drm_gem_object *obj;
1052         struct drm_i915_gem_object *obj_priv;
1053         int ret = 0;
1054
1055         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1056         if (obj == NULL)
1057                 return -ENOENT;
1058         obj_priv = to_intel_bo(obj);
1059
1060         /* Bounds check destination. */
1061         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1062                 ret = -EINVAL;
1063                 goto out;
1064         }
1065
1066         if (args->size == 0)
1067                 goto out;
1068
1069         if (!access_ok(VERIFY_READ,
1070                        (char __user *)(uintptr_t)args->data_ptr,
1071                        args->size)) {
1072                 ret = -EFAULT;
1073                 goto out;
1074         }
1075
1076         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1077          * it would end up going through the fenced access, and we'll get
1078          * different detiling behavior between reading and writing.
1079          * pread/pwrite currently are reading and writing from the CPU
1080          * perspective, requiring manual detiling by the client.
1081          */
1082         if (obj_priv->phys_obj)
1083                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
1084         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1085                  obj_priv->gtt_space &&
1086                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1087                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
1088                 if (ret == -EFAULT) {
1089                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
1090                                                        file_priv);
1091                 }
1092         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1093                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
1094         } else {
1095                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1096                 if (ret == -EFAULT) {
1097                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1098                                                          file_priv);
1099                 }
1100         }
1101
1102 #if WATCH_PWRITE
1103         if (ret)
1104                 DRM_INFO("pwrite failed %d\n", ret);
1105 #endif
1106
1107 out:
1108         drm_gem_object_unreference_unlocked(obj);
1109         return ret;
1110 }
1111
1112 /**
1113  * Called when user space prepares to use an object with the CPU, either
1114  * through the mmap ioctl's mapping or a GTT mapping.
1115  */
1116 int
1117 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1118                           struct drm_file *file_priv)
1119 {
1120         struct drm_i915_private *dev_priv = dev->dev_private;
1121         struct drm_i915_gem_set_domain *args = data;
1122         struct drm_gem_object *obj;
1123         struct drm_i915_gem_object *obj_priv;
1124         uint32_t read_domains = args->read_domains;
1125         uint32_t write_domain = args->write_domain;
1126         int ret;
1127
1128         if (!(dev->driver->driver_features & DRIVER_GEM))
1129                 return -ENODEV;
1130
1131         /* Only handle setting domains to types used by the CPU. */
1132         if (write_domain & I915_GEM_GPU_DOMAINS)
1133                 return -EINVAL;
1134
1135         if (read_domains & I915_GEM_GPU_DOMAINS)
1136                 return -EINVAL;
1137
1138         /* Having something in the write domain implies it's in the read
1139          * domain, and only that read domain.  Enforce that in the request.
1140          */
1141         if (write_domain != 0 && read_domains != write_domain)
1142                 return -EINVAL;
1143
1144         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1145         if (obj == NULL)
1146                 return -ENOENT;
1147         obj_priv = to_intel_bo(obj);
1148
1149         ret = i915_mutex_lock_interruptible(dev);
1150         if (ret) {
1151                 drm_gem_object_unreference_unlocked(obj);
1152                 return ret;
1153         }
1154
1155         intel_mark_busy(dev, obj);
1156
1157         if (read_domains & I915_GEM_DOMAIN_GTT) {
1158                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1159
1160                 /* Update the LRU on the fence for the CPU access that's
1161                  * about to occur.
1162                  */
1163                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1164                         struct drm_i915_fence_reg *reg =
1165                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1166                         list_move_tail(&reg->lru_list,
1167                                        &dev_priv->mm.fence_list);
1168                 }
1169
1170                 /* Silently promote "you're not bound, there was nothing to do"
1171                  * to success, since the client was just asking us to
1172                  * make sure everything was done.
1173                  */
1174                 if (ret == -EINVAL)
1175                         ret = 0;
1176         } else {
1177                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1178         }
1179
1180         /* Maintain LRU order of "inactive" objects */
1181         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1182                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1183
1184         drm_gem_object_unreference(obj);
1185         mutex_unlock(&dev->struct_mutex);
1186         return ret;
1187 }
1188
1189 /**
1190  * Called when user space has done writes to this buffer
1191  */
1192 int
1193 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1194                       struct drm_file *file_priv)
1195 {
1196         struct drm_i915_gem_sw_finish *args = data;
1197         struct drm_gem_object *obj;
1198         int ret = 0;
1199
1200         if (!(dev->driver->driver_features & DRIVER_GEM))
1201                 return -ENODEV;
1202
1203         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1204         if (obj == NULL)
1205                 return -ENOENT;
1206
1207         ret = i915_mutex_lock_interruptible(dev);
1208         if (ret) {
1209                 drm_gem_object_unreference_unlocked(obj);
1210                 return ret;
1211         }
1212
1213         /* Pinned buffers may be scanout, so flush the cache */
1214         if (to_intel_bo(obj)->pin_count)
1215                 i915_gem_object_flush_cpu_write_domain(obj);
1216
1217         drm_gem_object_unreference(obj);
1218         mutex_unlock(&dev->struct_mutex);
1219         return ret;
1220 }
1221
1222 /**
1223  * Maps the contents of an object, returning the address it is mapped
1224  * into.
1225  *
1226  * While the mapping holds a reference on the contents of the object, it doesn't
1227  * imply a ref on the object itself.
1228  */
1229 int
1230 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1231                    struct drm_file *file_priv)
1232 {
1233         struct drm_i915_gem_mmap *args = data;
1234         struct drm_gem_object *obj;
1235         loff_t offset;
1236         unsigned long addr;
1237
1238         if (!(dev->driver->driver_features & DRIVER_GEM))
1239                 return -ENODEV;
1240
1241         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1242         if (obj == NULL)
1243                 return -ENOENT;
1244
1245         offset = args->offset;
1246
1247         down_write(&current->mm->mmap_sem);
1248         addr = do_mmap(obj->filp, 0, args->size,
1249                        PROT_READ | PROT_WRITE, MAP_SHARED,
1250                        args->offset);
1251         up_write(&current->mm->mmap_sem);
1252         drm_gem_object_unreference_unlocked(obj);
1253         if (IS_ERR((void *)addr))
1254                 return addr;
1255
1256         args->addr_ptr = (uint64_t) addr;
1257
1258         return 0;
1259 }
1260
1261 /**
1262  * i915_gem_fault - fault a page into the GTT
1263  * vma: VMA in question
1264  * vmf: fault info
1265  *
1266  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1267  * from userspace.  The fault handler takes care of binding the object to
1268  * the GTT (if needed), allocating and programming a fence register (again,
1269  * only if needed based on whether the old reg is still valid or the object
1270  * is tiled) and inserting a new PTE into the faulting process.
1271  *
1272  * Note that the faulting process may involve evicting existing objects
1273  * from the GTT and/or fence registers to make room.  So performance may
1274  * suffer if the GTT working set is large or there are few fence registers
1275  * left.
1276  */
1277 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1278 {
1279         struct drm_gem_object *obj = vma->vm_private_data;
1280         struct drm_device *dev = obj->dev;
1281         drm_i915_private_t *dev_priv = dev->dev_private;
1282         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1283         pgoff_t page_offset;
1284         unsigned long pfn;
1285         int ret = 0;
1286         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1287
1288         /* We don't use vmf->pgoff since that has the fake offset */
1289         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1290                 PAGE_SHIFT;
1291
1292         /* Now bind it into the GTT if needed */
1293         mutex_lock(&dev->struct_mutex);
1294         if (!obj_priv->gtt_space) {
1295                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1296                 if (ret)
1297                         goto unlock;
1298
1299                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1300                 if (ret)
1301                         goto unlock;
1302         }
1303
1304         /* Need a new fence register? */
1305         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1306                 ret = i915_gem_object_get_fence_reg(obj, true);
1307                 if (ret)
1308                         goto unlock;
1309         }
1310
1311         if (i915_gem_object_is_inactive(obj_priv))
1312                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1313
1314         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1315                 page_offset;
1316
1317         /* Finally, remap it using the new GTT offset */
1318         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1319 unlock:
1320         mutex_unlock(&dev->struct_mutex);
1321
1322         switch (ret) {
1323         case 0:
1324         case -ERESTARTSYS:
1325                 return VM_FAULT_NOPAGE;
1326         case -ENOMEM:
1327         case -EAGAIN:
1328                 return VM_FAULT_OOM;
1329         default:
1330                 return VM_FAULT_SIGBUS;
1331         }
1332 }
1333
1334 /**
1335  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1336  * @obj: obj in question
1337  *
1338  * GEM memory mapping works by handing back to userspace a fake mmap offset
1339  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1340  * up the object based on the offset and sets up the various memory mapping
1341  * structures.
1342  *
1343  * This routine allocates and attaches a fake offset for @obj.
1344  */
1345 static int
1346 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1347 {
1348         struct drm_device *dev = obj->dev;
1349         struct drm_gem_mm *mm = dev->mm_private;
1350         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1351         struct drm_map_list *list;
1352         struct drm_local_map *map;
1353         int ret = 0;
1354
1355         /* Set the object up for mmap'ing */
1356         list = &obj->map_list;
1357         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1358         if (!list->map)
1359                 return -ENOMEM;
1360
1361         map = list->map;
1362         map->type = _DRM_GEM;
1363         map->size = obj->size;
1364         map->handle = obj;
1365
1366         /* Get a DRM GEM mmap offset allocated... */
1367         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1368                                                     obj->size / PAGE_SIZE, 0, 0);
1369         if (!list->file_offset_node) {
1370                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1371                 ret = -ENOSPC;
1372                 goto out_free_list;
1373         }
1374
1375         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1376                                                   obj->size / PAGE_SIZE, 0);
1377         if (!list->file_offset_node) {
1378                 ret = -ENOMEM;
1379                 goto out_free_list;
1380         }
1381
1382         list->hash.key = list->file_offset_node->start;
1383         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1384         if (ret) {
1385                 DRM_ERROR("failed to add to map hash\n");
1386                 goto out_free_mm;
1387         }
1388
1389         /* By now we should be all set, any drm_mmap request on the offset
1390          * below will get to our mmap & fault handler */
1391         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1392
1393         return 0;
1394
1395 out_free_mm:
1396         drm_mm_put_block(list->file_offset_node);
1397 out_free_list:
1398         kfree(list->map);
1399
1400         return ret;
1401 }
1402
1403 /**
1404  * i915_gem_release_mmap - remove physical page mappings
1405  * @obj: obj in question
1406  *
1407  * Preserve the reservation of the mmapping with the DRM core code, but
1408  * relinquish ownership of the pages back to the system.
1409  *
1410  * It is vital that we remove the page mapping if we have mapped a tiled
1411  * object through the GTT and then lose the fence register due to
1412  * resource pressure. Similarly if the object has been moved out of the
1413  * aperture, than pages mapped into userspace must be revoked. Removing the
1414  * mapping will then trigger a page fault on the next user access, allowing
1415  * fixup by i915_gem_fault().
1416  */
1417 void
1418 i915_gem_release_mmap(struct drm_gem_object *obj)
1419 {
1420         struct drm_device *dev = obj->dev;
1421         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1422
1423         if (dev->dev_mapping)
1424                 unmap_mapping_range(dev->dev_mapping,
1425                                     obj_priv->mmap_offset, obj->size, 1);
1426 }
1427
1428 static void
1429 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1430 {
1431         struct drm_device *dev = obj->dev;
1432         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1433         struct drm_gem_mm *mm = dev->mm_private;
1434         struct drm_map_list *list;
1435
1436         list = &obj->map_list;
1437         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1438
1439         if (list->file_offset_node) {
1440                 drm_mm_put_block(list->file_offset_node);
1441                 list->file_offset_node = NULL;
1442         }
1443
1444         if (list->map) {
1445                 kfree(list->map);
1446                 list->map = NULL;
1447         }
1448
1449         obj_priv->mmap_offset = 0;
1450 }
1451
1452 /**
1453  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1454  * @obj: object to check
1455  *
1456  * Return the required GTT alignment for an object, taking into account
1457  * potential fence register mapping if needed.
1458  */
1459 static uint32_t
1460 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1461 {
1462         struct drm_device *dev = obj->dev;
1463         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1464         int start, i;
1465
1466         /*
1467          * Minimum alignment is 4k (GTT page size), but might be greater
1468          * if a fence register is needed for the object.
1469          */
1470         if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1471                 return 4096;
1472
1473         /*
1474          * Previous chips need to be aligned to the size of the smallest
1475          * fence register that can contain the object.
1476          */
1477         if (INTEL_INFO(dev)->gen == 3)
1478                 start = 1024*1024;
1479         else
1480                 start = 512*1024;
1481
1482         for (i = start; i < obj->size; i <<= 1)
1483                 ;
1484
1485         return i;
1486 }
1487
1488 /**
1489  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1490  * @dev: DRM device
1491  * @data: GTT mapping ioctl data
1492  * @file_priv: GEM object info
1493  *
1494  * Simply returns the fake offset to userspace so it can mmap it.
1495  * The mmap call will end up in drm_gem_mmap(), which will set things
1496  * up so we can get faults in the handler above.
1497  *
1498  * The fault handler will take care of binding the object into the GTT
1499  * (since it may have been evicted to make room for something), allocating
1500  * a fence register, and mapping the appropriate aperture address into
1501  * userspace.
1502  */
1503 int
1504 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1505                         struct drm_file *file_priv)
1506 {
1507         struct drm_i915_gem_mmap_gtt *args = data;
1508         struct drm_gem_object *obj;
1509         struct drm_i915_gem_object *obj_priv;
1510         int ret;
1511
1512         if (!(dev->driver->driver_features & DRIVER_GEM))
1513                 return -ENODEV;
1514
1515         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1516         if (obj == NULL)
1517                 return -ENOENT;
1518
1519         ret = i915_mutex_lock_interruptible(dev);
1520         if (ret) {
1521                 drm_gem_object_unreference_unlocked(obj);
1522                 return ret;
1523         }
1524
1525         obj_priv = to_intel_bo(obj);
1526
1527         if (obj_priv->madv != I915_MADV_WILLNEED) {
1528                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1529                 drm_gem_object_unreference(obj);
1530                 mutex_unlock(&dev->struct_mutex);
1531                 return -EINVAL;
1532         }
1533
1534
1535         if (!obj_priv->mmap_offset) {
1536                 ret = i915_gem_create_mmap_offset(obj);
1537                 if (ret) {
1538                         drm_gem_object_unreference(obj);
1539                         mutex_unlock(&dev->struct_mutex);
1540                         return ret;
1541                 }
1542         }
1543
1544         args->offset = obj_priv->mmap_offset;
1545
1546         /*
1547          * Pull it into the GTT so that we have a page list (makes the
1548          * initial fault faster and any subsequent flushing possible).
1549          */
1550         if (!obj_priv->agp_mem) {
1551                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1552                 if (ret) {
1553                         drm_gem_object_unreference(obj);
1554                         mutex_unlock(&dev->struct_mutex);
1555                         return ret;
1556                 }
1557         }
1558
1559         drm_gem_object_unreference(obj);
1560         mutex_unlock(&dev->struct_mutex);
1561
1562         return 0;
1563 }
1564
1565 static void
1566 i915_gem_object_put_pages(struct drm_gem_object *obj)
1567 {
1568         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1569         int page_count = obj->size / PAGE_SIZE;
1570         int i;
1571
1572         BUG_ON(obj_priv->pages_refcount == 0);
1573         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1574
1575         if (--obj_priv->pages_refcount != 0)
1576                 return;
1577
1578         if (obj_priv->tiling_mode != I915_TILING_NONE)
1579                 i915_gem_object_save_bit_17_swizzle(obj);
1580
1581         if (obj_priv->madv == I915_MADV_DONTNEED)
1582                 obj_priv->dirty = 0;
1583
1584         for (i = 0; i < page_count; i++) {
1585                 if (obj_priv->dirty)
1586                         set_page_dirty(obj_priv->pages[i]);
1587
1588                 if (obj_priv->madv == I915_MADV_WILLNEED)
1589                         mark_page_accessed(obj_priv->pages[i]);
1590
1591                 page_cache_release(obj_priv->pages[i]);
1592         }
1593         obj_priv->dirty = 0;
1594
1595         drm_free_large(obj_priv->pages);
1596         obj_priv->pages = NULL;
1597 }
1598
1599 static uint32_t
1600 i915_gem_next_request_seqno(struct drm_device *dev,
1601                             struct intel_ring_buffer *ring)
1602 {
1603         drm_i915_private_t *dev_priv = dev->dev_private;
1604
1605         ring->outstanding_lazy_request = true;
1606         return dev_priv->next_seqno;
1607 }
1608
1609 static void
1610 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1611                                struct intel_ring_buffer *ring)
1612 {
1613         struct drm_device *dev = obj->dev;
1614         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1615         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1616
1617         BUG_ON(ring == NULL);
1618         obj_priv->ring = ring;
1619
1620         /* Add a reference if we're newly entering the active list. */
1621         if (!obj_priv->active) {
1622                 drm_gem_object_reference(obj);
1623                 obj_priv->active = 1;
1624         }
1625
1626         /* Move from whatever list we were on to the tail of execution. */
1627         list_move_tail(&obj_priv->list, &ring->active_list);
1628         obj_priv->last_rendering_seqno = seqno;
1629 }
1630
1631 static void
1632 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1633 {
1634         struct drm_device *dev = obj->dev;
1635         drm_i915_private_t *dev_priv = dev->dev_private;
1636         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1637
1638         BUG_ON(!obj_priv->active);
1639         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1640         obj_priv->last_rendering_seqno = 0;
1641 }
1642
1643 /* Immediately discard the backing storage */
1644 static void
1645 i915_gem_object_truncate(struct drm_gem_object *obj)
1646 {
1647         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1648         struct inode *inode;
1649
1650         /* Our goal here is to return as much of the memory as
1651          * is possible back to the system as we are called from OOM.
1652          * To do this we must instruct the shmfs to drop all of its
1653          * backing pages, *now*. Here we mirror the actions taken
1654          * when by shmem_delete_inode() to release the backing store.
1655          */
1656         inode = obj->filp->f_path.dentry->d_inode;
1657         truncate_inode_pages(inode->i_mapping, 0);
1658         if (inode->i_op->truncate_range)
1659                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1660
1661         obj_priv->madv = __I915_MADV_PURGED;
1662 }
1663
1664 static inline int
1665 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1666 {
1667         return obj_priv->madv == I915_MADV_DONTNEED;
1668 }
1669
1670 static void
1671 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1672 {
1673         struct drm_device *dev = obj->dev;
1674         drm_i915_private_t *dev_priv = dev->dev_private;
1675         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1676
1677         if (obj_priv->pin_count != 0)
1678                 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
1679         else
1680                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1681
1682         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1683
1684         obj_priv->last_rendering_seqno = 0;
1685         obj_priv->ring = NULL;
1686         if (obj_priv->active) {
1687                 obj_priv->active = 0;
1688                 drm_gem_object_unreference(obj);
1689         }
1690         WARN_ON(i915_verify_lists(dev));
1691 }
1692
1693 static void
1694 i915_gem_process_flushing_list(struct drm_device *dev,
1695                                uint32_t flush_domains,
1696                                struct intel_ring_buffer *ring)
1697 {
1698         drm_i915_private_t *dev_priv = dev->dev_private;
1699         struct drm_i915_gem_object *obj_priv, *next;
1700
1701         list_for_each_entry_safe(obj_priv, next,
1702                                  &dev_priv->mm.gpu_write_list,
1703                                  gpu_write_list) {
1704                 struct drm_gem_object *obj = &obj_priv->base;
1705
1706                 if (obj->write_domain & flush_domains &&
1707                     obj_priv->ring == ring) {
1708                         uint32_t old_write_domain = obj->write_domain;
1709
1710                         obj->write_domain = 0;
1711                         list_del_init(&obj_priv->gpu_write_list);
1712                         i915_gem_object_move_to_active(obj, ring);
1713
1714                         /* update the fence lru list */
1715                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1716                                 struct drm_i915_fence_reg *reg =
1717                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1718                                 list_move_tail(&reg->lru_list,
1719                                                 &dev_priv->mm.fence_list);
1720                         }
1721
1722                         trace_i915_gem_object_change_domain(obj,
1723                                                             obj->read_domains,
1724                                                             old_write_domain);
1725                 }
1726         }
1727 }
1728
1729 uint32_t
1730 i915_add_request(struct drm_device *dev,
1731                  struct drm_file *file,
1732                  struct drm_i915_gem_request *request,
1733                  struct intel_ring_buffer *ring)
1734 {
1735         drm_i915_private_t *dev_priv = dev->dev_private;
1736         struct drm_i915_file_private *file_priv = NULL;
1737         uint32_t seqno;
1738         int was_empty;
1739
1740         if (file != NULL)
1741                 file_priv = file->driver_priv;
1742
1743         if (request == NULL) {
1744                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1745                 if (request == NULL)
1746                         return 0;
1747         }
1748
1749         seqno = ring->add_request(dev, ring, 0);
1750         ring->outstanding_lazy_request = false;
1751
1752         request->seqno = seqno;
1753         request->ring = ring;
1754         request->emitted_jiffies = jiffies;
1755         was_empty = list_empty(&ring->request_list);
1756         list_add_tail(&request->list, &ring->request_list);
1757
1758         if (file_priv) {
1759                 spin_lock(&file_priv->mm.lock);
1760                 request->file_priv = file_priv;
1761                 list_add_tail(&request->client_list,
1762                               &file_priv->mm.request_list);
1763                 spin_unlock(&file_priv->mm.lock);
1764         }
1765
1766         if (!dev_priv->mm.suspended) {
1767                 mod_timer(&dev_priv->hangcheck_timer,
1768                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1769                 if (was_empty)
1770                         queue_delayed_work(dev_priv->wq,
1771                                            &dev_priv->mm.retire_work, HZ);
1772         }
1773         return seqno;
1774 }
1775
1776 /**
1777  * Command execution barrier
1778  *
1779  * Ensures that all commands in the ring are finished
1780  * before signalling the CPU
1781  */
1782 static void
1783 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1784 {
1785         uint32_t flush_domains = 0;
1786
1787         /* The sampler always gets flushed on i965 (sigh) */
1788         if (INTEL_INFO(dev)->gen >= 4)
1789                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1790
1791         ring->flush(dev, ring,
1792                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1793 }
1794
1795 static inline void
1796 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1797 {
1798         struct drm_i915_file_private *file_priv = request->file_priv;
1799
1800         if (!file_priv)
1801                 return;
1802
1803         spin_lock(&file_priv->mm.lock);
1804         list_del(&request->client_list);
1805         request->file_priv = NULL;
1806         spin_unlock(&file_priv->mm.lock);
1807 }
1808
1809 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1810                                       struct intel_ring_buffer *ring)
1811 {
1812         while (!list_empty(&ring->request_list)) {
1813                 struct drm_i915_gem_request *request;
1814
1815                 request = list_first_entry(&ring->request_list,
1816                                            struct drm_i915_gem_request,
1817                                            list);
1818
1819                 list_del(&request->list);
1820                 i915_gem_request_remove_from_client(request);
1821                 kfree(request);
1822         }
1823
1824         while (!list_empty(&ring->active_list)) {
1825                 struct drm_i915_gem_object *obj_priv;
1826
1827                 obj_priv = list_first_entry(&ring->active_list,
1828                                             struct drm_i915_gem_object,
1829                                             list);
1830
1831                 obj_priv->base.write_domain = 0;
1832                 list_del_init(&obj_priv->gpu_write_list);
1833                 i915_gem_object_move_to_inactive(&obj_priv->base);
1834         }
1835 }
1836
1837 void i915_gem_reset(struct drm_device *dev)
1838 {
1839         struct drm_i915_private *dev_priv = dev->dev_private;
1840         struct drm_i915_gem_object *obj_priv;
1841         int i;
1842
1843         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1844         if (HAS_BSD(dev))
1845                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1846
1847         /* Remove anything from the flushing lists. The GPU cache is likely
1848          * to be lost on reset along with the data, so simply move the
1849          * lost bo to the inactive list.
1850          */
1851         while (!list_empty(&dev_priv->mm.flushing_list)) {
1852                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1853                                             struct drm_i915_gem_object,
1854                                             list);
1855
1856                 obj_priv->base.write_domain = 0;
1857                 list_del_init(&obj_priv->gpu_write_list);
1858                 i915_gem_object_move_to_inactive(&obj_priv->base);
1859         }
1860
1861         /* Move everything out of the GPU domains to ensure we do any
1862          * necessary invalidation upon reuse.
1863          */
1864         list_for_each_entry(obj_priv,
1865                             &dev_priv->mm.inactive_list,
1866                             list)
1867         {
1868                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1869         }
1870
1871         /* The fence registers are invalidated so clear them out */
1872         for (i = 0; i < 16; i++) {
1873                 struct drm_i915_fence_reg *reg;
1874
1875                 reg = &dev_priv->fence_regs[i];
1876                 if (!reg->obj)
1877                         continue;
1878
1879                 i915_gem_clear_fence_reg(reg->obj);
1880         }
1881 }
1882
1883 /**
1884  * This function clears the request list as sequence numbers are passed.
1885  */
1886 static void
1887 i915_gem_retire_requests_ring(struct drm_device *dev,
1888                               struct intel_ring_buffer *ring)
1889 {
1890         drm_i915_private_t *dev_priv = dev->dev_private;
1891         uint32_t seqno;
1892
1893         if (!ring->status_page.page_addr ||
1894             list_empty(&ring->request_list))
1895                 return;
1896
1897         WARN_ON(i915_verify_lists(dev));
1898
1899         seqno = ring->get_seqno(dev, ring);
1900         while (!list_empty(&ring->request_list)) {
1901                 struct drm_i915_gem_request *request;
1902
1903                 request = list_first_entry(&ring->request_list,
1904                                            struct drm_i915_gem_request,
1905                                            list);
1906
1907                 if (!i915_seqno_passed(seqno, request->seqno))
1908                         break;
1909
1910                 trace_i915_gem_request_retire(dev, request->seqno);
1911
1912                 list_del(&request->list);
1913                 i915_gem_request_remove_from_client(request);
1914                 kfree(request);
1915         }
1916
1917         /* Move any buffers on the active list that are no longer referenced
1918          * by the ringbuffer to the flushing/inactive lists as appropriate.
1919          */
1920         while (!list_empty(&ring->active_list)) {
1921                 struct drm_gem_object *obj;
1922                 struct drm_i915_gem_object *obj_priv;
1923
1924                 obj_priv = list_first_entry(&ring->active_list,
1925                                             struct drm_i915_gem_object,
1926                                             list);
1927
1928                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1929                         break;
1930
1931                 obj = &obj_priv->base;
1932                 if (obj->write_domain != 0)
1933                         i915_gem_object_move_to_flushing(obj);
1934                 else
1935                         i915_gem_object_move_to_inactive(obj);
1936         }
1937
1938         if (unlikely (dev_priv->trace_irq_seqno &&
1939                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1940                 ring->user_irq_put(dev, ring);
1941                 dev_priv->trace_irq_seqno = 0;
1942         }
1943
1944         WARN_ON(i915_verify_lists(dev));
1945 }
1946
1947 void
1948 i915_gem_retire_requests(struct drm_device *dev)
1949 {
1950         drm_i915_private_t *dev_priv = dev->dev_private;
1951
1952         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1953             struct drm_i915_gem_object *obj_priv, *tmp;
1954
1955             /* We must be careful that during unbind() we do not
1956              * accidentally infinitely recurse into retire requests.
1957              * Currently:
1958              *   retire -> free -> unbind -> wait -> retire_ring
1959              */
1960             list_for_each_entry_safe(obj_priv, tmp,
1961                                      &dev_priv->mm.deferred_free_list,
1962                                      list)
1963                     i915_gem_free_object_tail(&obj_priv->base);
1964         }
1965
1966         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1967         if (HAS_BSD(dev))
1968                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1969 }
1970
1971 static void
1972 i915_gem_retire_work_handler(struct work_struct *work)
1973 {
1974         drm_i915_private_t *dev_priv;
1975         struct drm_device *dev;
1976
1977         dev_priv = container_of(work, drm_i915_private_t,
1978                                 mm.retire_work.work);
1979         dev = dev_priv->dev;
1980
1981         /* Come back later if the device is busy... */
1982         if (!mutex_trylock(&dev->struct_mutex)) {
1983                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1984                 return;
1985         }
1986
1987         i915_gem_retire_requests(dev);
1988
1989         if (!dev_priv->mm.suspended &&
1990                 (!list_empty(&dev_priv->render_ring.request_list) ||
1991                         (HAS_BSD(dev) &&
1992                          !list_empty(&dev_priv->bsd_ring.request_list))))
1993                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1994         mutex_unlock(&dev->struct_mutex);
1995 }
1996
1997 int
1998 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1999                      bool interruptible, struct intel_ring_buffer *ring)
2000 {
2001         drm_i915_private_t *dev_priv = dev->dev_private;
2002         u32 ier;
2003         int ret = 0;
2004
2005         BUG_ON(seqno == 0);
2006
2007         if (atomic_read(&dev_priv->mm.wedged))
2008                 return -EAGAIN;
2009
2010         if (ring->outstanding_lazy_request) {
2011                 seqno = i915_add_request(dev, NULL, NULL, ring);
2012                 if (seqno == 0)
2013                         return -ENOMEM;
2014         }
2015         BUG_ON(seqno == dev_priv->next_seqno);
2016
2017         if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
2018                 if (HAS_PCH_SPLIT(dev))
2019                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2020                 else
2021                         ier = I915_READ(IER);
2022                 if (!ier) {
2023                         DRM_ERROR("something (likely vbetool) disabled "
2024                                   "interrupts, re-enabling\n");
2025                         i915_driver_irq_preinstall(dev);
2026                         i915_driver_irq_postinstall(dev);
2027                 }
2028
2029                 trace_i915_gem_request_wait_begin(dev, seqno);
2030
2031                 ring->waiting_gem_seqno = seqno;
2032                 ring->user_irq_get(dev, ring);
2033                 if (interruptible)
2034                         ret = wait_event_interruptible(ring->irq_queue,
2035                                 i915_seqno_passed(
2036                                         ring->get_seqno(dev, ring), seqno)
2037                                 || atomic_read(&dev_priv->mm.wedged));
2038                 else
2039                         wait_event(ring->irq_queue,
2040                                 i915_seqno_passed(
2041                                         ring->get_seqno(dev, ring), seqno)
2042                                 || atomic_read(&dev_priv->mm.wedged));
2043
2044                 ring->user_irq_put(dev, ring);
2045                 ring->waiting_gem_seqno = 0;
2046
2047                 trace_i915_gem_request_wait_end(dev, seqno);
2048         }
2049         if (atomic_read(&dev_priv->mm.wedged))
2050                 ret = -EAGAIN;
2051
2052         if (ret && ret != -ERESTARTSYS)
2053                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2054                           __func__, ret, seqno, ring->get_seqno(dev, ring),
2055                           dev_priv->next_seqno);
2056
2057         /* Directly dispatch request retiring.  While we have the work queue
2058          * to handle this, the waiter on a request often wants an associated
2059          * buffer to have made it to the inactive list, and we would need
2060          * a separate wait queue to handle that.
2061          */
2062         if (ret == 0)
2063                 i915_gem_retire_requests_ring(dev, ring);
2064
2065         return ret;
2066 }
2067
2068 /**
2069  * Waits for a sequence number to be signaled, and cleans up the
2070  * request and object lists appropriately for that event.
2071  */
2072 static int
2073 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2074                   struct intel_ring_buffer *ring)
2075 {
2076         return i915_do_wait_request(dev, seqno, 1, ring);
2077 }
2078
2079 static void
2080 i915_gem_flush_ring(struct drm_device *dev,
2081                     struct drm_file *file_priv,
2082                     struct intel_ring_buffer *ring,
2083                     uint32_t invalidate_domains,
2084                     uint32_t flush_domains)
2085 {
2086         ring->flush(dev, ring, invalidate_domains, flush_domains);
2087         i915_gem_process_flushing_list(dev, flush_domains, ring);
2088 }
2089
2090 static void
2091 i915_gem_flush(struct drm_device *dev,
2092                struct drm_file *file_priv,
2093                uint32_t invalidate_domains,
2094                uint32_t flush_domains,
2095                uint32_t flush_rings)
2096 {
2097         drm_i915_private_t *dev_priv = dev->dev_private;
2098
2099         if (flush_domains & I915_GEM_DOMAIN_CPU)
2100                 drm_agp_chipset_flush(dev);
2101
2102         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2103                 if (flush_rings & RING_RENDER)
2104                         i915_gem_flush_ring(dev, file_priv,
2105                                             &dev_priv->render_ring,
2106                                             invalidate_domains, flush_domains);
2107                 if (flush_rings & RING_BSD)
2108                         i915_gem_flush_ring(dev, file_priv,
2109                                             &dev_priv->bsd_ring,
2110                                             invalidate_domains, flush_domains);
2111         }
2112 }
2113
2114 /**
2115  * Ensures that all rendering to the object has completed and the object is
2116  * safe to unbind from the GTT or access from the CPU.
2117  */
2118 static int
2119 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2120                                bool interruptible)
2121 {
2122         struct drm_device *dev = obj->dev;
2123         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2124         int ret;
2125
2126         /* This function only exists to support waiting for existing rendering,
2127          * not for emitting required flushes.
2128          */
2129         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2130
2131         /* If there is rendering queued on the buffer being evicted, wait for
2132          * it.
2133          */
2134         if (obj_priv->active) {
2135                 ret = i915_do_wait_request(dev,
2136                                            obj_priv->last_rendering_seqno,
2137                                            interruptible,
2138                                            obj_priv->ring);
2139                 if (ret)
2140                         return ret;
2141         }
2142
2143         return 0;
2144 }
2145
2146 /**
2147  * Unbinds an object from the GTT aperture.
2148  */
2149 int
2150 i915_gem_object_unbind(struct drm_gem_object *obj)
2151 {
2152         struct drm_device *dev = obj->dev;
2153         struct drm_i915_private *dev_priv = dev->dev_private;
2154         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2155         int ret = 0;
2156
2157         if (obj_priv->gtt_space == NULL)
2158                 return 0;
2159
2160         if (obj_priv->pin_count != 0) {
2161                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2162                 return -EINVAL;
2163         }
2164
2165         /* blow away mappings if mapped through GTT */
2166         i915_gem_release_mmap(obj);
2167
2168         /* Move the object to the CPU domain to ensure that
2169          * any possible CPU writes while it's not in the GTT
2170          * are flushed when we go to remap it. This will
2171          * also ensure that all pending GPU writes are finished
2172          * before we unbind.
2173          */
2174         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2175         if (ret == -ERESTARTSYS)
2176                 return ret;
2177         /* Continue on if we fail due to EIO, the GPU is hung so we
2178          * should be safe and we need to cleanup or else we might
2179          * cause memory corruption through use-after-free.
2180          */
2181         if (ret) {
2182                 i915_gem_clflush_object(obj);
2183                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2184         }
2185
2186         /* release the fence reg _after_ flushing */
2187         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2188                 i915_gem_clear_fence_reg(obj);
2189
2190         drm_unbind_agp(obj_priv->agp_mem);
2191         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2192
2193         i915_gem_object_put_pages(obj);
2194         BUG_ON(obj_priv->pages_refcount);
2195
2196         i915_gem_info_remove_gtt(dev_priv, obj->size);
2197         list_del_init(&obj_priv->list);
2198
2199         drm_mm_put_block(obj_priv->gtt_space);
2200         obj_priv->gtt_space = NULL;
2201
2202         if (i915_gem_object_is_purgeable(obj_priv))
2203                 i915_gem_object_truncate(obj);
2204
2205         trace_i915_gem_object_unbind(obj);
2206
2207         return ret;
2208 }
2209
2210 static int i915_ring_idle(struct drm_device *dev,
2211                           struct intel_ring_buffer *ring)
2212 {
2213         i915_gem_flush_ring(dev, NULL, ring,
2214                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2215         return i915_wait_request(dev,
2216                                  i915_gem_next_request_seqno(dev, ring),
2217                                  ring);
2218 }
2219
2220 int
2221 i915_gpu_idle(struct drm_device *dev)
2222 {
2223         drm_i915_private_t *dev_priv = dev->dev_private;
2224         bool lists_empty;
2225         int ret;
2226
2227         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2228                        list_empty(&dev_priv->render_ring.active_list) &&
2229                        (!HAS_BSD(dev) ||
2230                         list_empty(&dev_priv->bsd_ring.active_list)));
2231         if (lists_empty)
2232                 return 0;
2233
2234         /* Flush everything onto the inactive list. */
2235         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2236         if (ret)
2237                 return ret;
2238
2239         if (HAS_BSD(dev)) {
2240                 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2241                 if (ret)
2242                         return ret;
2243         }
2244
2245         return 0;
2246 }
2247
2248 static int
2249 i915_gem_object_get_pages(struct drm_gem_object *obj,
2250                           gfp_t gfpmask)
2251 {
2252         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2253         int page_count, i;
2254         struct address_space *mapping;
2255         struct inode *inode;
2256         struct page *page;
2257
2258         BUG_ON(obj_priv->pages_refcount
2259                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2260
2261         if (obj_priv->pages_refcount++ != 0)
2262                 return 0;
2263
2264         /* Get the list of pages out of our struct file.  They'll be pinned
2265          * at this point until we release them.
2266          */
2267         page_count = obj->size / PAGE_SIZE;
2268         BUG_ON(obj_priv->pages != NULL);
2269         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2270         if (obj_priv->pages == NULL) {
2271                 obj_priv->pages_refcount--;
2272                 return -ENOMEM;
2273         }
2274
2275         inode = obj->filp->f_path.dentry->d_inode;
2276         mapping = inode->i_mapping;
2277         for (i = 0; i < page_count; i++) {
2278                 page = read_cache_page_gfp(mapping, i,
2279                                            GFP_HIGHUSER |
2280                                            __GFP_COLD |
2281                                            __GFP_RECLAIMABLE |
2282                                            gfpmask);
2283                 if (IS_ERR(page))
2284                         goto err_pages;
2285
2286                 obj_priv->pages[i] = page;
2287         }
2288
2289         if (obj_priv->tiling_mode != I915_TILING_NONE)
2290                 i915_gem_object_do_bit_17_swizzle(obj);
2291
2292         return 0;
2293
2294 err_pages:
2295         while (i--)
2296                 page_cache_release(obj_priv->pages[i]);
2297
2298         drm_free_large(obj_priv->pages);
2299         obj_priv->pages = NULL;
2300         obj_priv->pages_refcount--;
2301         return PTR_ERR(page);
2302 }
2303
2304 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2305 {
2306         struct drm_gem_object *obj = reg->obj;
2307         struct drm_device *dev = obj->dev;
2308         drm_i915_private_t *dev_priv = dev->dev_private;
2309         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2310         int regnum = obj_priv->fence_reg;
2311         uint64_t val;
2312
2313         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2314                     0xfffff000) << 32;
2315         val |= obj_priv->gtt_offset & 0xfffff000;
2316         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2317                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2318
2319         if (obj_priv->tiling_mode == I915_TILING_Y)
2320                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2321         val |= I965_FENCE_REG_VALID;
2322
2323         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2324 }
2325
2326 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2327 {
2328         struct drm_gem_object *obj = reg->obj;
2329         struct drm_device *dev = obj->dev;
2330         drm_i915_private_t *dev_priv = dev->dev_private;
2331         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2332         int regnum = obj_priv->fence_reg;
2333         uint64_t val;
2334
2335         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2336                     0xfffff000) << 32;
2337         val |= obj_priv->gtt_offset & 0xfffff000;
2338         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2339         if (obj_priv->tiling_mode == I915_TILING_Y)
2340                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2341         val |= I965_FENCE_REG_VALID;
2342
2343         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2344 }
2345
2346 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2347 {
2348         struct drm_gem_object *obj = reg->obj;
2349         struct drm_device *dev = obj->dev;
2350         drm_i915_private_t *dev_priv = dev->dev_private;
2351         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2352         int regnum = obj_priv->fence_reg;
2353         int tile_width;
2354         uint32_t fence_reg, val;
2355         uint32_t pitch_val;
2356
2357         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2358             (obj_priv->gtt_offset & (obj->size - 1))) {
2359                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2360                      __func__, obj_priv->gtt_offset, obj->size);
2361                 return;
2362         }
2363
2364         if (obj_priv->tiling_mode == I915_TILING_Y &&
2365             HAS_128_BYTE_Y_TILING(dev))
2366                 tile_width = 128;
2367         else
2368                 tile_width = 512;
2369
2370         /* Note: pitch better be a power of two tile widths */
2371         pitch_val = obj_priv->stride / tile_width;
2372         pitch_val = ffs(pitch_val) - 1;
2373
2374         if (obj_priv->tiling_mode == I915_TILING_Y &&
2375             HAS_128_BYTE_Y_TILING(dev))
2376                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2377         else
2378                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2379
2380         val = obj_priv->gtt_offset;
2381         if (obj_priv->tiling_mode == I915_TILING_Y)
2382                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2383         val |= I915_FENCE_SIZE_BITS(obj->size);
2384         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2385         val |= I830_FENCE_REG_VALID;
2386
2387         if (regnum < 8)
2388                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2389         else
2390                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2391         I915_WRITE(fence_reg, val);
2392 }
2393
2394 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2395 {
2396         struct drm_gem_object *obj = reg->obj;
2397         struct drm_device *dev = obj->dev;
2398         drm_i915_private_t *dev_priv = dev->dev_private;
2399         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2400         int regnum = obj_priv->fence_reg;
2401         uint32_t val;
2402         uint32_t pitch_val;
2403         uint32_t fence_size_bits;
2404
2405         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2406             (obj_priv->gtt_offset & (obj->size - 1))) {
2407                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2408                      __func__, obj_priv->gtt_offset);
2409                 return;
2410         }
2411
2412         pitch_val = obj_priv->stride / 128;
2413         pitch_val = ffs(pitch_val) - 1;
2414         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2415
2416         val = obj_priv->gtt_offset;
2417         if (obj_priv->tiling_mode == I915_TILING_Y)
2418                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2419         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2420         WARN_ON(fence_size_bits & ~0x00000f00);
2421         val |= fence_size_bits;
2422         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2423         val |= I830_FENCE_REG_VALID;
2424
2425         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2426 }
2427
2428 static int i915_find_fence_reg(struct drm_device *dev,
2429                                bool interruptible)
2430 {
2431         struct drm_i915_fence_reg *reg = NULL;
2432         struct drm_i915_gem_object *obj_priv = NULL;
2433         struct drm_i915_private *dev_priv = dev->dev_private;
2434         struct drm_gem_object *obj = NULL;
2435         int i, avail, ret;
2436
2437         /* First try to find a free reg */
2438         avail = 0;
2439         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2440                 reg = &dev_priv->fence_regs[i];
2441                 if (!reg->obj)
2442                         return i;
2443
2444                 obj_priv = to_intel_bo(reg->obj);
2445                 if (!obj_priv->pin_count)
2446                     avail++;
2447         }
2448
2449         if (avail == 0)
2450                 return -ENOSPC;
2451
2452         /* None available, try to steal one or wait for a user to finish */
2453         i = I915_FENCE_REG_NONE;
2454         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2455                             lru_list) {
2456                 obj = reg->obj;
2457                 obj_priv = to_intel_bo(obj);
2458
2459                 if (obj_priv->pin_count)
2460                         continue;
2461
2462                 /* found one! */
2463                 i = obj_priv->fence_reg;
2464                 break;
2465         }
2466
2467         BUG_ON(i == I915_FENCE_REG_NONE);
2468
2469         /* We only have a reference on obj from the active list. put_fence_reg
2470          * might drop that one, causing a use-after-free in it. So hold a
2471          * private reference to obj like the other callers of put_fence_reg
2472          * (set_tiling ioctl) do. */
2473         drm_gem_object_reference(obj);
2474         ret = i915_gem_object_put_fence_reg(obj, interruptible);
2475         drm_gem_object_unreference(obj);
2476         if (ret != 0)
2477                 return ret;
2478
2479         return i;
2480 }
2481
2482 /**
2483  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2484  * @obj: object to map through a fence reg
2485  *
2486  * When mapping objects through the GTT, userspace wants to be able to write
2487  * to them without having to worry about swizzling if the object is tiled.
2488  *
2489  * This function walks the fence regs looking for a free one for @obj,
2490  * stealing one if it can't find any.
2491  *
2492  * It then sets up the reg based on the object's properties: address, pitch
2493  * and tiling format.
2494  */
2495 int
2496 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2497                               bool interruptible)
2498 {
2499         struct drm_device *dev = obj->dev;
2500         struct drm_i915_private *dev_priv = dev->dev_private;
2501         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2502         struct drm_i915_fence_reg *reg = NULL;
2503         int ret;
2504
2505         /* Just update our place in the LRU if our fence is getting used. */
2506         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2507                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2508                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2509                 return 0;
2510         }
2511
2512         switch (obj_priv->tiling_mode) {
2513         case I915_TILING_NONE:
2514                 WARN(1, "allocating a fence for non-tiled object?\n");
2515                 break;
2516         case I915_TILING_X:
2517                 if (!obj_priv->stride)
2518                         return -EINVAL;
2519                 WARN((obj_priv->stride & (512 - 1)),
2520                      "object 0x%08x is X tiled but has non-512B pitch\n",
2521                      obj_priv->gtt_offset);
2522                 break;
2523         case I915_TILING_Y:
2524                 if (!obj_priv->stride)
2525                         return -EINVAL;
2526                 WARN((obj_priv->stride & (128 - 1)),
2527                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2528                      obj_priv->gtt_offset);
2529                 break;
2530         }
2531
2532         ret = i915_find_fence_reg(dev, interruptible);
2533         if (ret < 0)
2534                 return ret;
2535
2536         obj_priv->fence_reg = ret;
2537         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2538         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2539
2540         reg->obj = obj;
2541
2542         switch (INTEL_INFO(dev)->gen) {
2543         case 6:
2544                 sandybridge_write_fence_reg(reg);
2545                 break;
2546         case 5:
2547         case 4:
2548                 i965_write_fence_reg(reg);
2549                 break;
2550         case 3:
2551                 i915_write_fence_reg(reg);
2552                 break;
2553         case 2:
2554                 i830_write_fence_reg(reg);
2555                 break;
2556         }
2557
2558         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2559                         obj_priv->tiling_mode);
2560
2561         return 0;
2562 }
2563
2564 /**
2565  * i915_gem_clear_fence_reg - clear out fence register info
2566  * @obj: object to clear
2567  *
2568  * Zeroes out the fence register itself and clears out the associated
2569  * data structures in dev_priv and obj_priv.
2570  */
2571 static void
2572 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2573 {
2574         struct drm_device *dev = obj->dev;
2575         drm_i915_private_t *dev_priv = dev->dev_private;
2576         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2577         struct drm_i915_fence_reg *reg =
2578                 &dev_priv->fence_regs[obj_priv->fence_reg];
2579         uint32_t fence_reg;
2580
2581         switch (INTEL_INFO(dev)->gen) {
2582         case 6:
2583                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2584                              (obj_priv->fence_reg * 8), 0);
2585                 break;
2586         case 5:
2587         case 4:
2588                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2589                 break;
2590         case 3:
2591                 if (obj_priv->fence_reg >= 8)
2592                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2593                 else
2594         case 2:
2595                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2596
2597                 I915_WRITE(fence_reg, 0);
2598                 break;
2599         }
2600
2601         reg->obj = NULL;
2602         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2603         list_del_init(&reg->lru_list);
2604 }
2605
2606 /**
2607  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2608  * to the buffer to finish, and then resets the fence register.
2609  * @obj: tiled object holding a fence register.
2610  * @bool: whether the wait upon the fence is interruptible
2611  *
2612  * Zeroes out the fence register itself and clears out the associated
2613  * data structures in dev_priv and obj_priv.
2614  */
2615 int
2616 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2617                               bool interruptible)
2618 {
2619         struct drm_device *dev = obj->dev;
2620         struct drm_i915_private *dev_priv = dev->dev_private;
2621         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2622         struct drm_i915_fence_reg *reg;
2623
2624         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2625                 return 0;
2626
2627         /* If we've changed tiling, GTT-mappings of the object
2628          * need to re-fault to ensure that the correct fence register
2629          * setup is in place.
2630          */
2631         i915_gem_release_mmap(obj);
2632
2633         /* On the i915, GPU access to tiled buffers is via a fence,
2634          * therefore we must wait for any outstanding access to complete
2635          * before clearing the fence.
2636          */
2637         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2638         if (reg->gpu) {
2639                 int ret;
2640
2641                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2642                 if (ret)
2643                         return ret;
2644
2645                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2646                 if (ret)
2647                         return ret;
2648
2649                 reg->gpu = false;
2650         }
2651
2652         i915_gem_object_flush_gtt_write_domain(obj);
2653         i915_gem_clear_fence_reg(obj);
2654
2655         return 0;
2656 }
2657
2658 /**
2659  * Finds free space in the GTT aperture and binds the object there.
2660  */
2661 static int
2662 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2663 {
2664         struct drm_device *dev = obj->dev;
2665         drm_i915_private_t *dev_priv = dev->dev_private;
2666         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2667         struct drm_mm_node *free_space;
2668         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2669         int ret;
2670
2671         if (obj_priv->madv != I915_MADV_WILLNEED) {
2672                 DRM_ERROR("Attempting to bind a purgeable object\n");
2673                 return -EINVAL;
2674         }
2675
2676         if (alignment == 0)
2677                 alignment = i915_gem_get_gtt_alignment(obj);
2678         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2679                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2680                 return -EINVAL;
2681         }
2682
2683         /* If the object is bigger than the entire aperture, reject it early
2684          * before evicting everything in a vain attempt to find space.
2685          */
2686         if (obj->size > dev_priv->mm.gtt_total) {
2687                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2688                 return -E2BIG;
2689         }
2690
2691  search_free:
2692         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2693                                         obj->size, alignment, 0);
2694         if (free_space != NULL) {
2695                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2696                                                        alignment);
2697                 if (obj_priv->gtt_space != NULL)
2698                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2699         }
2700         if (obj_priv->gtt_space == NULL) {
2701                 /* If the gtt is empty and we're still having trouble
2702                  * fitting our object in, we're out of memory.
2703                  */
2704                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2705                 if (ret)
2706                         return ret;
2707
2708                 goto search_free;
2709         }
2710
2711         ret = i915_gem_object_get_pages(obj, gfpmask);
2712         if (ret) {
2713                 drm_mm_put_block(obj_priv->gtt_space);
2714                 obj_priv->gtt_space = NULL;
2715
2716                 if (ret == -ENOMEM) {
2717                         /* first try to clear up some space from the GTT */
2718                         ret = i915_gem_evict_something(dev, obj->size,
2719                                                        alignment);
2720                         if (ret) {
2721                                 /* now try to shrink everyone else */
2722                                 if (gfpmask) {
2723                                         gfpmask = 0;
2724                                         goto search_free;
2725                                 }
2726
2727                                 return ret;
2728                         }
2729
2730                         goto search_free;
2731                 }
2732
2733                 return ret;
2734         }
2735
2736         /* Create an AGP memory structure pointing at our pages, and bind it
2737          * into the GTT.
2738          */
2739         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2740                                                obj_priv->pages,
2741                                                obj->size >> PAGE_SHIFT,
2742                                                obj_priv->gtt_offset,
2743                                                obj_priv->agp_type);
2744         if (obj_priv->agp_mem == NULL) {
2745                 i915_gem_object_put_pages(obj);
2746                 drm_mm_put_block(obj_priv->gtt_space);
2747                 obj_priv->gtt_space = NULL;
2748
2749                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2750                 if (ret)
2751                         return ret;
2752
2753                 goto search_free;
2754         }
2755
2756         /* keep track of bounds object by adding it to the inactive list */
2757         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2758         i915_gem_info_add_gtt(dev_priv, obj->size);
2759
2760         /* Assert that the object is not currently in any GPU domain. As it
2761          * wasn't in the GTT, there shouldn't be any way it could have been in
2762          * a GPU cache
2763          */
2764         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2765         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2766
2767         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2768
2769         return 0;
2770 }
2771
2772 void
2773 i915_gem_clflush_object(struct drm_gem_object *obj)
2774 {
2775         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2776
2777         /* If we don't have a page list set up, then we're not pinned
2778          * to GPU, and we can ignore the cache flush because it'll happen
2779          * again at bind time.
2780          */
2781         if (obj_priv->pages == NULL)
2782                 return;
2783
2784         trace_i915_gem_object_clflush(obj);
2785
2786         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2787 }
2788
2789 /** Flushes any GPU write domain for the object if it's dirty. */
2790 static int
2791 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2792                                        bool pipelined)
2793 {
2794         struct drm_device *dev = obj->dev;
2795         uint32_t old_write_domain;
2796
2797         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2798                 return 0;
2799
2800         /* Queue the GPU write cache flushing we need. */
2801         old_write_domain = obj->write_domain;
2802         i915_gem_flush_ring(dev, NULL,
2803                             to_intel_bo(obj)->ring,
2804                             0, obj->write_domain);
2805         BUG_ON(obj->write_domain);
2806
2807         trace_i915_gem_object_change_domain(obj,
2808                                             obj->read_domains,
2809                                             old_write_domain);
2810
2811         if (pipelined)
2812                 return 0;
2813
2814         return i915_gem_object_wait_rendering(obj, true);
2815 }
2816
2817 /** Flushes the GTT write domain for the object if it's dirty. */
2818 static void
2819 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2820 {
2821         uint32_t old_write_domain;
2822
2823         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2824                 return;
2825
2826         /* No actual flushing is required for the GTT write domain.   Writes
2827          * to it immediately go to main memory as far as we know, so there's
2828          * no chipset flush.  It also doesn't land in render cache.
2829          */
2830         old_write_domain = obj->write_domain;
2831         obj->write_domain = 0;
2832
2833         trace_i915_gem_object_change_domain(obj,
2834                                             obj->read_domains,
2835                                             old_write_domain);
2836 }
2837
2838 /** Flushes the CPU write domain for the object if it's dirty. */
2839 static void
2840 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2841 {
2842         struct drm_device *dev = obj->dev;
2843         uint32_t old_write_domain;
2844
2845         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2846                 return;
2847
2848         i915_gem_clflush_object(obj);
2849         drm_agp_chipset_flush(dev);
2850         old_write_domain = obj->write_domain;
2851         obj->write_domain = 0;
2852
2853         trace_i915_gem_object_change_domain(obj,
2854                                             obj->read_domains,
2855                                             old_write_domain);
2856 }
2857
2858 /**
2859  * Moves a single object to the GTT read, and possibly write domain.
2860  *
2861  * This function returns when the move is complete, including waiting on
2862  * flushes to occur.
2863  */
2864 int
2865 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2866 {
2867         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2868         uint32_t old_write_domain, old_read_domains;
2869         int ret;
2870
2871         /* Not valid to be called on unbound objects. */
2872         if (obj_priv->gtt_space == NULL)
2873                 return -EINVAL;
2874
2875         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2876         if (ret != 0)
2877                 return ret;
2878
2879         i915_gem_object_flush_cpu_write_domain(obj);
2880
2881         if (write) {
2882                 ret = i915_gem_object_wait_rendering(obj, true);
2883                 if (ret)
2884                         return ret;
2885         }
2886
2887         old_write_domain = obj->write_domain;
2888         old_read_domains = obj->read_domains;
2889
2890         /* It should now be out of any other write domains, and we can update
2891          * the domain values for our changes.
2892          */
2893         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2894         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2895         if (write) {
2896                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2897                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2898                 obj_priv->dirty = 1;
2899         }
2900
2901         trace_i915_gem_object_change_domain(obj,
2902                                             old_read_domains,
2903                                             old_write_domain);
2904
2905         return 0;
2906 }
2907
2908 /*
2909  * Prepare buffer for display plane. Use uninterruptible for possible flush
2910  * wait, as in modesetting process we're not supposed to be interrupted.
2911  */
2912 int
2913 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2914                                      bool pipelined)
2915 {
2916         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2917         uint32_t old_read_domains;
2918         int ret;
2919
2920         /* Not valid to be called on unbound objects. */
2921         if (obj_priv->gtt_space == NULL)
2922                 return -EINVAL;
2923
2924         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2925         if (ret)
2926                 return ret;
2927
2928         /* Currently, we are always called from an non-interruptible context. */
2929         if (!pipelined) {
2930                 ret = i915_gem_object_wait_rendering(obj, false);
2931                 if (ret)
2932                         return ret;
2933         }
2934
2935         i915_gem_object_flush_cpu_write_domain(obj);
2936
2937         old_read_domains = obj->read_domains;
2938         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2939
2940         trace_i915_gem_object_change_domain(obj,
2941                                             old_read_domains,
2942                                             obj->write_domain);
2943
2944         return 0;
2945 }
2946
2947 /**
2948  * Moves a single object to the CPU read, and possibly write domain.
2949  *
2950  * This function returns when the move is complete, including waiting on
2951  * flushes to occur.
2952  */
2953 static int
2954 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2955 {
2956         uint32_t old_write_domain, old_read_domains;
2957         int ret;
2958
2959         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2960         if (ret != 0)
2961                 return ret;
2962
2963         i915_gem_object_flush_gtt_write_domain(obj);
2964
2965         /* If we have a partially-valid cache of the object in the CPU,
2966          * finish invalidating it and free the per-page flags.
2967          */
2968         i915_gem_object_set_to_full_cpu_read_domain(obj);
2969
2970         if (write) {
2971                 ret = i915_gem_object_wait_rendering(obj, true);
2972                 if (ret)
2973                         return ret;
2974         }
2975
2976         old_write_domain = obj->write_domain;
2977         old_read_domains = obj->read_domains;
2978
2979         /* Flush the CPU cache if it's still invalid. */
2980         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2981                 i915_gem_clflush_object(obj);
2982
2983                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2984         }
2985
2986         /* It should now be out of any other write domains, and we can update
2987          * the domain values for our changes.
2988          */
2989         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2990
2991         /* If we're writing through the CPU, then the GPU read domains will
2992          * need to be invalidated at next use.
2993          */
2994         if (write) {
2995                 obj->read_domains = I915_GEM_DOMAIN_CPU;
2996                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2997         }
2998
2999         trace_i915_gem_object_change_domain(obj,
3000                                             old_read_domains,
3001                                             old_write_domain);
3002
3003         return 0;
3004 }
3005
3006 /*
3007  * Set the next domain for the specified object. This
3008  * may not actually perform the necessary flushing/invaliding though,
3009  * as that may want to be batched with other set_domain operations
3010  *
3011  * This is (we hope) the only really tricky part of gem. The goal
3012  * is fairly simple -- track which caches hold bits of the object
3013  * and make sure they remain coherent. A few concrete examples may
3014  * help to explain how it works. For shorthand, we use the notation
3015  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3016  * a pair of read and write domain masks.
3017  *
3018  * Case 1: the batch buffer
3019  *
3020  *      1. Allocated
3021  *      2. Written by CPU
3022  *      3. Mapped to GTT
3023  *      4. Read by GPU
3024  *      5. Unmapped from GTT
3025  *      6. Freed
3026  *
3027  *      Let's take these a step at a time
3028  *
3029  *      1. Allocated
3030  *              Pages allocated from the kernel may still have
3031  *              cache contents, so we set them to (CPU, CPU) always.
3032  *      2. Written by CPU (using pwrite)
3033  *              The pwrite function calls set_domain (CPU, CPU) and
3034  *              this function does nothing (as nothing changes)
3035  *      3. Mapped by GTT
3036  *              This function asserts that the object is not
3037  *              currently in any GPU-based read or write domains
3038  *      4. Read by GPU
3039  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3040  *              As write_domain is zero, this function adds in the
3041  *              current read domains (CPU+COMMAND, 0).
3042  *              flush_domains is set to CPU.
3043  *              invalidate_domains is set to COMMAND
3044  *              clflush is run to get data out of the CPU caches
3045  *              then i915_dev_set_domain calls i915_gem_flush to
3046  *              emit an MI_FLUSH and drm_agp_chipset_flush
3047  *      5. Unmapped from GTT
3048  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3049  *              flush_domains and invalidate_domains end up both zero
3050  *              so no flushing/invalidating happens
3051  *      6. Freed
3052  *              yay, done
3053  *
3054  * Case 2: The shared render buffer
3055  *
3056  *      1. Allocated
3057  *      2. Mapped to GTT
3058  *      3. Read/written by GPU
3059  *      4. set_domain to (CPU,CPU)
3060  *      5. Read/written by CPU
3061  *      6. Read/written by GPU
3062  *
3063  *      1. Allocated
3064  *              Same as last example, (CPU, CPU)
3065  *      2. Mapped to GTT
3066  *              Nothing changes (assertions find that it is not in the GPU)
3067  *      3. Read/written by GPU
3068  *              execbuffer calls set_domain (RENDER, RENDER)
3069  *              flush_domains gets CPU
3070  *              invalidate_domains gets GPU
3071  *              clflush (obj)
3072  *              MI_FLUSH and drm_agp_chipset_flush
3073  *      4. set_domain (CPU, CPU)
3074  *              flush_domains gets GPU
3075  *              invalidate_domains gets CPU
3076  *              wait_rendering (obj) to make sure all drawing is complete.
3077  *              This will include an MI_FLUSH to get the data from GPU
3078  *              to memory
3079  *              clflush (obj) to invalidate the CPU cache
3080  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3081  *      5. Read/written by CPU
3082  *              cache lines are loaded and dirtied
3083  *      6. Read written by GPU
3084  *              Same as last GPU access
3085  *
3086  * Case 3: The constant buffer
3087  *
3088  *      1. Allocated
3089  *      2. Written by CPU
3090  *      3. Read by GPU
3091  *      4. Updated (written) by CPU again
3092  *      5. Read by GPU
3093  *
3094  *      1. Allocated
3095  *              (CPU, CPU)
3096  *      2. Written by CPU
3097  *              (CPU, CPU)
3098  *      3. Read by GPU
3099  *              (CPU+RENDER, 0)
3100  *              flush_domains = CPU
3101  *              invalidate_domains = RENDER
3102  *              clflush (obj)
3103  *              MI_FLUSH
3104  *              drm_agp_chipset_flush
3105  *      4. Updated (written) by CPU again
3106  *              (CPU, CPU)
3107  *              flush_domains = 0 (no previous write domain)
3108  *              invalidate_domains = 0 (no new read domains)
3109  *      5. Read by GPU
3110  *              (CPU+RENDER, 0)
3111  *              flush_domains = CPU
3112  *              invalidate_domains = RENDER
3113  *              clflush (obj)
3114  *              MI_FLUSH
3115  *              drm_agp_chipset_flush
3116  */
3117 static void
3118 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3119 {
3120         struct drm_device               *dev = obj->dev;
3121         struct drm_i915_private         *dev_priv = dev->dev_private;
3122         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3123         uint32_t                        invalidate_domains = 0;
3124         uint32_t                        flush_domains = 0;
3125         uint32_t                        old_read_domains;
3126
3127         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3128         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3129
3130         intel_mark_busy(dev, obj);
3131
3132         /*
3133          * If the object isn't moving to a new write domain,
3134          * let the object stay in multiple read domains
3135          */
3136         if (obj->pending_write_domain == 0)
3137                 obj->pending_read_domains |= obj->read_domains;
3138         else
3139                 obj_priv->dirty = 1;
3140
3141         /*
3142          * Flush the current write domain if
3143          * the new read domains don't match. Invalidate
3144          * any read domains which differ from the old
3145          * write domain
3146          */
3147         if (obj->write_domain &&
3148             obj->write_domain != obj->pending_read_domains) {
3149                 flush_domains |= obj->write_domain;
3150                 invalidate_domains |=
3151                         obj->pending_read_domains & ~obj->write_domain;
3152         }
3153         /*
3154          * Invalidate any read caches which may have
3155          * stale data. That is, any new read domains.
3156          */
3157         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3158         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3159                 i915_gem_clflush_object(obj);
3160
3161         old_read_domains = obj->read_domains;
3162
3163         /* The actual obj->write_domain will be updated with
3164          * pending_write_domain after we emit the accumulated flush for all
3165          * of our domain changes in execbuffers (which clears objects'
3166          * write_domains).  So if we have a current write domain that we
3167          * aren't changing, set pending_write_domain to that.
3168          */
3169         if (flush_domains == 0 && obj->pending_write_domain == 0)
3170                 obj->pending_write_domain = obj->write_domain;
3171         obj->read_domains = obj->pending_read_domains;
3172
3173         dev->invalidate_domains |= invalidate_domains;
3174         dev->flush_domains |= flush_domains;
3175         if (obj_priv->ring)
3176                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3177
3178         trace_i915_gem_object_change_domain(obj,
3179                                             old_read_domains,
3180                                             obj->write_domain);
3181 }
3182
3183 /**
3184  * Moves the object from a partially CPU read to a full one.
3185  *
3186  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3187  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3188  */
3189 static void
3190 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3191 {
3192         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3193
3194         if (!obj_priv->page_cpu_valid)
3195                 return;
3196
3197         /* If we're partially in the CPU read domain, finish moving it in.
3198          */
3199         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3200                 int i;
3201
3202                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3203                         if (obj_priv->page_cpu_valid[i])
3204                                 continue;
3205                         drm_clflush_pages(obj_priv->pages + i, 1);
3206                 }
3207         }
3208
3209         /* Free the page_cpu_valid mappings which are now stale, whether
3210          * or not we've got I915_GEM_DOMAIN_CPU.
3211          */
3212         kfree(obj_priv->page_cpu_valid);
3213         obj_priv->page_cpu_valid = NULL;
3214 }
3215
3216 /**
3217  * Set the CPU read domain on a range of the object.
3218  *
3219  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3220  * not entirely valid.  The page_cpu_valid member of the object flags which
3221  * pages have been flushed, and will be respected by
3222  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3223  * of the whole object.
3224  *
3225  * This function returns when the move is complete, including waiting on
3226  * flushes to occur.
3227  */
3228 static int
3229 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3230                                           uint64_t offset, uint64_t size)
3231 {
3232         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3233         uint32_t old_read_domains;
3234         int i, ret;
3235
3236         if (offset == 0 && size == obj->size)
3237                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3238
3239         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3240         if (ret != 0)
3241                 return ret;
3242         i915_gem_object_flush_gtt_write_domain(obj);
3243
3244         /* If we're already fully in the CPU read domain, we're done. */
3245         if (obj_priv->page_cpu_valid == NULL &&
3246             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3247                 return 0;
3248
3249         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3250          * newly adding I915_GEM_DOMAIN_CPU
3251          */
3252         if (obj_priv->page_cpu_valid == NULL) {
3253                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3254                                                    GFP_KERNEL);
3255                 if (obj_priv->page_cpu_valid == NULL)
3256                         return -ENOMEM;
3257         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3258                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3259
3260         /* Flush the cache on any pages that are still invalid from the CPU's
3261          * perspective.
3262          */
3263         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3264              i++) {
3265                 if (obj_priv->page_cpu_valid[i])
3266                         continue;
3267
3268                 drm_clflush_pages(obj_priv->pages + i, 1);
3269
3270                 obj_priv->page_cpu_valid[i] = 1;
3271         }
3272
3273         /* It should now be out of any other write domains, and we can update
3274          * the domain values for our changes.
3275          */
3276         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3277
3278         old_read_domains = obj->read_domains;
3279         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3280
3281         trace_i915_gem_object_change_domain(obj,
3282                                             old_read_domains,
3283                                             obj->write_domain);
3284
3285         return 0;
3286 }
3287
3288 /**
3289  * Pin an object to the GTT and evaluate the relocations landing in it.
3290  */
3291 static int
3292 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3293                                  struct drm_file *file_priv,
3294                                  struct drm_i915_gem_exec_object2 *entry,
3295                                  struct drm_i915_gem_relocation_entry *relocs)
3296 {
3297         struct drm_device *dev = obj->dev;
3298         drm_i915_private_t *dev_priv = dev->dev_private;
3299         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3300         int i, ret;
3301         void __iomem *reloc_page;
3302         bool need_fence;
3303
3304         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3305                      obj_priv->tiling_mode != I915_TILING_NONE;
3306
3307         /* Check fence reg constraints and rebind if necessary */
3308         if (need_fence &&
3309             !i915_gem_object_fence_offset_ok(obj,
3310                                              obj_priv->tiling_mode)) {
3311                 ret = i915_gem_object_unbind(obj);
3312                 if (ret)
3313                         return ret;
3314         }
3315
3316         /* Choose the GTT offset for our buffer and put it there. */
3317         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3318         if (ret)
3319                 return ret;
3320
3321         /*
3322          * Pre-965 chips need a fence register set up in order to
3323          * properly handle blits to/from tiled surfaces.
3324          */
3325         if (need_fence) {
3326                 ret = i915_gem_object_get_fence_reg(obj, true);
3327                 if (ret != 0) {
3328                         i915_gem_object_unpin(obj);
3329                         return ret;
3330                 }
3331
3332                 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
3333         }
3334
3335         entry->offset = obj_priv->gtt_offset;
3336
3337         /* Apply the relocations, using the GTT aperture to avoid cache
3338          * flushing requirements.
3339          */
3340         for (i = 0; i < entry->relocation_count; i++) {
3341                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3342                 struct drm_gem_object *target_obj;
3343                 struct drm_i915_gem_object *target_obj_priv;
3344                 uint32_t reloc_val, reloc_offset;
3345                 uint32_t __iomem *reloc_entry;
3346
3347                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3348                                                    reloc->target_handle);
3349                 if (target_obj == NULL) {
3350                         i915_gem_object_unpin(obj);
3351                         return -ENOENT;
3352                 }
3353                 target_obj_priv = to_intel_bo(target_obj);
3354
3355 #if WATCH_RELOC
3356                 DRM_INFO("%s: obj %p offset %08x target %d "
3357                          "read %08x write %08x gtt %08x "
3358                          "presumed %08x delta %08x\n",
3359                          __func__,
3360                          obj,
3361                          (int) reloc->offset,
3362                          (int) reloc->target_handle,
3363                          (int) reloc->read_domains,
3364                          (int) reloc->write_domain,
3365                          (int) target_obj_priv->gtt_offset,
3366                          (int) reloc->presumed_offset,
3367                          reloc->delta);
3368 #endif
3369
3370                 /* The target buffer should have appeared before us in the
3371                  * exec_object list, so it should have a GTT space bound by now.
3372                  */
3373                 if (target_obj_priv->gtt_space == NULL) {
3374                         DRM_ERROR("No GTT space found for object %d\n",
3375                                   reloc->target_handle);
3376                         drm_gem_object_unreference(target_obj);
3377                         i915_gem_object_unpin(obj);
3378                         return -EINVAL;
3379                 }
3380
3381                 /* Validate that the target is in a valid r/w GPU domain */
3382                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3383                         DRM_ERROR("reloc with multiple write domains: "
3384                                   "obj %p target %d offset %d "
3385                                   "read %08x write %08x",
3386                                   obj, reloc->target_handle,
3387                                   (int) reloc->offset,
3388                                   reloc->read_domains,
3389                                   reloc->write_domain);
3390                         drm_gem_object_unreference(target_obj);
3391                         i915_gem_object_unpin(obj);
3392                         return -EINVAL;
3393                 }
3394                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3395                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3396                         DRM_ERROR("reloc with read/write CPU domains: "
3397                                   "obj %p target %d offset %d "
3398                                   "read %08x write %08x",
3399                                   obj, reloc->target_handle,
3400                                   (int) reloc->offset,
3401                                   reloc->read_domains,
3402                                   reloc->write_domain);
3403                         drm_gem_object_unreference(target_obj);
3404                         i915_gem_object_unpin(obj);
3405                         return -EINVAL;
3406                 }
3407                 if (reloc->write_domain && target_obj->pending_write_domain &&
3408                     reloc->write_domain != target_obj->pending_write_domain) {
3409                         DRM_ERROR("Write domain conflict: "
3410                                   "obj %p target %d offset %d "
3411                                   "new %08x old %08x\n",
3412                                   obj, reloc->target_handle,
3413                                   (int) reloc->offset,
3414                                   reloc->write_domain,
3415                                   target_obj->pending_write_domain);
3416                         drm_gem_object_unreference(target_obj);
3417                         i915_gem_object_unpin(obj);
3418                         return -EINVAL;
3419                 }
3420
3421                 target_obj->pending_read_domains |= reloc->read_domains;
3422                 target_obj->pending_write_domain |= reloc->write_domain;
3423
3424                 /* If the relocation already has the right value in it, no
3425                  * more work needs to be done.
3426                  */
3427                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3428                         drm_gem_object_unreference(target_obj);
3429                         continue;
3430                 }
3431
3432                 /* Check that the relocation address is valid... */
3433                 if (reloc->offset > obj->size - 4) {
3434                         DRM_ERROR("Relocation beyond object bounds: "
3435                                   "obj %p target %d offset %d size %d.\n",
3436                                   obj, reloc->target_handle,
3437                                   (int) reloc->offset, (int) obj->size);
3438                         drm_gem_object_unreference(target_obj);
3439                         i915_gem_object_unpin(obj);
3440                         return -EINVAL;
3441                 }
3442                 if (reloc->offset & 3) {
3443                         DRM_ERROR("Relocation not 4-byte aligned: "
3444                                   "obj %p target %d offset %d.\n",
3445                                   obj, reloc->target_handle,
3446                                   (int) reloc->offset);
3447                         drm_gem_object_unreference(target_obj);
3448                         i915_gem_object_unpin(obj);
3449                         return -EINVAL;
3450                 }
3451
3452                 /* and points to somewhere within the target object. */
3453                 if (reloc->delta >= target_obj->size) {
3454                         DRM_ERROR("Relocation beyond target object bounds: "
3455                                   "obj %p target %d delta %d size %d.\n",
3456                                   obj, reloc->target_handle,
3457                                   (int) reloc->delta, (int) target_obj->size);
3458                         drm_gem_object_unreference(target_obj);
3459                         i915_gem_object_unpin(obj);
3460                         return -EINVAL;
3461                 }
3462
3463                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3464                 if (ret != 0) {
3465                         drm_gem_object_unreference(target_obj);
3466                         i915_gem_object_unpin(obj);
3467                         return ret;
3468                 }
3469
3470                 /* Map the page containing the relocation we're going to
3471                  * perform.
3472                  */
3473                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3474                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3475                                                       (reloc_offset &
3476                                                        ~(PAGE_SIZE - 1)),
3477                                                       KM_USER0);
3478                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3479                                                    (reloc_offset & (PAGE_SIZE - 1)));
3480                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3481
3482                 writel(reloc_val, reloc_entry);
3483                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3484
3485                 /* The updated presumed offset for this entry will be
3486                  * copied back out to the user.
3487                  */
3488                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3489
3490                 drm_gem_object_unreference(target_obj);
3491         }
3492
3493         return 0;
3494 }
3495
3496 /* Throttle our rendering by waiting until the ring has completed our requests
3497  * emitted over 20 msec ago.
3498  *
3499  * Note that if we were to use the current jiffies each time around the loop,
3500  * we wouldn't escape the function with any frames outstanding if the time to
3501  * render a frame was over 20ms.
3502  *
3503  * This should get us reasonable parallelism between CPU and GPU but also
3504  * relatively low latency when blocking on a particular request to finish.
3505  */
3506 static int
3507 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3508 {
3509         struct drm_i915_private *dev_priv = dev->dev_private;
3510         struct drm_i915_file_private *file_priv = file->driver_priv;
3511         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3512         struct drm_i915_gem_request *request;
3513         struct intel_ring_buffer *ring = NULL;
3514         u32 seqno = 0;
3515         int ret;
3516
3517         spin_lock(&file_priv->mm.lock);
3518         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3519                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3520                         break;
3521
3522                 ring = request->ring;
3523                 seqno = request->seqno;
3524         }
3525         spin_unlock(&file_priv->mm.lock);
3526
3527         if (seqno == 0)
3528                 return 0;
3529
3530         ret = 0;
3531         if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3532                 /* And wait for the seqno passing without holding any locks and
3533                  * causing extra latency for others. This is safe as the irq
3534                  * generation is designed to be run atomically and so is
3535                  * lockless.
3536                  */
3537                 ring->user_irq_get(dev, ring);
3538                 ret = wait_event_interruptible(ring->irq_queue,
3539                                                i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3540                                                || atomic_read(&dev_priv->mm.wedged));
3541                 ring->user_irq_put(dev, ring);
3542
3543                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3544                         ret = -EIO;
3545         }
3546
3547         if (ret == 0)
3548                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3549
3550         return ret;
3551 }
3552
3553 static int
3554 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3555                               uint32_t buffer_count,
3556                               struct drm_i915_gem_relocation_entry **relocs)
3557 {
3558         uint32_t reloc_count = 0, reloc_index = 0, i;
3559         int ret;
3560
3561         *relocs = NULL;
3562         for (i = 0; i < buffer_count; i++) {
3563                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3564                         return -EINVAL;
3565                 reloc_count += exec_list[i].relocation_count;
3566         }
3567
3568         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3569         if (*relocs == NULL) {
3570                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3571                 return -ENOMEM;
3572         }
3573
3574         for (i = 0; i < buffer_count; i++) {
3575                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3576
3577                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3578
3579                 ret = copy_from_user(&(*relocs)[reloc_index],
3580                                      user_relocs,
3581                                      exec_list[i].relocation_count *
3582                                      sizeof(**relocs));
3583                 if (ret != 0) {
3584                         drm_free_large(*relocs);
3585                         *relocs = NULL;
3586                         return -EFAULT;
3587                 }
3588
3589                 reloc_index += exec_list[i].relocation_count;
3590         }
3591
3592         return 0;
3593 }
3594
3595 static int
3596 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3597                             uint32_t buffer_count,
3598                             struct drm_i915_gem_relocation_entry *relocs)
3599 {
3600         uint32_t reloc_count = 0, i;
3601         int ret = 0;
3602
3603         if (relocs == NULL)
3604             return 0;
3605
3606         for (i = 0; i < buffer_count; i++) {
3607                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3608                 int unwritten;
3609
3610                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3611
3612                 unwritten = copy_to_user(user_relocs,
3613                                          &relocs[reloc_count],
3614                                          exec_list[i].relocation_count *
3615                                          sizeof(*relocs));
3616
3617                 if (unwritten) {
3618                         ret = -EFAULT;
3619                         goto err;
3620                 }
3621
3622                 reloc_count += exec_list[i].relocation_count;
3623         }
3624
3625 err:
3626         drm_free_large(relocs);
3627
3628         return ret;
3629 }
3630
3631 static int
3632 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3633                            uint64_t exec_offset)
3634 {
3635         uint32_t exec_start, exec_len;
3636
3637         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3638         exec_len = (uint32_t) exec->batch_len;
3639
3640         if ((exec_start | exec_len) & 0x7)
3641                 return -EINVAL;
3642
3643         if (!exec_start)
3644                 return -EINVAL;
3645
3646         return 0;
3647 }
3648
3649 static int
3650 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3651                                struct drm_gem_object **object_list,
3652                                int count)
3653 {
3654         drm_i915_private_t *dev_priv = dev->dev_private;
3655         struct drm_i915_gem_object *obj_priv;
3656         DEFINE_WAIT(wait);
3657         int i, ret = 0;
3658
3659         for (;;) {
3660                 prepare_to_wait(&dev_priv->pending_flip_queue,
3661                                 &wait, TASK_INTERRUPTIBLE);
3662                 for (i = 0; i < count; i++) {
3663                         obj_priv = to_intel_bo(object_list[i]);
3664                         if (atomic_read(&obj_priv->pending_flip) > 0)
3665                                 break;
3666                 }
3667                 if (i == count)
3668                         break;
3669
3670                 if (!signal_pending(current)) {
3671                         mutex_unlock(&dev->struct_mutex);
3672                         schedule();
3673                         mutex_lock(&dev->struct_mutex);
3674                         continue;
3675                 }
3676                 ret = -ERESTARTSYS;
3677                 break;
3678         }
3679         finish_wait(&dev_priv->pending_flip_queue, &wait);
3680
3681         return ret;
3682 }
3683
3684 static int
3685 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3686                        struct drm_file *file_priv,
3687                        struct drm_i915_gem_execbuffer2 *args,
3688                        struct drm_i915_gem_exec_object2 *exec_list)
3689 {
3690         drm_i915_private_t *dev_priv = dev->dev_private;
3691         struct drm_gem_object **object_list = NULL;
3692         struct drm_gem_object *batch_obj;
3693         struct drm_i915_gem_object *obj_priv;
3694         struct drm_clip_rect *cliprects = NULL;
3695         struct drm_i915_gem_relocation_entry *relocs = NULL;
3696         struct drm_i915_gem_request *request = NULL;
3697         int ret, ret2, i, pinned = 0;
3698         uint64_t exec_offset;
3699         uint32_t reloc_index;
3700         int pin_tries, flips;
3701
3702         struct intel_ring_buffer *ring = NULL;
3703
3704         ret = i915_gem_check_is_wedged(dev);
3705         if (ret)
3706                 return ret;
3707
3708 #if WATCH_EXEC
3709         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3710                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3711 #endif
3712         if (args->flags & I915_EXEC_BSD) {
3713                 if (!HAS_BSD(dev)) {
3714                         DRM_ERROR("execbuf with wrong flag\n");
3715                         return -EINVAL;
3716                 }
3717                 ring = &dev_priv->bsd_ring;
3718         } else {
3719                 ring = &dev_priv->render_ring;
3720         }
3721
3722         if (args->buffer_count < 1) {
3723                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3724                 return -EINVAL;
3725         }
3726         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3727         if (object_list == NULL) {
3728                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3729                           args->buffer_count);
3730                 ret = -ENOMEM;
3731                 goto pre_mutex_err;
3732         }
3733
3734         if (args->num_cliprects != 0) {
3735                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3736                                     GFP_KERNEL);
3737                 if (cliprects == NULL) {
3738                         ret = -ENOMEM;
3739                         goto pre_mutex_err;
3740                 }
3741
3742                 ret = copy_from_user(cliprects,
3743                                      (struct drm_clip_rect __user *)
3744                                      (uintptr_t) args->cliprects_ptr,
3745                                      sizeof(*cliprects) * args->num_cliprects);
3746                 if (ret != 0) {
3747                         DRM_ERROR("copy %d cliprects failed: %d\n",
3748                                   args->num_cliprects, ret);
3749                         ret = -EFAULT;
3750                         goto pre_mutex_err;
3751                 }
3752         }
3753
3754         request = kzalloc(sizeof(*request), GFP_KERNEL);
3755         if (request == NULL) {
3756                 ret = -ENOMEM;
3757                 goto pre_mutex_err;
3758         }
3759
3760         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3761                                             &relocs);
3762         if (ret != 0)
3763                 goto pre_mutex_err;
3764
3765         ret = i915_mutex_lock_interruptible(dev);
3766         if (ret)
3767                 goto pre_mutex_err;
3768
3769         if (dev_priv->mm.suspended) {
3770                 mutex_unlock(&dev->struct_mutex);
3771                 ret = -EBUSY;
3772                 goto pre_mutex_err;
3773         }
3774
3775         /* Look up object handles */
3776         flips = 0;
3777         for (i = 0; i < args->buffer_count; i++) {
3778                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3779                                                        exec_list[i].handle);
3780                 if (object_list[i] == NULL) {
3781                         DRM_ERROR("Invalid object handle %d at index %d\n",
3782                                    exec_list[i].handle, i);
3783                         /* prevent error path from reading uninitialized data */
3784                         args->buffer_count = i + 1;
3785                         ret = -ENOENT;
3786                         goto err;
3787                 }
3788
3789                 obj_priv = to_intel_bo(object_list[i]);
3790                 if (obj_priv->in_execbuffer) {
3791                         DRM_ERROR("Object %p appears more than once in object list\n",
3792                                    object_list[i]);
3793                         /* prevent error path from reading uninitialized data */
3794                         args->buffer_count = i + 1;
3795                         ret = -EINVAL;
3796                         goto err;
3797                 }
3798                 obj_priv->in_execbuffer = true;
3799                 flips += atomic_read(&obj_priv->pending_flip);
3800         }
3801
3802         if (flips > 0) {
3803                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3804                                                      args->buffer_count);
3805                 if (ret)
3806                         goto err;
3807         }
3808
3809         /* Pin and relocate */
3810         for (pin_tries = 0; ; pin_tries++) {
3811                 ret = 0;
3812                 reloc_index = 0;
3813
3814                 for (i = 0; i < args->buffer_count; i++) {
3815                         object_list[i]->pending_read_domains = 0;
3816                         object_list[i]->pending_write_domain = 0;
3817                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3818                                                                file_priv,
3819                                                                &exec_list[i],
3820                                                                &relocs[reloc_index]);
3821                         if (ret)
3822                                 break;
3823                         pinned = i + 1;
3824                         reloc_index += exec_list[i].relocation_count;
3825                 }
3826                 /* success */
3827                 if (ret == 0)
3828                         break;
3829
3830                 /* error other than GTT full, or we've already tried again */
3831                 if (ret != -ENOSPC || pin_tries >= 1) {
3832                         if (ret != -ERESTARTSYS) {
3833                                 unsigned long long total_size = 0;
3834                                 int num_fences = 0;
3835                                 for (i = 0; i < args->buffer_count; i++) {
3836                                         obj_priv = to_intel_bo(object_list[i]);
3837
3838                                         total_size += object_list[i]->size;
3839                                         num_fences +=
3840                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3841                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3842                                 }
3843                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3844                                           pinned+1, args->buffer_count,
3845                                           total_size, num_fences,
3846                                           ret);
3847                                 DRM_ERROR("%u objects [%u pinned, %u GTT], "
3848                                           "%zu object bytes [%zu pinned], "
3849                                           "%zu /%zu gtt bytes\n",
3850                                           dev_priv->mm.object_count,
3851                                           dev_priv->mm.pin_count,
3852                                           dev_priv->mm.gtt_count,
3853                                           dev_priv->mm.object_memory,
3854                                           dev_priv->mm.pin_memory,
3855                                           dev_priv->mm.gtt_memory,
3856                                           dev_priv->mm.gtt_total);
3857                         }
3858                         goto err;
3859                 }
3860
3861                 /* unpin all of our buffers */
3862                 for (i = 0; i < pinned; i++)
3863                         i915_gem_object_unpin(object_list[i]);
3864                 pinned = 0;
3865
3866                 /* evict everyone we can from the aperture */
3867                 ret = i915_gem_evict_everything(dev);
3868                 if (ret && ret != -ENOSPC)
3869                         goto err;
3870         }
3871
3872         /* Set the pending read domains for the batch buffer to COMMAND */
3873         batch_obj = object_list[args->buffer_count-1];
3874         if (batch_obj->pending_write_domain) {
3875                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3876                 ret = -EINVAL;
3877                 goto err;
3878         }
3879         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3880
3881         /* Sanity check the batch buffer, prior to moving objects */
3882         exec_offset = exec_list[args->buffer_count - 1].offset;
3883         ret = i915_gem_check_execbuffer (args, exec_offset);
3884         if (ret != 0) {
3885                 DRM_ERROR("execbuf with invalid offset/length\n");
3886                 goto err;
3887         }
3888
3889         /* Zero the global flush/invalidate flags. These
3890          * will be modified as new domains are computed
3891          * for each object
3892          */
3893         dev->invalidate_domains = 0;
3894         dev->flush_domains = 0;
3895         dev_priv->mm.flush_rings = 0;
3896
3897         for (i = 0; i < args->buffer_count; i++) {
3898                 struct drm_gem_object *obj = object_list[i];
3899
3900                 /* Compute new gpu domains and update invalidate/flush */
3901                 i915_gem_object_set_to_gpu_domain(obj);
3902         }
3903
3904         if (dev->invalidate_domains | dev->flush_domains) {
3905 #if WATCH_EXEC
3906                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3907                           __func__,
3908                          dev->invalidate_domains,
3909                          dev->flush_domains);
3910 #endif
3911                 i915_gem_flush(dev, file_priv,
3912                                dev->invalidate_domains,
3913                                dev->flush_domains,
3914                                dev_priv->mm.flush_rings);
3915         }
3916
3917         for (i = 0; i < args->buffer_count; i++) {
3918                 struct drm_gem_object *obj = object_list[i];
3919                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3920                 uint32_t old_write_domain = obj->write_domain;
3921
3922                 obj->write_domain = obj->pending_write_domain;
3923                 if (obj->write_domain)
3924                         list_move_tail(&obj_priv->gpu_write_list,
3925                                        &dev_priv->mm.gpu_write_list);
3926
3927                 trace_i915_gem_object_change_domain(obj,
3928                                                     obj->read_domains,
3929                                                     old_write_domain);
3930         }
3931
3932 #if WATCH_COHERENCY
3933         for (i = 0; i < args->buffer_count; i++) {
3934                 i915_gem_object_check_coherency(object_list[i],
3935                                                 exec_list[i].handle);
3936         }
3937 #endif
3938
3939 #if WATCH_EXEC
3940         i915_gem_dump_object(batch_obj,
3941                               args->batch_len,
3942                               __func__,
3943                               ~0);
3944 #endif
3945
3946         /* Exec the batchbuffer */
3947         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3948                         cliprects, exec_offset);
3949         if (ret) {
3950                 DRM_ERROR("dispatch failed %d\n", ret);
3951                 goto err;
3952         }
3953
3954         /*
3955          * Ensure that the commands in the batch buffer are
3956          * finished before the interrupt fires
3957          */
3958         i915_retire_commands(dev, ring);
3959
3960         for (i = 0; i < args->buffer_count; i++) {
3961                 struct drm_gem_object *obj = object_list[i];
3962                 obj_priv = to_intel_bo(obj);
3963
3964                 i915_gem_object_move_to_active(obj, ring);
3965         }
3966
3967         i915_add_request(dev, file_priv, request, ring);
3968         request = NULL;
3969
3970 err:
3971         for (i = 0; i < pinned; i++)
3972                 i915_gem_object_unpin(object_list[i]);
3973
3974         for (i = 0; i < args->buffer_count; i++) {
3975                 if (object_list[i]) {
3976                         obj_priv = to_intel_bo(object_list[i]);
3977                         obj_priv->in_execbuffer = false;
3978                 }
3979                 drm_gem_object_unreference(object_list[i]);
3980         }
3981
3982         mutex_unlock(&dev->struct_mutex);
3983
3984 pre_mutex_err:
3985         /* Copy the updated relocations out regardless of current error
3986          * state.  Failure to update the relocs would mean that the next
3987          * time userland calls execbuf, it would do so with presumed offset
3988          * state that didn't match the actual object state.
3989          */
3990         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3991                                            relocs);
3992         if (ret2 != 0) {
3993                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3994
3995                 if (ret == 0)
3996                         ret = ret2;
3997         }
3998
3999         drm_free_large(object_list);
4000         kfree(cliprects);
4001         kfree(request);
4002
4003         return ret;
4004 }
4005
4006 /*
4007  * Legacy execbuffer just creates an exec2 list from the original exec object
4008  * list array and passes it to the real function.
4009  */
4010 int
4011 i915_gem_execbuffer(struct drm_device *dev, void *data,
4012                     struct drm_file *file_priv)
4013 {
4014         struct drm_i915_gem_execbuffer *args = data;
4015         struct drm_i915_gem_execbuffer2 exec2;
4016         struct drm_i915_gem_exec_object *exec_list = NULL;
4017         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4018         int ret, i;
4019
4020 #if WATCH_EXEC
4021         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4022                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4023 #endif
4024
4025         if (args->buffer_count < 1) {
4026                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4027                 return -EINVAL;
4028         }
4029
4030         /* Copy in the exec list from userland */
4031         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4032         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4033         if (exec_list == NULL || exec2_list == NULL) {
4034                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4035                           args->buffer_count);
4036                 drm_free_large(exec_list);
4037                 drm_free_large(exec2_list);
4038                 return -ENOMEM;
4039         }
4040         ret = copy_from_user(exec_list,
4041                              (struct drm_i915_relocation_entry __user *)
4042                              (uintptr_t) args->buffers_ptr,
4043                              sizeof(*exec_list) * args->buffer_count);
4044         if (ret != 0) {
4045                 DRM_ERROR("copy %d exec entries failed %d\n",
4046                           args->buffer_count, ret);
4047                 drm_free_large(exec_list);
4048                 drm_free_large(exec2_list);
4049                 return -EFAULT;
4050         }
4051
4052         for (i = 0; i < args->buffer_count; i++) {
4053                 exec2_list[i].handle = exec_list[i].handle;
4054                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4055                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4056                 exec2_list[i].alignment = exec_list[i].alignment;
4057                 exec2_list[i].offset = exec_list[i].offset;
4058                 if (INTEL_INFO(dev)->gen < 4)
4059                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4060                 else
4061                         exec2_list[i].flags = 0;
4062         }
4063
4064         exec2.buffers_ptr = args->buffers_ptr;
4065         exec2.buffer_count = args->buffer_count;
4066         exec2.batch_start_offset = args->batch_start_offset;
4067         exec2.batch_len = args->batch_len;
4068         exec2.DR1 = args->DR1;
4069         exec2.DR4 = args->DR4;
4070         exec2.num_cliprects = args->num_cliprects;
4071         exec2.cliprects_ptr = args->cliprects_ptr;
4072         exec2.flags = I915_EXEC_RENDER;
4073
4074         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4075         if (!ret) {
4076                 /* Copy the new buffer offsets back to the user's exec list. */
4077                 for (i = 0; i < args->buffer_count; i++)
4078                         exec_list[i].offset = exec2_list[i].offset;
4079                 /* ... and back out to userspace */
4080                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4081                                    (uintptr_t) args->buffers_ptr,
4082                                    exec_list,
4083                                    sizeof(*exec_list) * args->buffer_count);
4084                 if (ret) {
4085                         ret = -EFAULT;
4086                         DRM_ERROR("failed to copy %d exec entries "
4087                                   "back to user (%d)\n",
4088                                   args->buffer_count, ret);
4089                 }
4090         }
4091
4092         drm_free_large(exec_list);
4093         drm_free_large(exec2_list);
4094         return ret;
4095 }
4096
4097 int
4098 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4099                      struct drm_file *file_priv)
4100 {
4101         struct drm_i915_gem_execbuffer2 *args = data;
4102         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4103         int ret;
4104
4105 #if WATCH_EXEC
4106         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4107                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4108 #endif
4109
4110         if (args->buffer_count < 1) {
4111                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4112                 return -EINVAL;
4113         }
4114
4115         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4116         if (exec2_list == NULL) {
4117                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4118                           args->buffer_count);
4119                 return -ENOMEM;
4120         }
4121         ret = copy_from_user(exec2_list,
4122                              (struct drm_i915_relocation_entry __user *)
4123                              (uintptr_t) args->buffers_ptr,
4124                              sizeof(*exec2_list) * args->buffer_count);
4125         if (ret != 0) {
4126                 DRM_ERROR("copy %d exec entries failed %d\n",
4127                           args->buffer_count, ret);
4128                 drm_free_large(exec2_list);
4129                 return -EFAULT;
4130         }
4131
4132         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4133         if (!ret) {
4134                 /* Copy the new buffer offsets back to the user's exec list. */
4135                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4136                                    (uintptr_t) args->buffers_ptr,
4137                                    exec2_list,
4138                                    sizeof(*exec2_list) * args->buffer_count);
4139                 if (ret) {
4140                         ret = -EFAULT;
4141                         DRM_ERROR("failed to copy %d exec entries "
4142                                   "back to user (%d)\n",
4143                                   args->buffer_count, ret);
4144                 }
4145         }
4146
4147         drm_free_large(exec2_list);
4148         return ret;
4149 }
4150
4151 int
4152 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4153 {
4154         struct drm_device *dev = obj->dev;
4155         struct drm_i915_private *dev_priv = dev->dev_private;
4156         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4157         int ret;
4158
4159         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4160         WARN_ON(i915_verify_lists(dev));
4161
4162         if (obj_priv->gtt_space != NULL) {
4163                 if (alignment == 0)
4164                         alignment = i915_gem_get_gtt_alignment(obj);
4165                 if (obj_priv->gtt_offset & (alignment - 1)) {
4166                         WARN(obj_priv->pin_count,
4167                              "bo is already pinned with incorrect alignment:"
4168                              " offset=%x, req.alignment=%x\n",
4169                              obj_priv->gtt_offset, alignment);
4170                         ret = i915_gem_object_unbind(obj);
4171                         if (ret)
4172                                 return ret;
4173                 }
4174         }
4175
4176         if (obj_priv->gtt_space == NULL) {
4177                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4178                 if (ret)
4179                         return ret;
4180         }
4181
4182         obj_priv->pin_count++;
4183
4184         /* If the object is not active and not pending a flush,
4185          * remove it from the inactive list
4186          */
4187         if (obj_priv->pin_count == 1) {
4188                 i915_gem_info_add_pin(dev_priv, obj->size);
4189                 if (!obj_priv->active)
4190                         list_move_tail(&obj_priv->list,
4191                                        &dev_priv->mm.pinned_list);
4192         }
4193
4194         WARN_ON(i915_verify_lists(dev));
4195         return 0;
4196 }
4197
4198 void
4199 i915_gem_object_unpin(struct drm_gem_object *obj)
4200 {
4201         struct drm_device *dev = obj->dev;
4202         drm_i915_private_t *dev_priv = dev->dev_private;
4203         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4204
4205         WARN_ON(i915_verify_lists(dev));
4206         obj_priv->pin_count--;
4207         BUG_ON(obj_priv->pin_count < 0);
4208         BUG_ON(obj_priv->gtt_space == NULL);
4209
4210         /* If the object is no longer pinned, and is
4211          * neither active nor being flushed, then stick it on
4212          * the inactive list
4213          */
4214         if (obj_priv->pin_count == 0) {
4215                 if (!obj_priv->active)
4216                         list_move_tail(&obj_priv->list,
4217                                        &dev_priv->mm.inactive_list);
4218                 i915_gem_info_remove_pin(dev_priv, obj->size);
4219         }
4220         WARN_ON(i915_verify_lists(dev));
4221 }
4222
4223 int
4224 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4225                    struct drm_file *file_priv)
4226 {
4227         struct drm_i915_gem_pin *args = data;
4228         struct drm_gem_object *obj;
4229         struct drm_i915_gem_object *obj_priv;
4230         int ret;
4231
4232         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4233         if (obj == NULL) {
4234                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4235                           args->handle);
4236                 return -ENOENT;
4237         }
4238         obj_priv = to_intel_bo(obj);
4239
4240         ret = i915_mutex_lock_interruptible(dev);
4241         if (ret) {
4242                 drm_gem_object_unreference_unlocked(obj);
4243                 return ret;
4244         }
4245
4246         if (obj_priv->madv != I915_MADV_WILLNEED) {
4247                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4248                 drm_gem_object_unreference(obj);
4249                 mutex_unlock(&dev->struct_mutex);
4250                 return -EINVAL;
4251         }
4252
4253         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4254                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4255                           args->handle);
4256                 drm_gem_object_unreference(obj);
4257                 mutex_unlock(&dev->struct_mutex);
4258                 return -EINVAL;
4259         }
4260
4261         obj_priv->user_pin_count++;
4262         obj_priv->pin_filp = file_priv;
4263         if (obj_priv->user_pin_count == 1) {
4264                 ret = i915_gem_object_pin(obj, args->alignment);
4265                 if (ret != 0) {
4266                         drm_gem_object_unreference(obj);
4267                         mutex_unlock(&dev->struct_mutex);
4268                         return ret;
4269                 }
4270         }
4271
4272         /* XXX - flush the CPU caches for pinned objects
4273          * as the X server doesn't manage domains yet
4274          */
4275         i915_gem_object_flush_cpu_write_domain(obj);
4276         args->offset = obj_priv->gtt_offset;
4277         drm_gem_object_unreference(obj);
4278         mutex_unlock(&dev->struct_mutex);
4279
4280         return 0;
4281 }
4282
4283 int
4284 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4285                      struct drm_file *file_priv)
4286 {
4287         struct drm_i915_gem_pin *args = data;
4288         struct drm_gem_object *obj;
4289         struct drm_i915_gem_object *obj_priv;
4290         int ret;
4291
4292         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4293         if (obj == NULL) {
4294                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4295                           args->handle);
4296                 return -ENOENT;
4297         }
4298
4299         obj_priv = to_intel_bo(obj);
4300
4301         ret = i915_mutex_lock_interruptible(dev);
4302         if (ret) {
4303                 drm_gem_object_unreference_unlocked(obj);
4304                 return ret;
4305         }
4306
4307         if (obj_priv->pin_filp != file_priv) {
4308                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4309                           args->handle);
4310                 drm_gem_object_unreference(obj);
4311                 mutex_unlock(&dev->struct_mutex);
4312                 return -EINVAL;
4313         }
4314         obj_priv->user_pin_count--;
4315         if (obj_priv->user_pin_count == 0) {
4316                 obj_priv->pin_filp = NULL;
4317                 i915_gem_object_unpin(obj);
4318         }
4319
4320         drm_gem_object_unreference(obj);
4321         mutex_unlock(&dev->struct_mutex);
4322         return 0;
4323 }
4324
4325 int
4326 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4327                     struct drm_file *file_priv)
4328 {
4329         struct drm_i915_gem_busy *args = data;
4330         struct drm_gem_object *obj;
4331         struct drm_i915_gem_object *obj_priv;
4332         int ret;
4333
4334         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4335         if (obj == NULL) {
4336                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4337                           args->handle);
4338                 return -ENOENT;
4339         }
4340
4341         ret = i915_mutex_lock_interruptible(dev);
4342         if (ret) {
4343                 drm_gem_object_unreference_unlocked(obj);
4344                 return ret;
4345         }
4346
4347         /* Count all active objects as busy, even if they are currently not used
4348          * by the gpu. Users of this interface expect objects to eventually
4349          * become non-busy without any further actions, therefore emit any
4350          * necessary flushes here.
4351          */
4352         obj_priv = to_intel_bo(obj);
4353         args->busy = obj_priv->active;
4354         if (args->busy) {
4355                 /* Unconditionally flush objects, even when the gpu still uses this
4356                  * object. Userspace calling this function indicates that it wants to
4357                  * use this buffer rather sooner than later, so issuing the required
4358                  * flush earlier is beneficial.
4359                  */
4360                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4361                         i915_gem_flush_ring(dev, file_priv,
4362                                             obj_priv->ring,
4363                                             0, obj->write_domain);
4364
4365                 /* Update the active list for the hardware's current position.
4366                  * Otherwise this only updates on a delayed timer or when irqs
4367                  * are actually unmasked, and our working set ends up being
4368                  * larger than required.
4369                  */
4370                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4371
4372                 args->busy = obj_priv->active;
4373         }
4374
4375         drm_gem_object_unreference(obj);
4376         mutex_unlock(&dev->struct_mutex);
4377         return 0;
4378 }
4379
4380 int
4381 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4382                         struct drm_file *file_priv)
4383 {
4384     return i915_gem_ring_throttle(dev, file_priv);
4385 }
4386
4387 int
4388 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4389                        struct drm_file *file_priv)
4390 {
4391         struct drm_i915_gem_madvise *args = data;
4392         struct drm_gem_object *obj;
4393         struct drm_i915_gem_object *obj_priv;
4394         int ret;
4395
4396         switch (args->madv) {
4397         case I915_MADV_DONTNEED:
4398         case I915_MADV_WILLNEED:
4399             break;
4400         default:
4401             return -EINVAL;
4402         }
4403
4404         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4405         if (obj == NULL) {
4406                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4407                           args->handle);
4408                 return -ENOENT;
4409         }
4410         obj_priv = to_intel_bo(obj);
4411
4412         ret = i915_mutex_lock_interruptible(dev);
4413         if (ret) {
4414                 drm_gem_object_unreference_unlocked(obj);
4415                 return ret;
4416         }
4417
4418         if (obj_priv->pin_count) {
4419                 drm_gem_object_unreference(obj);
4420                 mutex_unlock(&dev->struct_mutex);
4421
4422                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4423                 return -EINVAL;
4424         }
4425
4426         if (obj_priv->madv != __I915_MADV_PURGED)
4427                 obj_priv->madv = args->madv;
4428
4429         /* if the object is no longer bound, discard its backing storage */
4430         if (i915_gem_object_is_purgeable(obj_priv) &&
4431             obj_priv->gtt_space == NULL)
4432                 i915_gem_object_truncate(obj);
4433
4434         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4435
4436         drm_gem_object_unreference(obj);
4437         mutex_unlock(&dev->struct_mutex);
4438
4439         return 0;
4440 }
4441
4442 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4443                                               size_t size)
4444 {
4445         struct drm_i915_private *dev_priv = dev->dev_private;
4446         struct drm_i915_gem_object *obj;
4447
4448         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4449         if (obj == NULL)
4450                 return NULL;
4451
4452         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4453                 kfree(obj);
4454                 return NULL;
4455         }
4456
4457         i915_gem_info_add_obj(dev_priv, size);
4458
4459         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4460         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4461
4462         obj->agp_type = AGP_USER_MEMORY;
4463         obj->base.driver_private = NULL;
4464         obj->fence_reg = I915_FENCE_REG_NONE;
4465         INIT_LIST_HEAD(&obj->list);
4466         INIT_LIST_HEAD(&obj->gpu_write_list);
4467         obj->madv = I915_MADV_WILLNEED;
4468
4469         trace_i915_gem_object_create(&obj->base);
4470
4471         return &obj->base;
4472 }
4473
4474 int i915_gem_init_object(struct drm_gem_object *obj)
4475 {
4476         BUG();
4477
4478         return 0;
4479 }
4480
4481 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4482 {
4483         struct drm_device *dev = obj->dev;
4484         drm_i915_private_t *dev_priv = dev->dev_private;
4485         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4486         int ret;
4487
4488         ret = i915_gem_object_unbind(obj);
4489         if (ret == -ERESTARTSYS) {
4490                 list_move(&obj_priv->list,
4491                           &dev_priv->mm.deferred_free_list);
4492                 return;
4493         }
4494
4495         if (obj_priv->mmap_offset)
4496                 i915_gem_free_mmap_offset(obj);
4497
4498         drm_gem_object_release(obj);
4499         i915_gem_info_remove_obj(dev_priv, obj->size);
4500
4501         kfree(obj_priv->page_cpu_valid);
4502         kfree(obj_priv->bit_17);
4503         kfree(obj_priv);
4504 }
4505
4506 void i915_gem_free_object(struct drm_gem_object *obj)
4507 {
4508         struct drm_device *dev = obj->dev;
4509         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4510
4511         trace_i915_gem_object_destroy(obj);
4512
4513         while (obj_priv->pin_count > 0)
4514                 i915_gem_object_unpin(obj);
4515
4516         if (obj_priv->phys_obj)
4517                 i915_gem_detach_phys_object(dev, obj);
4518
4519         i915_gem_free_object_tail(obj);
4520 }
4521
4522 int
4523 i915_gem_idle(struct drm_device *dev)
4524 {
4525         drm_i915_private_t *dev_priv = dev->dev_private;
4526         int ret;
4527
4528         mutex_lock(&dev->struct_mutex);
4529
4530         if (dev_priv->mm.suspended ||
4531                         (dev_priv->render_ring.gem_object == NULL) ||
4532                         (HAS_BSD(dev) &&
4533                          dev_priv->bsd_ring.gem_object == NULL)) {
4534                 mutex_unlock(&dev->struct_mutex);
4535                 return 0;
4536         }
4537
4538         ret = i915_gpu_idle(dev);
4539         if (ret) {
4540                 mutex_unlock(&dev->struct_mutex);
4541                 return ret;
4542         }
4543
4544         /* Under UMS, be paranoid and evict. */
4545         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4546                 ret = i915_gem_evict_inactive(dev);
4547                 if (ret) {
4548                         mutex_unlock(&dev->struct_mutex);
4549                         return ret;
4550                 }
4551         }
4552
4553         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4554          * We need to replace this with a semaphore, or something.
4555          * And not confound mm.suspended!
4556          */
4557         dev_priv->mm.suspended = 1;
4558         del_timer_sync(&dev_priv->hangcheck_timer);
4559
4560         i915_kernel_lost_context(dev);
4561         i915_gem_cleanup_ringbuffer(dev);
4562
4563         mutex_unlock(&dev->struct_mutex);
4564
4565         /* Cancel the retire work handler, which should be idle now. */
4566         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4567
4568         return 0;
4569 }
4570
4571 /*
4572  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4573  * over cache flushing.
4574  */
4575 static int
4576 i915_gem_init_pipe_control(struct drm_device *dev)
4577 {
4578         drm_i915_private_t *dev_priv = dev->dev_private;
4579         struct drm_gem_object *obj;
4580         struct drm_i915_gem_object *obj_priv;
4581         int ret;
4582
4583         obj = i915_gem_alloc_object(dev, 4096);
4584         if (obj == NULL) {
4585                 DRM_ERROR("Failed to allocate seqno page\n");
4586                 ret = -ENOMEM;
4587                 goto err;
4588         }
4589         obj_priv = to_intel_bo(obj);
4590         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4591
4592         ret = i915_gem_object_pin(obj, 4096);
4593         if (ret)
4594                 goto err_unref;
4595
4596         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4597         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4598         if (dev_priv->seqno_page == NULL)
4599                 goto err_unpin;
4600
4601         dev_priv->seqno_obj = obj;
4602         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4603
4604         return 0;
4605
4606 err_unpin:
4607         i915_gem_object_unpin(obj);
4608 err_unref:
4609         drm_gem_object_unreference(obj);
4610 err:
4611         return ret;
4612 }
4613
4614
4615 static void
4616 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4617 {
4618         drm_i915_private_t *dev_priv = dev->dev_private;
4619         struct drm_gem_object *obj;
4620         struct drm_i915_gem_object *obj_priv;
4621
4622         obj = dev_priv->seqno_obj;
4623         obj_priv = to_intel_bo(obj);
4624         kunmap(obj_priv->pages[0]);
4625         i915_gem_object_unpin(obj);
4626         drm_gem_object_unreference(obj);
4627         dev_priv->seqno_obj = NULL;
4628
4629         dev_priv->seqno_page = NULL;
4630 }
4631
4632 int
4633 i915_gem_init_ringbuffer(struct drm_device *dev)
4634 {
4635         drm_i915_private_t *dev_priv = dev->dev_private;
4636         int ret;
4637
4638         if (HAS_PIPE_CONTROL(dev)) {
4639                 ret = i915_gem_init_pipe_control(dev);
4640                 if (ret)
4641                         return ret;
4642         }
4643
4644         ret = intel_init_render_ring_buffer(dev);
4645         if (ret)
4646                 goto cleanup_pipe_control;
4647
4648         if (HAS_BSD(dev)) {
4649                 ret = intel_init_bsd_ring_buffer(dev);
4650                 if (ret)
4651                         goto cleanup_render_ring;
4652         }
4653
4654         dev_priv->next_seqno = 1;
4655
4656         return 0;
4657
4658 cleanup_render_ring:
4659         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4660 cleanup_pipe_control:
4661         if (HAS_PIPE_CONTROL(dev))
4662                 i915_gem_cleanup_pipe_control(dev);
4663         return ret;
4664 }
4665
4666 void
4667 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4668 {
4669         drm_i915_private_t *dev_priv = dev->dev_private;
4670
4671         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4672         if (HAS_BSD(dev))
4673                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4674         if (HAS_PIPE_CONTROL(dev))
4675                 i915_gem_cleanup_pipe_control(dev);
4676 }
4677
4678 int
4679 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4680                        struct drm_file *file_priv)
4681 {
4682         drm_i915_private_t *dev_priv = dev->dev_private;
4683         int ret;
4684
4685         if (drm_core_check_feature(dev, DRIVER_MODESET))
4686                 return 0;
4687
4688         if (atomic_read(&dev_priv->mm.wedged)) {
4689                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4690                 atomic_set(&dev_priv->mm.wedged, 0);
4691         }
4692
4693         mutex_lock(&dev->struct_mutex);
4694         dev_priv->mm.suspended = 0;
4695
4696         ret = i915_gem_init_ringbuffer(dev);
4697         if (ret != 0) {
4698                 mutex_unlock(&dev->struct_mutex);
4699                 return ret;
4700         }
4701
4702         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4703         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4704         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4705         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4706         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4707         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4708         mutex_unlock(&dev->struct_mutex);
4709
4710         ret = drm_irq_install(dev);
4711         if (ret)
4712                 goto cleanup_ringbuffer;
4713
4714         return 0;
4715
4716 cleanup_ringbuffer:
4717         mutex_lock(&dev->struct_mutex);
4718         i915_gem_cleanup_ringbuffer(dev);
4719         dev_priv->mm.suspended = 1;
4720         mutex_unlock(&dev->struct_mutex);
4721
4722         return ret;
4723 }
4724
4725 int
4726 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4727                        struct drm_file *file_priv)
4728 {
4729         if (drm_core_check_feature(dev, DRIVER_MODESET))
4730                 return 0;
4731
4732         drm_irq_uninstall(dev);
4733         return i915_gem_idle(dev);
4734 }
4735
4736 void
4737 i915_gem_lastclose(struct drm_device *dev)
4738 {
4739         int ret;
4740
4741         if (drm_core_check_feature(dev, DRIVER_MODESET))
4742                 return;
4743
4744         ret = i915_gem_idle(dev);
4745         if (ret)
4746                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4747 }
4748
4749 void
4750 i915_gem_load(struct drm_device *dev)
4751 {
4752         int i;
4753         drm_i915_private_t *dev_priv = dev->dev_private;
4754
4755         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4756         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4757         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4758         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4759         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4760         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4761         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4762         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4763         if (HAS_BSD(dev)) {
4764                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4765                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4766         }
4767         for (i = 0; i < 16; i++)
4768                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4769         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4770                           i915_gem_retire_work_handler);
4771         init_completion(&dev_priv->error_completion);
4772         spin_lock(&shrink_list_lock);
4773         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4774         spin_unlock(&shrink_list_lock);
4775
4776         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4777         if (IS_GEN3(dev)) {
4778                 u32 tmp = I915_READ(MI_ARB_STATE);
4779                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4780                         /* arb state is a masked write, so set bit + bit in mask */
4781                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4782                         I915_WRITE(MI_ARB_STATE, tmp);
4783                 }
4784         }
4785
4786         /* Old X drivers will take 0-2 for front, back, depth buffers */
4787         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4788                 dev_priv->fence_reg_start = 3;
4789
4790         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4791                 dev_priv->num_fence_regs = 16;
4792         else
4793                 dev_priv->num_fence_regs = 8;
4794
4795         /* Initialize fence registers to zero */
4796         switch (INTEL_INFO(dev)->gen) {
4797         case 6:
4798                 for (i = 0; i < 16; i++)
4799                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4800                 break;
4801         case 5:
4802         case 4:
4803                 for (i = 0; i < 16; i++)
4804                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4805                 break;
4806         case 3:
4807                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4808                         for (i = 0; i < 8; i++)
4809                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4810         case 2:
4811                 for (i = 0; i < 8; i++)
4812                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4813                 break;
4814         }
4815         i915_gem_detect_bit_6_swizzle(dev);
4816         init_waitqueue_head(&dev_priv->pending_flip_queue);
4817 }
4818
4819 /*
4820  * Create a physically contiguous memory object for this object
4821  * e.g. for cursor + overlay regs
4822  */
4823 static int i915_gem_init_phys_object(struct drm_device *dev,
4824                                      int id, int size, int align)
4825 {
4826         drm_i915_private_t *dev_priv = dev->dev_private;
4827         struct drm_i915_gem_phys_object *phys_obj;
4828         int ret;
4829
4830         if (dev_priv->mm.phys_objs[id - 1] || !size)
4831                 return 0;
4832
4833         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4834         if (!phys_obj)
4835                 return -ENOMEM;
4836
4837         phys_obj->id = id;
4838
4839         phys_obj->handle = drm_pci_alloc(dev, size, align);
4840         if (!phys_obj->handle) {
4841                 ret = -ENOMEM;
4842                 goto kfree_obj;
4843         }
4844 #ifdef CONFIG_X86
4845         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4846 #endif
4847
4848         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4849
4850         return 0;
4851 kfree_obj:
4852         kfree(phys_obj);
4853         return ret;
4854 }
4855
4856 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4857 {
4858         drm_i915_private_t *dev_priv = dev->dev_private;
4859         struct drm_i915_gem_phys_object *phys_obj;
4860
4861         if (!dev_priv->mm.phys_objs[id - 1])
4862                 return;
4863
4864         phys_obj = dev_priv->mm.phys_objs[id - 1];
4865         if (phys_obj->cur_obj) {
4866                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4867         }
4868
4869 #ifdef CONFIG_X86
4870         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4871 #endif
4872         drm_pci_free(dev, phys_obj->handle);
4873         kfree(phys_obj);
4874         dev_priv->mm.phys_objs[id - 1] = NULL;
4875 }
4876
4877 void i915_gem_free_all_phys_object(struct drm_device *dev)
4878 {
4879         int i;
4880
4881         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4882                 i915_gem_free_phys_object(dev, i);
4883 }
4884
4885 void i915_gem_detach_phys_object(struct drm_device *dev,
4886                                  struct drm_gem_object *obj)
4887 {
4888         struct drm_i915_gem_object *obj_priv;
4889         int i;
4890         int ret;
4891         int page_count;
4892
4893         obj_priv = to_intel_bo(obj);
4894         if (!obj_priv->phys_obj)
4895                 return;
4896
4897         ret = i915_gem_object_get_pages(obj, 0);
4898         if (ret)
4899                 goto out;
4900
4901         page_count = obj->size / PAGE_SIZE;
4902
4903         for (i = 0; i < page_count; i++) {
4904                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4905                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4906
4907                 memcpy(dst, src, PAGE_SIZE);
4908                 kunmap_atomic(dst, KM_USER0);
4909         }
4910         drm_clflush_pages(obj_priv->pages, page_count);
4911         drm_agp_chipset_flush(dev);
4912
4913         i915_gem_object_put_pages(obj);
4914 out:
4915         obj_priv->phys_obj->cur_obj = NULL;
4916         obj_priv->phys_obj = NULL;
4917 }
4918
4919 int
4920 i915_gem_attach_phys_object(struct drm_device *dev,
4921                             struct drm_gem_object *obj,
4922                             int id,
4923                             int align)
4924 {
4925         drm_i915_private_t *dev_priv = dev->dev_private;
4926         struct drm_i915_gem_object *obj_priv;
4927         int ret = 0;
4928         int page_count;
4929         int i;
4930
4931         if (id > I915_MAX_PHYS_OBJECT)
4932                 return -EINVAL;
4933
4934         obj_priv = to_intel_bo(obj);
4935
4936         if (obj_priv->phys_obj) {
4937                 if (obj_priv->phys_obj->id == id)
4938                         return 0;
4939                 i915_gem_detach_phys_object(dev, obj);
4940         }
4941
4942         /* create a new object */
4943         if (!dev_priv->mm.phys_objs[id - 1]) {
4944                 ret = i915_gem_init_phys_object(dev, id,
4945                                                 obj->size, align);
4946                 if (ret) {
4947                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4948                         goto out;
4949                 }
4950         }
4951
4952         /* bind to the object */
4953         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4954         obj_priv->phys_obj->cur_obj = obj;
4955
4956         ret = i915_gem_object_get_pages(obj, 0);
4957         if (ret) {
4958                 DRM_ERROR("failed to get page list\n");
4959                 goto out;
4960         }
4961
4962         page_count = obj->size / PAGE_SIZE;
4963
4964         for (i = 0; i < page_count; i++) {
4965                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4966                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4967
4968                 memcpy(dst, src, PAGE_SIZE);
4969                 kunmap_atomic(src, KM_USER0);
4970         }
4971
4972         i915_gem_object_put_pages(obj);
4973
4974         return 0;
4975 out:
4976         return ret;
4977 }
4978
4979 static int
4980 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4981                      struct drm_i915_gem_pwrite *args,
4982                      struct drm_file *file_priv)
4983 {
4984         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4985         void *obj_addr;
4986         int ret;
4987         char __user *user_data;
4988
4989         user_data = (char __user *) (uintptr_t) args->data_ptr;
4990         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4991
4992         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4993         ret = copy_from_user(obj_addr, user_data, args->size);
4994         if (ret)
4995                 return -EFAULT;
4996
4997         drm_agp_chipset_flush(dev);
4998         return 0;
4999 }
5000
5001 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5002 {
5003         struct drm_i915_file_private *file_priv = file->driver_priv;
5004
5005         /* Clean up our request list when the client is going away, so that
5006          * later retire_requests won't dereference our soon-to-be-gone
5007          * file_priv.
5008          */
5009         spin_lock(&file_priv->mm.lock);
5010         while (!list_empty(&file_priv->mm.request_list)) {
5011                 struct drm_i915_gem_request *request;
5012
5013                 request = list_first_entry(&file_priv->mm.request_list,
5014                                            struct drm_i915_gem_request,
5015                                            client_list);
5016                 list_del(&request->client_list);
5017                 request->file_priv = NULL;
5018         }
5019         spin_unlock(&file_priv->mm.lock);
5020 }
5021
5022 static int
5023 i915_gpu_is_active(struct drm_device *dev)
5024 {
5025         drm_i915_private_t *dev_priv = dev->dev_private;
5026         int lists_empty;
5027
5028         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5029                       list_empty(&dev_priv->render_ring.active_list);
5030         if (HAS_BSD(dev))
5031                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
5032
5033         return !lists_empty;
5034 }
5035
5036 static int
5037 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
5038 {
5039         drm_i915_private_t *dev_priv, *next_dev;
5040         struct drm_i915_gem_object *obj_priv, *next_obj;
5041         int cnt = 0;
5042         int would_deadlock = 1;
5043
5044         /* "fast-path" to count number of available objects */
5045         if (nr_to_scan == 0) {
5046                 spin_lock(&shrink_list_lock);
5047                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5048                         struct drm_device *dev = dev_priv->dev;
5049
5050                         if (mutex_trylock(&dev->struct_mutex)) {
5051                                 list_for_each_entry(obj_priv,
5052                                                     &dev_priv->mm.inactive_list,
5053                                                     list)
5054                                         cnt++;
5055                                 mutex_unlock(&dev->struct_mutex);
5056                         }
5057                 }
5058                 spin_unlock(&shrink_list_lock);
5059
5060                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5061         }
5062
5063         spin_lock(&shrink_list_lock);
5064
5065 rescan:
5066         /* first scan for clean buffers */
5067         list_for_each_entry_safe(dev_priv, next_dev,
5068                                  &shrink_list, mm.shrink_list) {
5069                 struct drm_device *dev = dev_priv->dev;
5070
5071                 if (! mutex_trylock(&dev->struct_mutex))
5072                         continue;
5073
5074                 spin_unlock(&shrink_list_lock);
5075                 i915_gem_retire_requests(dev);
5076
5077                 list_for_each_entry_safe(obj_priv, next_obj,
5078                                          &dev_priv->mm.inactive_list,
5079                                          list) {
5080                         if (i915_gem_object_is_purgeable(obj_priv)) {
5081                                 i915_gem_object_unbind(&obj_priv->base);
5082                                 if (--nr_to_scan <= 0)
5083                                         break;
5084                         }
5085                 }
5086
5087                 spin_lock(&shrink_list_lock);
5088                 mutex_unlock(&dev->struct_mutex);
5089
5090                 would_deadlock = 0;
5091
5092                 if (nr_to_scan <= 0)
5093                         break;
5094         }
5095
5096         /* second pass, evict/count anything still on the inactive list */
5097         list_for_each_entry_safe(dev_priv, next_dev,
5098                                  &shrink_list, mm.shrink_list) {
5099                 struct drm_device *dev = dev_priv->dev;
5100
5101                 if (! mutex_trylock(&dev->struct_mutex))
5102                         continue;
5103
5104                 spin_unlock(&shrink_list_lock);
5105
5106                 list_for_each_entry_safe(obj_priv, next_obj,
5107                                          &dev_priv->mm.inactive_list,
5108                                          list) {
5109                         if (nr_to_scan > 0) {
5110                                 i915_gem_object_unbind(&obj_priv->base);
5111                                 nr_to_scan--;
5112                         } else
5113                                 cnt++;
5114                 }
5115
5116                 spin_lock(&shrink_list_lock);
5117                 mutex_unlock(&dev->struct_mutex);
5118
5119                 would_deadlock = 0;
5120         }
5121
5122         if (nr_to_scan) {
5123                 int active = 0;
5124
5125                 /*
5126                  * We are desperate for pages, so as a last resort, wait
5127                  * for the GPU to finish and discard whatever we can.
5128                  * This has a dramatic impact to reduce the number of
5129                  * OOM-killer events whilst running the GPU aggressively.
5130                  */
5131                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5132                         struct drm_device *dev = dev_priv->dev;
5133
5134                         if (!mutex_trylock(&dev->struct_mutex))
5135                                 continue;
5136
5137                         spin_unlock(&shrink_list_lock);
5138
5139                         if (i915_gpu_is_active(dev)) {
5140                                 i915_gpu_idle(dev);
5141                                 active++;
5142                         }
5143
5144                         spin_lock(&shrink_list_lock);
5145                         mutex_unlock(&dev->struct_mutex);
5146                 }
5147
5148                 if (active)
5149                         goto rescan;
5150         }
5151
5152         spin_unlock(&shrink_list_lock);
5153
5154         if (would_deadlock)
5155                 return -1;
5156         else if (cnt > 0)
5157                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5158         else
5159                 return 0;
5160 }
5161
5162 static struct shrinker shrinker = {
5163         .shrink = i915_gem_shrink,
5164         .seeks = DEFAULT_SEEKS,
5165 };
5166
5167 __init void
5168 i915_gem_shrinker_init(void)
5169 {
5170     register_shrinker(&shrinker);
5171 }
5172
5173 __exit void
5174 i915_gem_shrinker_exit(void)
5175 {
5176     unregister_shrinker(&shrinker);
5177 }