2 * Copyright © 2008 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include <linux/slab.h>
35 #include <linux/swap.h>
36 #include <linux/pci.h>
37 #include <linux/intel-gtt.h>
39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
41 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
42 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
43 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
44 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
49 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
50 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
55 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
56 struct drm_i915_gem_pwrite *args,
57 struct drm_file *file_priv);
58 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
61 i915_gem_object_get_pages(struct drm_gem_object *obj,
65 i915_gem_object_put_pages(struct drm_gem_object *obj);
67 static LIST_HEAD(shrink_list);
68 static DEFINE_SPINLOCK(shrink_list_lock);
70 /* some bookkeeping */
71 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
74 dev_priv->mm.object_count++;
75 dev_priv->mm.object_memory += size;
78 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
81 dev_priv->mm.object_count--;
82 dev_priv->mm.object_memory -= size;
85 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
88 dev_priv->mm.gtt_count++;
89 dev_priv->mm.gtt_memory += size;
92 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
95 dev_priv->mm.gtt_count--;
96 dev_priv->mm.gtt_memory -= size;
99 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
102 dev_priv->mm.pin_count++;
103 dev_priv->mm.pin_memory += size;
106 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
109 dev_priv->mm.pin_count--;
110 dev_priv->mm.pin_memory -= size;
114 i915_gem_check_is_wedged(struct drm_device *dev)
116 struct drm_i915_private *dev_priv = dev->dev_private;
117 struct completion *x = &dev_priv->error_completion;
121 if (!atomic_read(&dev_priv->mm.wedged))
124 ret = wait_for_completion_interruptible(x);
128 /* Success, we reset the GPU! */
129 if (!atomic_read(&dev_priv->mm.wedged))
132 /* GPU is hung, bump the completion count to account for
133 * the token we just consumed so that we never hit zero and
134 * end up waiting upon a subsequent completion event that
137 spin_lock_irqsave(&x->wait.lock, flags);
139 spin_unlock_irqrestore(&x->wait.lock, flags);
143 static int i915_mutex_lock_interruptible(struct drm_device *dev)
145 struct drm_i915_private *dev_priv = dev->dev_private;
148 ret = i915_gem_check_is_wedged(dev);
152 ret = mutex_lock_interruptible(&dev->struct_mutex);
156 if (atomic_read(&dev_priv->mm.wedged)) {
157 mutex_unlock(&dev->struct_mutex);
161 WARN_ON(i915_verify_lists(dev));
166 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
168 return obj_priv->gtt_space &&
170 obj_priv->pin_count == 0;
173 int i915_gem_do_init(struct drm_device *dev,
177 drm_i915_private_t *dev_priv = dev->dev_private;
180 (start & (PAGE_SIZE - 1)) != 0 ||
181 (end & (PAGE_SIZE - 1)) != 0) {
185 drm_mm_init(&dev_priv->mm.gtt_space, start,
188 dev_priv->mm.gtt_total = end - start;
194 i915_gem_init_ioctl(struct drm_device *dev, void *data,
195 struct drm_file *file_priv)
197 struct drm_i915_gem_init *args = data;
200 mutex_lock(&dev->struct_mutex);
201 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
202 mutex_unlock(&dev->struct_mutex);
208 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
209 struct drm_file *file_priv)
211 struct drm_i915_private *dev_priv = dev->dev_private;
212 struct drm_i915_gem_get_aperture *args = data;
214 if (!(dev->driver->driver_features & DRIVER_GEM))
217 mutex_lock(&dev->struct_mutex);
218 args->aper_size = dev_priv->mm.gtt_total;
219 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
220 mutex_unlock(&dev->struct_mutex);
227 * Creates a new mm object and returns a handle to it.
230 i915_gem_create_ioctl(struct drm_device *dev, void *data,
231 struct drm_file *file_priv)
233 struct drm_i915_gem_create *args = data;
234 struct drm_gem_object *obj;
238 args->size = roundup(args->size, PAGE_SIZE);
240 /* Allocate the new object */
241 obj = i915_gem_alloc_object(dev, args->size);
245 ret = drm_gem_handle_create(file_priv, obj, &handle);
247 drm_gem_object_release(obj);
248 i915_gem_info_remove_obj(dev->dev_private, obj->size);
253 /* drop reference from allocate - handle holds it now */
254 drm_gem_object_unreference(obj);
255 trace_i915_gem_object_create(obj);
257 args->handle = handle;
262 fast_shmem_read(struct page **pages,
263 loff_t page_base, int page_offset,
270 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
271 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
272 kunmap_atomic(vaddr);
277 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
279 drm_i915_private_t *dev_priv = obj->dev->dev_private;
280 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
282 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
283 obj_priv->tiling_mode != I915_TILING_NONE;
287 slow_shmem_copy(struct page *dst_page,
289 struct page *src_page,
293 char *dst_vaddr, *src_vaddr;
295 dst_vaddr = kmap(dst_page);
296 src_vaddr = kmap(src_page);
298 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
305 slow_shmem_bit17_copy(struct page *gpu_page,
307 struct page *cpu_page,
312 char *gpu_vaddr, *cpu_vaddr;
314 /* Use the unswizzled path if this page isn't affected. */
315 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
317 return slow_shmem_copy(cpu_page, cpu_offset,
318 gpu_page, gpu_offset, length);
320 return slow_shmem_copy(gpu_page, gpu_offset,
321 cpu_page, cpu_offset, length);
324 gpu_vaddr = kmap(gpu_page);
325 cpu_vaddr = kmap(cpu_page);
327 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
328 * XORing with the other bits (A9 for Y, A9 and A10 for X)
331 int cacheline_end = ALIGN(gpu_offset + 1, 64);
332 int this_length = min(cacheline_end - gpu_offset, length);
333 int swizzled_gpu_offset = gpu_offset ^ 64;
336 memcpy(cpu_vaddr + cpu_offset,
337 gpu_vaddr + swizzled_gpu_offset,
340 memcpy(gpu_vaddr + swizzled_gpu_offset,
341 cpu_vaddr + cpu_offset,
344 cpu_offset += this_length;
345 gpu_offset += this_length;
346 length -= this_length;
354 * This is the fast shmem pread path, which attempts to copy_from_user directly
355 * from the backing pages of the object to the user's address space. On a
356 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
359 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
360 struct drm_i915_gem_pread *args,
361 struct drm_file *file_priv)
363 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
365 loff_t offset, page_base;
366 char __user *user_data;
367 int page_offset, page_length;
369 user_data = (char __user *) (uintptr_t) args->data_ptr;
372 obj_priv = to_intel_bo(obj);
373 offset = args->offset;
376 /* Operation in this page
378 * page_base = page offset within aperture
379 * page_offset = offset within page
380 * page_length = bytes to copy for this page
382 page_base = (offset & ~(PAGE_SIZE-1));
383 page_offset = offset & (PAGE_SIZE-1);
384 page_length = remain;
385 if ((page_offset + remain) > PAGE_SIZE)
386 page_length = PAGE_SIZE - page_offset;
388 if (fast_shmem_read(obj_priv->pages,
389 page_base, page_offset,
390 user_data, page_length))
393 remain -= page_length;
394 user_data += page_length;
395 offset += page_length;
402 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
406 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
408 /* If we've insufficient memory to map in the pages, attempt
409 * to make some space by throwing out some old buffers.
411 if (ret == -ENOMEM) {
412 struct drm_device *dev = obj->dev;
414 ret = i915_gem_evict_something(dev, obj->size,
415 i915_gem_get_gtt_alignment(obj));
419 ret = i915_gem_object_get_pages(obj, 0);
426 * This is the fallback shmem pread path, which allocates temporary storage
427 * in kernel space to copy_to_user into outside of the struct_mutex, so we
428 * can copy out of the object's backing pages while holding the struct mutex
429 * and not take page faults.
432 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
433 struct drm_i915_gem_pread *args,
434 struct drm_file *file_priv)
436 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
437 struct mm_struct *mm = current->mm;
438 struct page **user_pages;
440 loff_t offset, pinned_pages, i;
441 loff_t first_data_page, last_data_page, num_pages;
442 int shmem_page_index, shmem_page_offset;
443 int data_page_index, data_page_offset;
446 uint64_t data_ptr = args->data_ptr;
447 int do_bit17_swizzling;
451 /* Pin the user pages containing the data. We can't fault while
452 * holding the struct mutex, yet we want to hold it while
453 * dereferencing the user data.
455 first_data_page = data_ptr / PAGE_SIZE;
456 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
457 num_pages = last_data_page - first_data_page + 1;
459 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
460 if (user_pages == NULL)
463 mutex_unlock(&dev->struct_mutex);
464 down_read(&mm->mmap_sem);
465 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
466 num_pages, 1, 0, user_pages, NULL);
467 up_read(&mm->mmap_sem);
468 mutex_lock(&dev->struct_mutex);
469 if (pinned_pages < num_pages) {
474 ret = i915_gem_object_set_cpu_read_domain_range(obj,
480 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
482 obj_priv = to_intel_bo(obj);
483 offset = args->offset;
486 /* Operation in this page
488 * shmem_page_index = page number within shmem file
489 * shmem_page_offset = offset within page in shmem file
490 * data_page_index = page number in get_user_pages return
491 * data_page_offset = offset with data_page_index page.
492 * page_length = bytes to copy for this page
494 shmem_page_index = offset / PAGE_SIZE;
495 shmem_page_offset = offset & ~PAGE_MASK;
496 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
497 data_page_offset = data_ptr & ~PAGE_MASK;
499 page_length = remain;
500 if ((shmem_page_offset + page_length) > PAGE_SIZE)
501 page_length = PAGE_SIZE - shmem_page_offset;
502 if ((data_page_offset + page_length) > PAGE_SIZE)
503 page_length = PAGE_SIZE - data_page_offset;
505 if (do_bit17_swizzling) {
506 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
508 user_pages[data_page_index],
513 slow_shmem_copy(user_pages[data_page_index],
515 obj_priv->pages[shmem_page_index],
520 remain -= page_length;
521 data_ptr += page_length;
522 offset += page_length;
526 for (i = 0; i < pinned_pages; i++) {
527 SetPageDirty(user_pages[i]);
528 page_cache_release(user_pages[i]);
530 drm_free_large(user_pages);
536 * Reads data from the object referenced by handle.
538 * On error, the contents of *data are undefined.
541 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
542 struct drm_file *file_priv)
544 struct drm_i915_gem_pread *args = data;
545 struct drm_gem_object *obj;
546 struct drm_i915_gem_object *obj_priv;
552 if (!access_ok(VERIFY_WRITE,
553 (char __user *)(uintptr_t)args->data_ptr,
557 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
562 ret = i915_mutex_lock_interruptible(dev);
566 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
571 obj_priv = to_intel_bo(obj);
573 /* Bounds check source. */
574 if (args->offset > obj->size || args->size > obj->size - args->offset) {
579 ret = i915_gem_object_get_pages_or_evict(obj);
583 ret = i915_gem_object_set_cpu_read_domain_range(obj,
590 if (!i915_gem_object_needs_bit17_swizzle(obj))
591 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
593 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
596 i915_gem_object_put_pages(obj);
598 drm_gem_object_unreference(obj);
600 mutex_unlock(&dev->struct_mutex);
604 /* This is the fast write path which cannot handle
605 * page faults in the source data
609 fast_user_write(struct io_mapping *mapping,
610 loff_t page_base, int page_offset,
611 char __user *user_data,
615 unsigned long unwritten;
617 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
618 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
620 io_mapping_unmap_atomic(vaddr_atomic);
624 /* Here's the write path which can sleep for
629 slow_kernel_write(struct io_mapping *mapping,
630 loff_t gtt_base, int gtt_offset,
631 struct page *user_page, int user_offset,
634 char __iomem *dst_vaddr;
637 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
638 src_vaddr = kmap(user_page);
640 memcpy_toio(dst_vaddr + gtt_offset,
641 src_vaddr + user_offset,
645 io_mapping_unmap(dst_vaddr);
649 fast_shmem_write(struct page **pages,
650 loff_t page_base, int page_offset,
657 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
658 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
659 kunmap_atomic(vaddr);
665 * This is the fast pwrite path, where we copy the data directly from the
666 * user into the GTT, uncached.
669 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
670 struct drm_i915_gem_pwrite *args,
671 struct drm_file *file_priv)
673 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
674 drm_i915_private_t *dev_priv = dev->dev_private;
676 loff_t offset, page_base;
677 char __user *user_data;
678 int page_offset, page_length;
680 user_data = (char __user *) (uintptr_t) args->data_ptr;
683 obj_priv = to_intel_bo(obj);
684 offset = obj_priv->gtt_offset + args->offset;
687 /* Operation in this page
689 * page_base = page offset within aperture
690 * page_offset = offset within page
691 * page_length = bytes to copy for this page
693 page_base = (offset & ~(PAGE_SIZE-1));
694 page_offset = offset & (PAGE_SIZE-1);
695 page_length = remain;
696 if ((page_offset + remain) > PAGE_SIZE)
697 page_length = PAGE_SIZE - page_offset;
699 /* If we get a fault while copying data, then (presumably) our
700 * source page isn't available. Return the error and we'll
701 * retry in the slow path.
703 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
704 page_offset, user_data, page_length))
708 remain -= page_length;
709 user_data += page_length;
710 offset += page_length;
717 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
718 * the memory and maps it using kmap_atomic for copying.
720 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
721 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
724 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
725 struct drm_i915_gem_pwrite *args,
726 struct drm_file *file_priv)
728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
729 drm_i915_private_t *dev_priv = dev->dev_private;
731 loff_t gtt_page_base, offset;
732 loff_t first_data_page, last_data_page, num_pages;
733 loff_t pinned_pages, i;
734 struct page **user_pages;
735 struct mm_struct *mm = current->mm;
736 int gtt_page_offset, data_page_offset, data_page_index, page_length;
738 uint64_t data_ptr = args->data_ptr;
742 /* Pin the user pages containing the data. We can't fault while
743 * holding the struct mutex, and all of the pwrite implementations
744 * want to hold it while dereferencing the user data.
746 first_data_page = data_ptr / PAGE_SIZE;
747 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
748 num_pages = last_data_page - first_data_page + 1;
750 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
751 if (user_pages == NULL)
754 mutex_unlock(&dev->struct_mutex);
755 down_read(&mm->mmap_sem);
756 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
757 num_pages, 0, 0, user_pages, NULL);
758 up_read(&mm->mmap_sem);
759 mutex_lock(&dev->struct_mutex);
760 if (pinned_pages < num_pages) {
762 goto out_unpin_pages;
765 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
767 goto out_unpin_pages;
769 obj_priv = to_intel_bo(obj);
770 offset = obj_priv->gtt_offset + args->offset;
773 /* Operation in this page
775 * gtt_page_base = page offset within aperture
776 * gtt_page_offset = offset within page in aperture
777 * data_page_index = page number in get_user_pages return
778 * data_page_offset = offset with data_page_index page.
779 * page_length = bytes to copy for this page
781 gtt_page_base = offset & PAGE_MASK;
782 gtt_page_offset = offset & ~PAGE_MASK;
783 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
784 data_page_offset = data_ptr & ~PAGE_MASK;
786 page_length = remain;
787 if ((gtt_page_offset + page_length) > PAGE_SIZE)
788 page_length = PAGE_SIZE - gtt_page_offset;
789 if ((data_page_offset + page_length) > PAGE_SIZE)
790 page_length = PAGE_SIZE - data_page_offset;
792 slow_kernel_write(dev_priv->mm.gtt_mapping,
793 gtt_page_base, gtt_page_offset,
794 user_pages[data_page_index],
798 remain -= page_length;
799 offset += page_length;
800 data_ptr += page_length;
804 for (i = 0; i < pinned_pages; i++)
805 page_cache_release(user_pages[i]);
806 drm_free_large(user_pages);
812 * This is the fast shmem pwrite path, which attempts to directly
813 * copy_from_user into the kmapped pages backing the object.
816 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
817 struct drm_i915_gem_pwrite *args,
818 struct drm_file *file_priv)
820 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
822 loff_t offset, page_base;
823 char __user *user_data;
824 int page_offset, page_length;
826 user_data = (char __user *) (uintptr_t) args->data_ptr;
829 obj_priv = to_intel_bo(obj);
830 offset = args->offset;
834 /* Operation in this page
836 * page_base = page offset within aperture
837 * page_offset = offset within page
838 * page_length = bytes to copy for this page
840 page_base = (offset & ~(PAGE_SIZE-1));
841 page_offset = offset & (PAGE_SIZE-1);
842 page_length = remain;
843 if ((page_offset + remain) > PAGE_SIZE)
844 page_length = PAGE_SIZE - page_offset;
846 if (fast_shmem_write(obj_priv->pages,
847 page_base, page_offset,
848 user_data, page_length))
851 remain -= page_length;
852 user_data += page_length;
853 offset += page_length;
860 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
861 * the memory and maps it using kmap_atomic for copying.
863 * This avoids taking mmap_sem for faulting on the user's address while the
864 * struct_mutex is held.
867 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
868 struct drm_i915_gem_pwrite *args,
869 struct drm_file *file_priv)
871 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
872 struct mm_struct *mm = current->mm;
873 struct page **user_pages;
875 loff_t offset, pinned_pages, i;
876 loff_t first_data_page, last_data_page, num_pages;
877 int shmem_page_index, shmem_page_offset;
878 int data_page_index, data_page_offset;
881 uint64_t data_ptr = args->data_ptr;
882 int do_bit17_swizzling;
886 /* Pin the user pages containing the data. We can't fault while
887 * holding the struct mutex, and all of the pwrite implementations
888 * want to hold it while dereferencing the user data.
890 first_data_page = data_ptr / PAGE_SIZE;
891 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
892 num_pages = last_data_page - first_data_page + 1;
894 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
895 if (user_pages == NULL)
898 mutex_unlock(&dev->struct_mutex);
899 down_read(&mm->mmap_sem);
900 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
901 num_pages, 0, 0, user_pages, NULL);
902 up_read(&mm->mmap_sem);
903 mutex_lock(&dev->struct_mutex);
904 if (pinned_pages < num_pages) {
909 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
913 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
915 obj_priv = to_intel_bo(obj);
916 offset = args->offset;
920 /* Operation in this page
922 * shmem_page_index = page number within shmem file
923 * shmem_page_offset = offset within page in shmem file
924 * data_page_index = page number in get_user_pages return
925 * data_page_offset = offset with data_page_index page.
926 * page_length = bytes to copy for this page
928 shmem_page_index = offset / PAGE_SIZE;
929 shmem_page_offset = offset & ~PAGE_MASK;
930 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
931 data_page_offset = data_ptr & ~PAGE_MASK;
933 page_length = remain;
934 if ((shmem_page_offset + page_length) > PAGE_SIZE)
935 page_length = PAGE_SIZE - shmem_page_offset;
936 if ((data_page_offset + page_length) > PAGE_SIZE)
937 page_length = PAGE_SIZE - data_page_offset;
939 if (do_bit17_swizzling) {
940 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
942 user_pages[data_page_index],
947 slow_shmem_copy(obj_priv->pages[shmem_page_index],
949 user_pages[data_page_index],
954 remain -= page_length;
955 data_ptr += page_length;
956 offset += page_length;
960 for (i = 0; i < pinned_pages; i++)
961 page_cache_release(user_pages[i]);
962 drm_free_large(user_pages);
968 * Writes data to the object referenced by handle.
970 * On error, the contents of the buffer that were to be modified are undefined.
973 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
974 struct drm_file *file)
976 struct drm_i915_gem_pwrite *args = data;
977 struct drm_gem_object *obj;
978 struct drm_i915_gem_object *obj_priv;
984 if (!access_ok(VERIFY_READ,
985 (char __user *)(uintptr_t)args->data_ptr,
989 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
994 ret = i915_mutex_lock_interruptible(dev);
998 obj = drm_gem_object_lookup(dev, file, args->handle);
1003 obj_priv = to_intel_bo(obj);
1005 /* Bounds check destination. */
1006 if (args->offset > obj->size || args->size > obj->size - args->offset) {
1011 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1012 * it would end up going through the fenced access, and we'll get
1013 * different detiling behavior between reading and writing.
1014 * pread/pwrite currently are reading and writing from the CPU
1015 * perspective, requiring manual detiling by the client.
1017 if (obj_priv->phys_obj)
1018 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1019 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1020 obj_priv->gtt_space &&
1021 obj->write_domain != I915_GEM_DOMAIN_CPU) {
1022 ret = i915_gem_object_pin(obj, 0);
1026 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1030 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1032 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1035 i915_gem_object_unpin(obj);
1037 ret = i915_gem_object_get_pages_or_evict(obj);
1041 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1046 if (!i915_gem_object_needs_bit17_swizzle(obj))
1047 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1049 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1052 i915_gem_object_put_pages(obj);
1056 drm_gem_object_unreference(obj);
1058 mutex_unlock(&dev->struct_mutex);
1063 * Called when user space prepares to use an object with the CPU, either
1064 * through the mmap ioctl's mapping or a GTT mapping.
1067 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1068 struct drm_file *file_priv)
1070 struct drm_i915_private *dev_priv = dev->dev_private;
1071 struct drm_i915_gem_set_domain *args = data;
1072 struct drm_gem_object *obj;
1073 struct drm_i915_gem_object *obj_priv;
1074 uint32_t read_domains = args->read_domains;
1075 uint32_t write_domain = args->write_domain;
1078 if (!(dev->driver->driver_features & DRIVER_GEM))
1081 /* Only handle setting domains to types used by the CPU. */
1082 if (write_domain & I915_GEM_GPU_DOMAINS)
1085 if (read_domains & I915_GEM_GPU_DOMAINS)
1088 /* Having something in the write domain implies it's in the read
1089 * domain, and only that read domain. Enforce that in the request.
1091 if (write_domain != 0 && read_domains != write_domain)
1094 ret = i915_mutex_lock_interruptible(dev);
1098 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1103 obj_priv = to_intel_bo(obj);
1105 intel_mark_busy(dev, obj);
1107 if (read_domains & I915_GEM_DOMAIN_GTT) {
1108 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1110 /* Update the LRU on the fence for the CPU access that's
1113 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1114 struct drm_i915_fence_reg *reg =
1115 &dev_priv->fence_regs[obj_priv->fence_reg];
1116 list_move_tail(®->lru_list,
1117 &dev_priv->mm.fence_list);
1120 /* Silently promote "you're not bound, there was nothing to do"
1121 * to success, since the client was just asking us to
1122 * make sure everything was done.
1127 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1130 /* Maintain LRU order of "inactive" objects */
1131 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1132 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1134 drm_gem_object_unreference(obj);
1136 mutex_unlock(&dev->struct_mutex);
1141 * Called when user space has done writes to this buffer
1144 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1145 struct drm_file *file_priv)
1147 struct drm_i915_gem_sw_finish *args = data;
1148 struct drm_gem_object *obj;
1151 if (!(dev->driver->driver_features & DRIVER_GEM))
1154 ret = i915_mutex_lock_interruptible(dev);
1158 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1164 /* Pinned buffers may be scanout, so flush the cache */
1165 if (to_intel_bo(obj)->pin_count)
1166 i915_gem_object_flush_cpu_write_domain(obj);
1168 drm_gem_object_unreference(obj);
1170 mutex_unlock(&dev->struct_mutex);
1175 * Maps the contents of an object, returning the address it is mapped
1178 * While the mapping holds a reference on the contents of the object, it doesn't
1179 * imply a ref on the object itself.
1182 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1183 struct drm_file *file_priv)
1185 struct drm_i915_gem_mmap *args = data;
1186 struct drm_gem_object *obj;
1190 if (!(dev->driver->driver_features & DRIVER_GEM))
1193 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1197 offset = args->offset;
1199 down_write(¤t->mm->mmap_sem);
1200 addr = do_mmap(obj->filp, 0, args->size,
1201 PROT_READ | PROT_WRITE, MAP_SHARED,
1203 up_write(¤t->mm->mmap_sem);
1204 drm_gem_object_unreference_unlocked(obj);
1205 if (IS_ERR((void *)addr))
1208 args->addr_ptr = (uint64_t) addr;
1214 * i915_gem_fault - fault a page into the GTT
1215 * vma: VMA in question
1218 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1219 * from userspace. The fault handler takes care of binding the object to
1220 * the GTT (if needed), allocating and programming a fence register (again,
1221 * only if needed based on whether the old reg is still valid or the object
1222 * is tiled) and inserting a new PTE into the faulting process.
1224 * Note that the faulting process may involve evicting existing objects
1225 * from the GTT and/or fence registers to make room. So performance may
1226 * suffer if the GTT working set is large or there are few fence registers
1229 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1231 struct drm_gem_object *obj = vma->vm_private_data;
1232 struct drm_device *dev = obj->dev;
1233 drm_i915_private_t *dev_priv = dev->dev_private;
1234 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1235 pgoff_t page_offset;
1238 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1240 /* We don't use vmf->pgoff since that has the fake offset */
1241 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1244 /* Now bind it into the GTT if needed */
1245 mutex_lock(&dev->struct_mutex);
1246 if (!obj_priv->gtt_space) {
1247 ret = i915_gem_object_bind_to_gtt(obj, 0);
1251 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1256 /* Need a new fence register? */
1257 if (obj_priv->tiling_mode != I915_TILING_NONE) {
1258 ret = i915_gem_object_get_fence_reg(obj, true);
1263 if (i915_gem_object_is_inactive(obj_priv))
1264 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1266 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1269 /* Finally, remap it using the new GTT offset */
1270 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1272 mutex_unlock(&dev->struct_mutex);
1277 return VM_FAULT_NOPAGE;
1280 return VM_FAULT_OOM;
1282 return VM_FAULT_SIGBUS;
1287 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1288 * @obj: obj in question
1290 * GEM memory mapping works by handing back to userspace a fake mmap offset
1291 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1292 * up the object based on the offset and sets up the various memory mapping
1295 * This routine allocates and attaches a fake offset for @obj.
1298 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1300 struct drm_device *dev = obj->dev;
1301 struct drm_gem_mm *mm = dev->mm_private;
1302 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1303 struct drm_map_list *list;
1304 struct drm_local_map *map;
1307 /* Set the object up for mmap'ing */
1308 list = &obj->map_list;
1309 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1314 map->type = _DRM_GEM;
1315 map->size = obj->size;
1318 /* Get a DRM GEM mmap offset allocated... */
1319 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1320 obj->size / PAGE_SIZE, 0, 0);
1321 if (!list->file_offset_node) {
1322 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1327 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1328 obj->size / PAGE_SIZE, 0);
1329 if (!list->file_offset_node) {
1334 list->hash.key = list->file_offset_node->start;
1335 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1337 DRM_ERROR("failed to add to map hash\n");
1341 /* By now we should be all set, any drm_mmap request on the offset
1342 * below will get to our mmap & fault handler */
1343 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1348 drm_mm_put_block(list->file_offset_node);
1356 * i915_gem_release_mmap - remove physical page mappings
1357 * @obj: obj in question
1359 * Preserve the reservation of the mmapping with the DRM core code, but
1360 * relinquish ownership of the pages back to the system.
1362 * It is vital that we remove the page mapping if we have mapped a tiled
1363 * object through the GTT and then lose the fence register due to
1364 * resource pressure. Similarly if the object has been moved out of the
1365 * aperture, than pages mapped into userspace must be revoked. Removing the
1366 * mapping will then trigger a page fault on the next user access, allowing
1367 * fixup by i915_gem_fault().
1370 i915_gem_release_mmap(struct drm_gem_object *obj)
1372 struct drm_device *dev = obj->dev;
1373 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1375 if (dev->dev_mapping)
1376 unmap_mapping_range(dev->dev_mapping,
1377 obj_priv->mmap_offset, obj->size, 1);
1381 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1383 struct drm_device *dev = obj->dev;
1384 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1385 struct drm_gem_mm *mm = dev->mm_private;
1386 struct drm_map_list *list;
1388 list = &obj->map_list;
1389 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1391 if (list->file_offset_node) {
1392 drm_mm_put_block(list->file_offset_node);
1393 list->file_offset_node = NULL;
1401 obj_priv->mmap_offset = 0;
1405 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1406 * @obj: object to check
1408 * Return the required GTT alignment for an object, taking into account
1409 * potential fence register mapping if needed.
1412 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1414 struct drm_device *dev = obj->dev;
1415 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1419 * Minimum alignment is 4k (GTT page size), but might be greater
1420 * if a fence register is needed for the object.
1422 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1426 * Previous chips need to be aligned to the size of the smallest
1427 * fence register that can contain the object.
1429 if (INTEL_INFO(dev)->gen == 3)
1434 for (i = start; i < obj->size; i <<= 1)
1441 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1443 * @data: GTT mapping ioctl data
1444 * @file_priv: GEM object info
1446 * Simply returns the fake offset to userspace so it can mmap it.
1447 * The mmap call will end up in drm_gem_mmap(), which will set things
1448 * up so we can get faults in the handler above.
1450 * The fault handler will take care of binding the object into the GTT
1451 * (since it may have been evicted to make room for something), allocating
1452 * a fence register, and mapping the appropriate aperture address into
1456 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1457 struct drm_file *file_priv)
1459 struct drm_i915_gem_mmap_gtt *args = data;
1460 struct drm_gem_object *obj;
1461 struct drm_i915_gem_object *obj_priv;
1464 if (!(dev->driver->driver_features & DRIVER_GEM))
1467 ret = i915_mutex_lock_interruptible(dev);
1471 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1476 obj_priv = to_intel_bo(obj);
1478 if (obj_priv->madv != I915_MADV_WILLNEED) {
1479 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1484 if (!obj_priv->mmap_offset) {
1485 ret = i915_gem_create_mmap_offset(obj);
1490 args->offset = obj_priv->mmap_offset;
1493 * Pull it into the GTT so that we have a page list (makes the
1494 * initial fault faster and any subsequent flushing possible).
1496 if (!obj_priv->agp_mem) {
1497 ret = i915_gem_object_bind_to_gtt(obj, 0);
1503 drm_gem_object_unreference(obj);
1505 mutex_unlock(&dev->struct_mutex);
1510 i915_gem_object_put_pages(struct drm_gem_object *obj)
1512 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1513 int page_count = obj->size / PAGE_SIZE;
1516 BUG_ON(obj_priv->pages_refcount == 0);
1517 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1519 if (--obj_priv->pages_refcount != 0)
1522 if (obj_priv->tiling_mode != I915_TILING_NONE)
1523 i915_gem_object_save_bit_17_swizzle(obj);
1525 if (obj_priv->madv == I915_MADV_DONTNEED)
1526 obj_priv->dirty = 0;
1528 for (i = 0; i < page_count; i++) {
1529 if (obj_priv->dirty)
1530 set_page_dirty(obj_priv->pages[i]);
1532 if (obj_priv->madv == I915_MADV_WILLNEED)
1533 mark_page_accessed(obj_priv->pages[i]);
1535 page_cache_release(obj_priv->pages[i]);
1537 obj_priv->dirty = 0;
1539 drm_free_large(obj_priv->pages);
1540 obj_priv->pages = NULL;
1544 i915_gem_next_request_seqno(struct drm_device *dev,
1545 struct intel_ring_buffer *ring)
1547 drm_i915_private_t *dev_priv = dev->dev_private;
1549 ring->outstanding_lazy_request = true;
1550 return dev_priv->next_seqno;
1554 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1555 struct intel_ring_buffer *ring)
1557 struct drm_device *dev = obj->dev;
1558 struct drm_i915_private *dev_priv = dev->dev_private;
1559 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1560 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1562 BUG_ON(ring == NULL);
1563 obj_priv->ring = ring;
1565 /* Add a reference if we're newly entering the active list. */
1566 if (!obj_priv->active) {
1567 drm_gem_object_reference(obj);
1568 obj_priv->active = 1;
1571 /* Move from whatever list we were on to the tail of execution. */
1572 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1573 list_move_tail(&obj_priv->ring_list, &ring->active_list);
1574 obj_priv->last_rendering_seqno = seqno;
1578 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1580 struct drm_device *dev = obj->dev;
1581 drm_i915_private_t *dev_priv = dev->dev_private;
1582 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1584 BUG_ON(!obj_priv->active);
1585 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1586 list_del_init(&obj_priv->ring_list);
1587 obj_priv->last_rendering_seqno = 0;
1590 /* Immediately discard the backing storage */
1592 i915_gem_object_truncate(struct drm_gem_object *obj)
1594 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1595 struct inode *inode;
1597 /* Our goal here is to return as much of the memory as
1598 * is possible back to the system as we are called from OOM.
1599 * To do this we must instruct the shmfs to drop all of its
1600 * backing pages, *now*. Here we mirror the actions taken
1601 * when by shmem_delete_inode() to release the backing store.
1603 inode = obj->filp->f_path.dentry->d_inode;
1604 truncate_inode_pages(inode->i_mapping, 0);
1605 if (inode->i_op->truncate_range)
1606 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1608 obj_priv->madv = __I915_MADV_PURGED;
1612 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1614 return obj_priv->madv == I915_MADV_DONTNEED;
1618 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1620 struct drm_device *dev = obj->dev;
1621 drm_i915_private_t *dev_priv = dev->dev_private;
1622 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1624 if (obj_priv->pin_count != 0)
1625 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1627 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1628 list_del_init(&obj_priv->ring_list);
1630 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1632 obj_priv->last_rendering_seqno = 0;
1633 obj_priv->ring = NULL;
1634 if (obj_priv->active) {
1635 obj_priv->active = 0;
1636 drm_gem_object_unreference(obj);
1638 WARN_ON(i915_verify_lists(dev));
1642 i915_gem_process_flushing_list(struct drm_device *dev,
1643 uint32_t flush_domains,
1644 struct intel_ring_buffer *ring)
1646 drm_i915_private_t *dev_priv = dev->dev_private;
1647 struct drm_i915_gem_object *obj_priv, *next;
1649 list_for_each_entry_safe(obj_priv, next,
1650 &ring->gpu_write_list,
1652 struct drm_gem_object *obj = &obj_priv->base;
1654 if (obj->write_domain & flush_domains) {
1655 uint32_t old_write_domain = obj->write_domain;
1657 obj->write_domain = 0;
1658 list_del_init(&obj_priv->gpu_write_list);
1659 i915_gem_object_move_to_active(obj, ring);
1661 /* update the fence lru list */
1662 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1663 struct drm_i915_fence_reg *reg =
1664 &dev_priv->fence_regs[obj_priv->fence_reg];
1665 list_move_tail(®->lru_list,
1666 &dev_priv->mm.fence_list);
1669 trace_i915_gem_object_change_domain(obj,
1677 i915_add_request(struct drm_device *dev,
1678 struct drm_file *file,
1679 struct drm_i915_gem_request *request,
1680 struct intel_ring_buffer *ring)
1682 drm_i915_private_t *dev_priv = dev->dev_private;
1683 struct drm_i915_file_private *file_priv = NULL;
1688 file_priv = file->driver_priv;
1690 if (request == NULL) {
1691 request = kzalloc(sizeof(*request), GFP_KERNEL);
1692 if (request == NULL)
1696 seqno = ring->add_request(dev, ring, 0);
1697 ring->outstanding_lazy_request = false;
1699 request->seqno = seqno;
1700 request->ring = ring;
1701 request->emitted_jiffies = jiffies;
1702 was_empty = list_empty(&ring->request_list);
1703 list_add_tail(&request->list, &ring->request_list);
1706 spin_lock(&file_priv->mm.lock);
1707 request->file_priv = file_priv;
1708 list_add_tail(&request->client_list,
1709 &file_priv->mm.request_list);
1710 spin_unlock(&file_priv->mm.lock);
1713 if (!dev_priv->mm.suspended) {
1714 mod_timer(&dev_priv->hangcheck_timer,
1715 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1717 queue_delayed_work(dev_priv->wq,
1718 &dev_priv->mm.retire_work, HZ);
1724 * Command execution barrier
1726 * Ensures that all commands in the ring are finished
1727 * before signalling the CPU
1730 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1732 uint32_t flush_domains = 0;
1734 /* The sampler always gets flushed on i965 (sigh) */
1735 if (INTEL_INFO(dev)->gen >= 4)
1736 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1738 ring->flush(dev, ring,
1739 I915_GEM_DOMAIN_COMMAND, flush_domains);
1743 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1745 struct drm_i915_file_private *file_priv = request->file_priv;
1750 spin_lock(&file_priv->mm.lock);
1751 list_del(&request->client_list);
1752 request->file_priv = NULL;
1753 spin_unlock(&file_priv->mm.lock);
1756 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1757 struct intel_ring_buffer *ring)
1759 while (!list_empty(&ring->request_list)) {
1760 struct drm_i915_gem_request *request;
1762 request = list_first_entry(&ring->request_list,
1763 struct drm_i915_gem_request,
1766 list_del(&request->list);
1767 i915_gem_request_remove_from_client(request);
1771 while (!list_empty(&ring->active_list)) {
1772 struct drm_i915_gem_object *obj_priv;
1774 obj_priv = list_first_entry(&ring->active_list,
1775 struct drm_i915_gem_object,
1778 obj_priv->base.write_domain = 0;
1779 list_del_init(&obj_priv->gpu_write_list);
1780 i915_gem_object_move_to_inactive(&obj_priv->base);
1784 void i915_gem_reset(struct drm_device *dev)
1786 struct drm_i915_private *dev_priv = dev->dev_private;
1787 struct drm_i915_gem_object *obj_priv;
1790 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1791 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1792 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1794 /* Remove anything from the flushing lists. The GPU cache is likely
1795 * to be lost on reset along with the data, so simply move the
1796 * lost bo to the inactive list.
1798 while (!list_empty(&dev_priv->mm.flushing_list)) {
1799 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1800 struct drm_i915_gem_object,
1803 obj_priv->base.write_domain = 0;
1804 list_del_init(&obj_priv->gpu_write_list);
1805 i915_gem_object_move_to_inactive(&obj_priv->base);
1808 /* Move everything out of the GPU domains to ensure we do any
1809 * necessary invalidation upon reuse.
1811 list_for_each_entry(obj_priv,
1812 &dev_priv->mm.inactive_list,
1815 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1818 /* The fence registers are invalidated so clear them out */
1819 for (i = 0; i < 16; i++) {
1820 struct drm_i915_fence_reg *reg;
1822 reg = &dev_priv->fence_regs[i];
1826 i915_gem_clear_fence_reg(reg->obj);
1831 * This function clears the request list as sequence numbers are passed.
1834 i915_gem_retire_requests_ring(struct drm_device *dev,
1835 struct intel_ring_buffer *ring)
1837 drm_i915_private_t *dev_priv = dev->dev_private;
1840 if (!ring->status_page.page_addr ||
1841 list_empty(&ring->request_list))
1844 WARN_ON(i915_verify_lists(dev));
1846 seqno = ring->get_seqno(dev, ring);
1847 while (!list_empty(&ring->request_list)) {
1848 struct drm_i915_gem_request *request;
1850 request = list_first_entry(&ring->request_list,
1851 struct drm_i915_gem_request,
1854 if (!i915_seqno_passed(seqno, request->seqno))
1857 trace_i915_gem_request_retire(dev, request->seqno);
1859 list_del(&request->list);
1860 i915_gem_request_remove_from_client(request);
1864 /* Move any buffers on the active list that are no longer referenced
1865 * by the ringbuffer to the flushing/inactive lists as appropriate.
1867 while (!list_empty(&ring->active_list)) {
1868 struct drm_gem_object *obj;
1869 struct drm_i915_gem_object *obj_priv;
1871 obj_priv = list_first_entry(&ring->active_list,
1872 struct drm_i915_gem_object,
1875 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1878 obj = &obj_priv->base;
1879 if (obj->write_domain != 0)
1880 i915_gem_object_move_to_flushing(obj);
1882 i915_gem_object_move_to_inactive(obj);
1885 if (unlikely (dev_priv->trace_irq_seqno &&
1886 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1887 ring->user_irq_put(dev, ring);
1888 dev_priv->trace_irq_seqno = 0;
1891 WARN_ON(i915_verify_lists(dev));
1895 i915_gem_retire_requests(struct drm_device *dev)
1897 drm_i915_private_t *dev_priv = dev->dev_private;
1899 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1900 struct drm_i915_gem_object *obj_priv, *tmp;
1902 /* We must be careful that during unbind() we do not
1903 * accidentally infinitely recurse into retire requests.
1905 * retire -> free -> unbind -> wait -> retire_ring
1907 list_for_each_entry_safe(obj_priv, tmp,
1908 &dev_priv->mm.deferred_free_list,
1910 i915_gem_free_object_tail(&obj_priv->base);
1913 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1914 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1915 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
1919 i915_gem_retire_work_handler(struct work_struct *work)
1921 drm_i915_private_t *dev_priv;
1922 struct drm_device *dev;
1924 dev_priv = container_of(work, drm_i915_private_t,
1925 mm.retire_work.work);
1926 dev = dev_priv->dev;
1928 /* Come back later if the device is busy... */
1929 if (!mutex_trylock(&dev->struct_mutex)) {
1930 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1934 i915_gem_retire_requests(dev);
1936 if (!dev_priv->mm.suspended &&
1937 (!list_empty(&dev_priv->render_ring.request_list) ||
1938 !list_empty(&dev_priv->bsd_ring.request_list) ||
1939 !list_empty(&dev_priv->blt_ring.request_list)))
1940 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1941 mutex_unlock(&dev->struct_mutex);
1945 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1946 bool interruptible, struct intel_ring_buffer *ring)
1948 drm_i915_private_t *dev_priv = dev->dev_private;
1954 if (atomic_read(&dev_priv->mm.wedged))
1957 if (ring->outstanding_lazy_request) {
1958 seqno = i915_add_request(dev, NULL, NULL, ring);
1962 BUG_ON(seqno == dev_priv->next_seqno);
1964 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
1965 if (HAS_PCH_SPLIT(dev))
1966 ier = I915_READ(DEIER) | I915_READ(GTIER);
1968 ier = I915_READ(IER);
1970 DRM_ERROR("something (likely vbetool) disabled "
1971 "interrupts, re-enabling\n");
1972 i915_driver_irq_preinstall(dev);
1973 i915_driver_irq_postinstall(dev);
1976 trace_i915_gem_request_wait_begin(dev, seqno);
1978 ring->waiting_gem_seqno = seqno;
1979 ring->user_irq_get(dev, ring);
1981 ret = wait_event_interruptible(ring->irq_queue,
1983 ring->get_seqno(dev, ring), seqno)
1984 || atomic_read(&dev_priv->mm.wedged));
1986 wait_event(ring->irq_queue,
1988 ring->get_seqno(dev, ring), seqno)
1989 || atomic_read(&dev_priv->mm.wedged));
1991 ring->user_irq_put(dev, ring);
1992 ring->waiting_gem_seqno = 0;
1994 trace_i915_gem_request_wait_end(dev, seqno);
1996 if (atomic_read(&dev_priv->mm.wedged))
1999 if (ret && ret != -ERESTARTSYS)
2000 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2001 __func__, ret, seqno, ring->get_seqno(dev, ring),
2002 dev_priv->next_seqno);
2004 /* Directly dispatch request retiring. While we have the work queue
2005 * to handle this, the waiter on a request often wants an associated
2006 * buffer to have made it to the inactive list, and we would need
2007 * a separate wait queue to handle that.
2010 i915_gem_retire_requests_ring(dev, ring);
2016 * Waits for a sequence number to be signaled, and cleans up the
2017 * request and object lists appropriately for that event.
2020 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2021 struct intel_ring_buffer *ring)
2023 return i915_do_wait_request(dev, seqno, 1, ring);
2027 i915_gem_flush_ring(struct drm_device *dev,
2028 struct drm_file *file_priv,
2029 struct intel_ring_buffer *ring,
2030 uint32_t invalidate_domains,
2031 uint32_t flush_domains)
2033 ring->flush(dev, ring, invalidate_domains, flush_domains);
2034 i915_gem_process_flushing_list(dev, flush_domains, ring);
2038 i915_gem_flush(struct drm_device *dev,
2039 struct drm_file *file_priv,
2040 uint32_t invalidate_domains,
2041 uint32_t flush_domains,
2042 uint32_t flush_rings)
2044 drm_i915_private_t *dev_priv = dev->dev_private;
2046 if (flush_domains & I915_GEM_DOMAIN_CPU)
2047 drm_agp_chipset_flush(dev);
2049 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2050 if (flush_rings & RING_RENDER)
2051 i915_gem_flush_ring(dev, file_priv,
2052 &dev_priv->render_ring,
2053 invalidate_domains, flush_domains);
2054 if (flush_rings & RING_BSD)
2055 i915_gem_flush_ring(dev, file_priv,
2056 &dev_priv->bsd_ring,
2057 invalidate_domains, flush_domains);
2058 if (flush_rings & RING_BLT)
2059 i915_gem_flush_ring(dev, file_priv,
2060 &dev_priv->blt_ring,
2061 invalidate_domains, flush_domains);
2066 * Ensures that all rendering to the object has completed and the object is
2067 * safe to unbind from the GTT or access from the CPU.
2070 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2073 struct drm_device *dev = obj->dev;
2074 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2077 /* This function only exists to support waiting for existing rendering,
2078 * not for emitting required flushes.
2080 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2082 /* If there is rendering queued on the buffer being evicted, wait for
2085 if (obj_priv->active) {
2086 ret = i915_do_wait_request(dev,
2087 obj_priv->last_rendering_seqno,
2098 * Unbinds an object from the GTT aperture.
2101 i915_gem_object_unbind(struct drm_gem_object *obj)
2103 struct drm_device *dev = obj->dev;
2104 struct drm_i915_private *dev_priv = dev->dev_private;
2105 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2108 if (obj_priv->gtt_space == NULL)
2111 if (obj_priv->pin_count != 0) {
2112 DRM_ERROR("Attempting to unbind pinned buffer\n");
2116 /* blow away mappings if mapped through GTT */
2117 i915_gem_release_mmap(obj);
2119 /* Move the object to the CPU domain to ensure that
2120 * any possible CPU writes while it's not in the GTT
2121 * are flushed when we go to remap it. This will
2122 * also ensure that all pending GPU writes are finished
2125 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2126 if (ret == -ERESTARTSYS)
2128 /* Continue on if we fail due to EIO, the GPU is hung so we
2129 * should be safe and we need to cleanup or else we might
2130 * cause memory corruption through use-after-free.
2133 i915_gem_clflush_object(obj);
2134 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2137 /* release the fence reg _after_ flushing */
2138 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2139 i915_gem_clear_fence_reg(obj);
2141 drm_unbind_agp(obj_priv->agp_mem);
2142 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2144 i915_gem_object_put_pages(obj);
2145 BUG_ON(obj_priv->pages_refcount);
2147 i915_gem_info_remove_gtt(dev_priv, obj->size);
2148 list_del_init(&obj_priv->mm_list);
2150 drm_mm_put_block(obj_priv->gtt_space);
2151 obj_priv->gtt_space = NULL;
2152 obj_priv->gtt_offset = 0;
2154 if (i915_gem_object_is_purgeable(obj_priv))
2155 i915_gem_object_truncate(obj);
2157 trace_i915_gem_object_unbind(obj);
2162 static int i915_ring_idle(struct drm_device *dev,
2163 struct intel_ring_buffer *ring)
2165 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2168 i915_gem_flush_ring(dev, NULL, ring,
2169 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2170 return i915_wait_request(dev,
2171 i915_gem_next_request_seqno(dev, ring),
2176 i915_gpu_idle(struct drm_device *dev)
2178 drm_i915_private_t *dev_priv = dev->dev_private;
2182 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2183 list_empty(&dev_priv->mm.active_list));
2187 /* Flush everything onto the inactive list. */
2188 ret = i915_ring_idle(dev, &dev_priv->render_ring);
2192 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2196 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2204 i915_gem_object_get_pages(struct drm_gem_object *obj,
2207 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2209 struct address_space *mapping;
2210 struct inode *inode;
2213 BUG_ON(obj_priv->pages_refcount
2214 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2216 if (obj_priv->pages_refcount++ != 0)
2219 /* Get the list of pages out of our struct file. They'll be pinned
2220 * at this point until we release them.
2222 page_count = obj->size / PAGE_SIZE;
2223 BUG_ON(obj_priv->pages != NULL);
2224 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2225 if (obj_priv->pages == NULL) {
2226 obj_priv->pages_refcount--;
2230 inode = obj->filp->f_path.dentry->d_inode;
2231 mapping = inode->i_mapping;
2232 for (i = 0; i < page_count; i++) {
2233 page = read_cache_page_gfp(mapping, i,
2241 obj_priv->pages[i] = page;
2244 if (obj_priv->tiling_mode != I915_TILING_NONE)
2245 i915_gem_object_do_bit_17_swizzle(obj);
2251 page_cache_release(obj_priv->pages[i]);
2253 drm_free_large(obj_priv->pages);
2254 obj_priv->pages = NULL;
2255 obj_priv->pages_refcount--;
2256 return PTR_ERR(page);
2259 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2261 struct drm_gem_object *obj = reg->obj;
2262 struct drm_device *dev = obj->dev;
2263 drm_i915_private_t *dev_priv = dev->dev_private;
2264 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2265 int regnum = obj_priv->fence_reg;
2268 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2270 val |= obj_priv->gtt_offset & 0xfffff000;
2271 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2272 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2274 if (obj_priv->tiling_mode == I915_TILING_Y)
2275 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2276 val |= I965_FENCE_REG_VALID;
2278 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2281 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2283 struct drm_gem_object *obj = reg->obj;
2284 struct drm_device *dev = obj->dev;
2285 drm_i915_private_t *dev_priv = dev->dev_private;
2286 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2287 int regnum = obj_priv->fence_reg;
2290 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2292 val |= obj_priv->gtt_offset & 0xfffff000;
2293 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2294 if (obj_priv->tiling_mode == I915_TILING_Y)
2295 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2296 val |= I965_FENCE_REG_VALID;
2298 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2301 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2303 struct drm_gem_object *obj = reg->obj;
2304 struct drm_device *dev = obj->dev;
2305 drm_i915_private_t *dev_priv = dev->dev_private;
2306 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2307 int regnum = obj_priv->fence_reg;
2309 uint32_t fence_reg, val;
2312 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2313 (obj_priv->gtt_offset & (obj->size - 1))) {
2314 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2315 __func__, obj_priv->gtt_offset, obj->size);
2319 if (obj_priv->tiling_mode == I915_TILING_Y &&
2320 HAS_128_BYTE_Y_TILING(dev))
2325 /* Note: pitch better be a power of two tile widths */
2326 pitch_val = obj_priv->stride / tile_width;
2327 pitch_val = ffs(pitch_val) - 1;
2329 if (obj_priv->tiling_mode == I915_TILING_Y &&
2330 HAS_128_BYTE_Y_TILING(dev))
2331 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2333 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2335 val = obj_priv->gtt_offset;
2336 if (obj_priv->tiling_mode == I915_TILING_Y)
2337 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2338 val |= I915_FENCE_SIZE_BITS(obj->size);
2339 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2340 val |= I830_FENCE_REG_VALID;
2343 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2345 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2346 I915_WRITE(fence_reg, val);
2349 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2351 struct drm_gem_object *obj = reg->obj;
2352 struct drm_device *dev = obj->dev;
2353 drm_i915_private_t *dev_priv = dev->dev_private;
2354 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2355 int regnum = obj_priv->fence_reg;
2358 uint32_t fence_size_bits;
2360 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2361 (obj_priv->gtt_offset & (obj->size - 1))) {
2362 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2363 __func__, obj_priv->gtt_offset);
2367 pitch_val = obj_priv->stride / 128;
2368 pitch_val = ffs(pitch_val) - 1;
2369 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2371 val = obj_priv->gtt_offset;
2372 if (obj_priv->tiling_mode == I915_TILING_Y)
2373 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2374 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2375 WARN_ON(fence_size_bits & ~0x00000f00);
2376 val |= fence_size_bits;
2377 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2378 val |= I830_FENCE_REG_VALID;
2380 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2383 static int i915_find_fence_reg(struct drm_device *dev,
2386 struct drm_i915_fence_reg *reg = NULL;
2387 struct drm_i915_gem_object *obj_priv = NULL;
2388 struct drm_i915_private *dev_priv = dev->dev_private;
2389 struct drm_gem_object *obj = NULL;
2392 /* First try to find a free reg */
2394 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2395 reg = &dev_priv->fence_regs[i];
2399 obj_priv = to_intel_bo(reg->obj);
2400 if (!obj_priv->pin_count)
2407 /* None available, try to steal one or wait for a user to finish */
2408 i = I915_FENCE_REG_NONE;
2409 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2412 obj_priv = to_intel_bo(obj);
2414 if (obj_priv->pin_count)
2418 i = obj_priv->fence_reg;
2422 BUG_ON(i == I915_FENCE_REG_NONE);
2424 /* We only have a reference on obj from the active list. put_fence_reg
2425 * might drop that one, causing a use-after-free in it. So hold a
2426 * private reference to obj like the other callers of put_fence_reg
2427 * (set_tiling ioctl) do. */
2428 drm_gem_object_reference(obj);
2429 ret = i915_gem_object_put_fence_reg(obj, interruptible);
2430 drm_gem_object_unreference(obj);
2438 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2439 * @obj: object to map through a fence reg
2441 * When mapping objects through the GTT, userspace wants to be able to write
2442 * to them without having to worry about swizzling if the object is tiled.
2444 * This function walks the fence regs looking for a free one for @obj,
2445 * stealing one if it can't find any.
2447 * It then sets up the reg based on the object's properties: address, pitch
2448 * and tiling format.
2451 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2454 struct drm_device *dev = obj->dev;
2455 struct drm_i915_private *dev_priv = dev->dev_private;
2456 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2457 struct drm_i915_fence_reg *reg = NULL;
2460 /* Just update our place in the LRU if our fence is getting used. */
2461 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2462 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2463 list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
2467 switch (obj_priv->tiling_mode) {
2468 case I915_TILING_NONE:
2469 WARN(1, "allocating a fence for non-tiled object?\n");
2472 if (!obj_priv->stride)
2474 WARN((obj_priv->stride & (512 - 1)),
2475 "object 0x%08x is X tiled but has non-512B pitch\n",
2476 obj_priv->gtt_offset);
2479 if (!obj_priv->stride)
2481 WARN((obj_priv->stride & (128 - 1)),
2482 "object 0x%08x is Y tiled but has non-128B pitch\n",
2483 obj_priv->gtt_offset);
2487 ret = i915_find_fence_reg(dev, interruptible);
2491 obj_priv->fence_reg = ret;
2492 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2493 list_add_tail(®->lru_list, &dev_priv->mm.fence_list);
2497 switch (INTEL_INFO(dev)->gen) {
2499 sandybridge_write_fence_reg(reg);
2503 i965_write_fence_reg(reg);
2506 i915_write_fence_reg(reg);
2509 i830_write_fence_reg(reg);
2513 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2514 obj_priv->tiling_mode);
2520 * i915_gem_clear_fence_reg - clear out fence register info
2521 * @obj: object to clear
2523 * Zeroes out the fence register itself and clears out the associated
2524 * data structures in dev_priv and obj_priv.
2527 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2529 struct drm_device *dev = obj->dev;
2530 drm_i915_private_t *dev_priv = dev->dev_private;
2531 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2532 struct drm_i915_fence_reg *reg =
2533 &dev_priv->fence_regs[obj_priv->fence_reg];
2536 switch (INTEL_INFO(dev)->gen) {
2538 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2539 (obj_priv->fence_reg * 8), 0);
2543 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2546 if (obj_priv->fence_reg >= 8)
2547 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2550 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2552 I915_WRITE(fence_reg, 0);
2557 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2558 list_del_init(®->lru_list);
2562 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2563 * to the buffer to finish, and then resets the fence register.
2564 * @obj: tiled object holding a fence register.
2565 * @bool: whether the wait upon the fence is interruptible
2567 * Zeroes out the fence register itself and clears out the associated
2568 * data structures in dev_priv and obj_priv.
2571 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2574 struct drm_device *dev = obj->dev;
2575 struct drm_i915_private *dev_priv = dev->dev_private;
2576 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2577 struct drm_i915_fence_reg *reg;
2579 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2582 /* If we've changed tiling, GTT-mappings of the object
2583 * need to re-fault to ensure that the correct fence register
2584 * setup is in place.
2586 i915_gem_release_mmap(obj);
2588 /* On the i915, GPU access to tiled buffers is via a fence,
2589 * therefore we must wait for any outstanding access to complete
2590 * before clearing the fence.
2592 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2596 ret = i915_gem_object_flush_gpu_write_domain(obj);
2600 ret = i915_gem_object_wait_rendering(obj, interruptible);
2607 i915_gem_object_flush_gtt_write_domain(obj);
2608 i915_gem_clear_fence_reg(obj);
2614 * Finds free space in the GTT aperture and binds the object there.
2617 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2619 struct drm_device *dev = obj->dev;
2620 drm_i915_private_t *dev_priv = dev->dev_private;
2621 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2622 struct drm_mm_node *free_space;
2623 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2626 if (obj_priv->madv != I915_MADV_WILLNEED) {
2627 DRM_ERROR("Attempting to bind a purgeable object\n");
2632 alignment = i915_gem_get_gtt_alignment(obj);
2633 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2634 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2638 /* If the object is bigger than the entire aperture, reject it early
2639 * before evicting everything in a vain attempt to find space.
2641 if (obj->size > dev_priv->mm.gtt_total) {
2642 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2647 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2648 obj->size, alignment, 0);
2649 if (free_space != NULL)
2650 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2652 if (obj_priv->gtt_space == NULL) {
2653 /* If the gtt is empty and we're still having trouble
2654 * fitting our object in, we're out of memory.
2656 ret = i915_gem_evict_something(dev, obj->size, alignment);
2663 ret = i915_gem_object_get_pages(obj, gfpmask);
2665 drm_mm_put_block(obj_priv->gtt_space);
2666 obj_priv->gtt_space = NULL;
2668 if (ret == -ENOMEM) {
2669 /* first try to clear up some space from the GTT */
2670 ret = i915_gem_evict_something(dev, obj->size,
2673 /* now try to shrink everyone else */
2688 /* Create an AGP memory structure pointing at our pages, and bind it
2691 obj_priv->agp_mem = drm_agp_bind_pages(dev,
2693 obj->size >> PAGE_SHIFT,
2694 obj_priv->gtt_space->start,
2695 obj_priv->agp_type);
2696 if (obj_priv->agp_mem == NULL) {
2697 i915_gem_object_put_pages(obj);
2698 drm_mm_put_block(obj_priv->gtt_space);
2699 obj_priv->gtt_space = NULL;
2701 ret = i915_gem_evict_something(dev, obj->size, alignment);
2708 /* keep track of bounds object by adding it to the inactive list */
2709 list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2710 i915_gem_info_add_gtt(dev_priv, obj->size);
2712 /* Assert that the object is not currently in any GPU domain. As it
2713 * wasn't in the GTT, there shouldn't be any way it could have been in
2716 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2717 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2719 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2720 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2726 i915_gem_clflush_object(struct drm_gem_object *obj)
2728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2730 /* If we don't have a page list set up, then we're not pinned
2731 * to GPU, and we can ignore the cache flush because it'll happen
2732 * again at bind time.
2734 if (obj_priv->pages == NULL)
2737 trace_i915_gem_object_clflush(obj);
2739 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2742 /** Flushes any GPU write domain for the object if it's dirty. */
2744 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2746 struct drm_device *dev = obj->dev;
2747 uint32_t old_write_domain;
2749 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2752 /* Queue the GPU write cache flushing we need. */
2753 old_write_domain = obj->write_domain;
2754 i915_gem_flush_ring(dev, NULL,
2755 to_intel_bo(obj)->ring,
2756 0, obj->write_domain);
2757 BUG_ON(obj->write_domain);
2759 trace_i915_gem_object_change_domain(obj,
2766 /** Flushes the GTT write domain for the object if it's dirty. */
2768 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2770 uint32_t old_write_domain;
2772 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2775 /* No actual flushing is required for the GTT write domain. Writes
2776 * to it immediately go to main memory as far as we know, so there's
2777 * no chipset flush. It also doesn't land in render cache.
2779 old_write_domain = obj->write_domain;
2780 obj->write_domain = 0;
2782 trace_i915_gem_object_change_domain(obj,
2787 /** Flushes the CPU write domain for the object if it's dirty. */
2789 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2791 struct drm_device *dev = obj->dev;
2792 uint32_t old_write_domain;
2794 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2797 i915_gem_clflush_object(obj);
2798 drm_agp_chipset_flush(dev);
2799 old_write_domain = obj->write_domain;
2800 obj->write_domain = 0;
2802 trace_i915_gem_object_change_domain(obj,
2808 * Moves a single object to the GTT read, and possibly write domain.
2810 * This function returns when the move is complete, including waiting on
2814 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2816 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2817 uint32_t old_write_domain, old_read_domains;
2820 /* Not valid to be called on unbound objects. */
2821 if (obj_priv->gtt_space == NULL)
2824 ret = i915_gem_object_flush_gpu_write_domain(obj);
2827 ret = i915_gem_object_wait_rendering(obj, true);
2831 i915_gem_object_flush_cpu_write_domain(obj);
2833 old_write_domain = obj->write_domain;
2834 old_read_domains = obj->read_domains;
2836 /* It should now be out of any other write domains, and we can update
2837 * the domain values for our changes.
2839 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2840 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2842 obj->read_domains = I915_GEM_DOMAIN_GTT;
2843 obj->write_domain = I915_GEM_DOMAIN_GTT;
2844 obj_priv->dirty = 1;
2847 trace_i915_gem_object_change_domain(obj,
2855 * Prepare buffer for display plane. Use uninterruptible for possible flush
2856 * wait, as in modesetting process we're not supposed to be interrupted.
2859 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2862 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2863 uint32_t old_read_domains;
2866 /* Not valid to be called on unbound objects. */
2867 if (obj_priv->gtt_space == NULL)
2870 ret = i915_gem_object_flush_gpu_write_domain(obj);
2874 /* Currently, we are always called from an non-interruptible context. */
2876 ret = i915_gem_object_wait_rendering(obj, false);
2881 i915_gem_object_flush_cpu_write_domain(obj);
2883 old_read_domains = obj->read_domains;
2884 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2886 trace_i915_gem_object_change_domain(obj,
2894 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2900 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2901 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring,
2902 0, obj->base.write_domain);
2904 return i915_gem_object_wait_rendering(&obj->base, interruptible);
2908 * Moves a single object to the CPU read, and possibly write domain.
2910 * This function returns when the move is complete, including waiting on
2914 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2916 uint32_t old_write_domain, old_read_domains;
2919 ret = i915_gem_object_flush_gpu_write_domain(obj);
2922 ret = i915_gem_object_wait_rendering(obj, true);
2926 i915_gem_object_flush_gtt_write_domain(obj);
2928 /* If we have a partially-valid cache of the object in the CPU,
2929 * finish invalidating it and free the per-page flags.
2931 i915_gem_object_set_to_full_cpu_read_domain(obj);
2933 old_write_domain = obj->write_domain;
2934 old_read_domains = obj->read_domains;
2936 /* Flush the CPU cache if it's still invalid. */
2937 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2938 i915_gem_clflush_object(obj);
2940 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2943 /* It should now be out of any other write domains, and we can update
2944 * the domain values for our changes.
2946 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2948 /* If we're writing through the CPU, then the GPU read domains will
2949 * need to be invalidated at next use.
2952 obj->read_domains = I915_GEM_DOMAIN_CPU;
2953 obj->write_domain = I915_GEM_DOMAIN_CPU;
2956 trace_i915_gem_object_change_domain(obj,
2964 * Set the next domain for the specified object. This
2965 * may not actually perform the necessary flushing/invaliding though,
2966 * as that may want to be batched with other set_domain operations
2968 * This is (we hope) the only really tricky part of gem. The goal
2969 * is fairly simple -- track which caches hold bits of the object
2970 * and make sure they remain coherent. A few concrete examples may
2971 * help to explain how it works. For shorthand, we use the notation
2972 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2973 * a pair of read and write domain masks.
2975 * Case 1: the batch buffer
2981 * 5. Unmapped from GTT
2984 * Let's take these a step at a time
2987 * Pages allocated from the kernel may still have
2988 * cache contents, so we set them to (CPU, CPU) always.
2989 * 2. Written by CPU (using pwrite)
2990 * The pwrite function calls set_domain (CPU, CPU) and
2991 * this function does nothing (as nothing changes)
2993 * This function asserts that the object is not
2994 * currently in any GPU-based read or write domains
2996 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2997 * As write_domain is zero, this function adds in the
2998 * current read domains (CPU+COMMAND, 0).
2999 * flush_domains is set to CPU.
3000 * invalidate_domains is set to COMMAND
3001 * clflush is run to get data out of the CPU caches
3002 * then i915_dev_set_domain calls i915_gem_flush to
3003 * emit an MI_FLUSH and drm_agp_chipset_flush
3004 * 5. Unmapped from GTT
3005 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3006 * flush_domains and invalidate_domains end up both zero
3007 * so no flushing/invalidating happens
3011 * Case 2: The shared render buffer
3015 * 3. Read/written by GPU
3016 * 4. set_domain to (CPU,CPU)
3017 * 5. Read/written by CPU
3018 * 6. Read/written by GPU
3021 * Same as last example, (CPU, CPU)
3023 * Nothing changes (assertions find that it is not in the GPU)
3024 * 3. Read/written by GPU
3025 * execbuffer calls set_domain (RENDER, RENDER)
3026 * flush_domains gets CPU
3027 * invalidate_domains gets GPU
3029 * MI_FLUSH and drm_agp_chipset_flush
3030 * 4. set_domain (CPU, CPU)
3031 * flush_domains gets GPU
3032 * invalidate_domains gets CPU
3033 * wait_rendering (obj) to make sure all drawing is complete.
3034 * This will include an MI_FLUSH to get the data from GPU
3036 * clflush (obj) to invalidate the CPU cache
3037 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3038 * 5. Read/written by CPU
3039 * cache lines are loaded and dirtied
3040 * 6. Read written by GPU
3041 * Same as last GPU access
3043 * Case 3: The constant buffer
3048 * 4. Updated (written) by CPU again
3057 * flush_domains = CPU
3058 * invalidate_domains = RENDER
3061 * drm_agp_chipset_flush
3062 * 4. Updated (written) by CPU again
3064 * flush_domains = 0 (no previous write domain)
3065 * invalidate_domains = 0 (no new read domains)
3068 * flush_domains = CPU
3069 * invalidate_domains = RENDER
3072 * drm_agp_chipset_flush
3075 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3076 struct intel_ring_buffer *ring)
3078 struct drm_device *dev = obj->dev;
3079 struct drm_i915_private *dev_priv = dev->dev_private;
3080 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3081 uint32_t invalidate_domains = 0;
3082 uint32_t flush_domains = 0;
3083 uint32_t old_read_domains;
3085 intel_mark_busy(dev, obj);
3088 * If the object isn't moving to a new write domain,
3089 * let the object stay in multiple read domains
3091 if (obj->pending_write_domain == 0)
3092 obj->pending_read_domains |= obj->read_domains;
3094 obj_priv->dirty = 1;
3097 * Flush the current write domain if
3098 * the new read domains don't match. Invalidate
3099 * any read domains which differ from the old
3102 if (obj->write_domain &&
3103 (obj->write_domain != obj->pending_read_domains ||
3104 obj_priv->ring != ring)) {
3105 flush_domains |= obj->write_domain;
3106 invalidate_domains |=
3107 obj->pending_read_domains & ~obj->write_domain;
3110 * Invalidate any read caches which may have
3111 * stale data. That is, any new read domains.
3113 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3114 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3115 i915_gem_clflush_object(obj);
3117 old_read_domains = obj->read_domains;
3119 /* The actual obj->write_domain will be updated with
3120 * pending_write_domain after we emit the accumulated flush for all
3121 * of our domain changes in execbuffers (which clears objects'
3122 * write_domains). So if we have a current write domain that we
3123 * aren't changing, set pending_write_domain to that.
3125 if (flush_domains == 0 && obj->pending_write_domain == 0)
3126 obj->pending_write_domain = obj->write_domain;
3127 obj->read_domains = obj->pending_read_domains;
3129 dev->invalidate_domains |= invalidate_domains;
3130 dev->flush_domains |= flush_domains;
3131 if (flush_domains & I915_GEM_GPU_DOMAINS)
3132 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3133 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3134 dev_priv->mm.flush_rings |= ring->id;
3136 trace_i915_gem_object_change_domain(obj,
3142 * Moves the object from a partially CPU read to a full one.
3144 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3145 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3148 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3150 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3152 if (!obj_priv->page_cpu_valid)
3155 /* If we're partially in the CPU read domain, finish moving it in.
3157 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3160 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3161 if (obj_priv->page_cpu_valid[i])
3163 drm_clflush_pages(obj_priv->pages + i, 1);
3167 /* Free the page_cpu_valid mappings which are now stale, whether
3168 * or not we've got I915_GEM_DOMAIN_CPU.
3170 kfree(obj_priv->page_cpu_valid);
3171 obj_priv->page_cpu_valid = NULL;
3175 * Set the CPU read domain on a range of the object.
3177 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3178 * not entirely valid. The page_cpu_valid member of the object flags which
3179 * pages have been flushed, and will be respected by
3180 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3181 * of the whole object.
3183 * This function returns when the move is complete, including waiting on
3187 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3188 uint64_t offset, uint64_t size)
3190 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3191 uint32_t old_read_domains;
3194 if (offset == 0 && size == obj->size)
3195 return i915_gem_object_set_to_cpu_domain(obj, 0);
3197 ret = i915_gem_object_flush_gpu_write_domain(obj);
3200 ret = i915_gem_object_wait_rendering(obj, true);
3204 i915_gem_object_flush_gtt_write_domain(obj);
3206 /* If we're already fully in the CPU read domain, we're done. */
3207 if (obj_priv->page_cpu_valid == NULL &&
3208 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3211 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3212 * newly adding I915_GEM_DOMAIN_CPU
3214 if (obj_priv->page_cpu_valid == NULL) {
3215 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3217 if (obj_priv->page_cpu_valid == NULL)
3219 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3220 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3222 /* Flush the cache on any pages that are still invalid from the CPU's
3225 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3227 if (obj_priv->page_cpu_valid[i])
3230 drm_clflush_pages(obj_priv->pages + i, 1);
3232 obj_priv->page_cpu_valid[i] = 1;
3235 /* It should now be out of any other write domains, and we can update
3236 * the domain values for our changes.
3238 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3240 old_read_domains = obj->read_domains;
3241 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3243 trace_i915_gem_object_change_domain(obj,
3251 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3252 struct drm_file *file_priv,
3253 struct drm_i915_gem_exec_object2 *entry,
3254 struct drm_i915_gem_relocation_entry *reloc)
3256 struct drm_device *dev = obj->base.dev;
3257 struct drm_gem_object *target_obj;
3258 uint32_t target_offset;
3261 target_obj = drm_gem_object_lookup(dev, file_priv,
3262 reloc->target_handle);
3263 if (target_obj == NULL)
3266 target_offset = to_intel_bo(target_obj)->gtt_offset;
3269 DRM_INFO("%s: obj %p offset %08x target %d "
3270 "read %08x write %08x gtt %08x "
3271 "presumed %08x delta %08x\n",
3274 (int) reloc->offset,
3275 (int) reloc->target_handle,
3276 (int) reloc->read_domains,
3277 (int) reloc->write_domain,
3278 (int) target_offset,
3279 (int) reloc->presumed_offset,
3283 /* The target buffer should have appeared before us in the
3284 * exec_object list, so it should have a GTT space bound by now.
3286 if (target_offset == 0) {
3287 DRM_ERROR("No GTT space found for object %d\n",
3288 reloc->target_handle);
3292 /* Validate that the target is in a valid r/w GPU domain */
3293 if (reloc->write_domain & (reloc->write_domain - 1)) {
3294 DRM_ERROR("reloc with multiple write domains: "
3295 "obj %p target %d offset %d "
3296 "read %08x write %08x",
3297 obj, reloc->target_handle,
3298 (int) reloc->offset,
3299 reloc->read_domains,
3300 reloc->write_domain);
3303 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3304 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3305 DRM_ERROR("reloc with read/write CPU domains: "
3306 "obj %p target %d offset %d "
3307 "read %08x write %08x",
3308 obj, reloc->target_handle,
3309 (int) reloc->offset,
3310 reloc->read_domains,
3311 reloc->write_domain);
3314 if (reloc->write_domain && target_obj->pending_write_domain &&
3315 reloc->write_domain != target_obj->pending_write_domain) {
3316 DRM_ERROR("Write domain conflict: "
3317 "obj %p target %d offset %d "
3318 "new %08x old %08x\n",
3319 obj, reloc->target_handle,
3320 (int) reloc->offset,
3321 reloc->write_domain,
3322 target_obj->pending_write_domain);
3326 target_obj->pending_read_domains |= reloc->read_domains;
3327 target_obj->pending_write_domain |= reloc->write_domain;
3329 /* If the relocation already has the right value in it, no
3330 * more work needs to be done.
3332 if (target_offset == reloc->presumed_offset)
3335 /* Check that the relocation address is valid... */
3336 if (reloc->offset > obj->base.size - 4) {
3337 DRM_ERROR("Relocation beyond object bounds: "
3338 "obj %p target %d offset %d size %d.\n",
3339 obj, reloc->target_handle,
3340 (int) reloc->offset,
3341 (int) obj->base.size);
3344 if (reloc->offset & 3) {
3345 DRM_ERROR("Relocation not 4-byte aligned: "
3346 "obj %p target %d offset %d.\n",
3347 obj, reloc->target_handle,
3348 (int) reloc->offset);
3352 /* and points to somewhere within the target object. */
3353 if (reloc->delta >= target_obj->size) {
3354 DRM_ERROR("Relocation beyond target object bounds: "
3355 "obj %p target %d delta %d size %d.\n",
3356 obj, reloc->target_handle,
3358 (int) target_obj->size);
3362 reloc->delta += target_offset;
3363 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3364 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3367 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3368 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3369 kunmap_atomic(vaddr);
3371 struct drm_i915_private *dev_priv = dev->dev_private;
3372 uint32_t __iomem *reloc_entry;
3373 void __iomem *reloc_page;
3375 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3379 /* Map the page containing the relocation we're going to perform. */
3380 reloc->offset += obj->gtt_offset;
3381 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3382 reloc->offset & PAGE_MASK);
3383 reloc_entry = (uint32_t __iomem *)
3384 (reloc_page + (reloc->offset & ~PAGE_MASK));
3385 iowrite32(reloc->delta, reloc_entry);
3386 io_mapping_unmap_atomic(reloc_page);
3389 /* and update the user's relocation entry */
3390 reloc->presumed_offset = target_offset;
3395 drm_gem_object_unreference(target_obj);
3400 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3401 struct drm_file *file_priv,
3402 struct drm_i915_gem_exec_object2 *entry)
3404 struct drm_i915_gem_relocation_entry __user *user_relocs;
3407 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3408 for (i = 0; i < entry->relocation_count; i++) {
3409 struct drm_i915_gem_relocation_entry reloc;
3411 if (__copy_from_user_inatomic(&reloc,
3416 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3420 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3421 &reloc.presumed_offset,
3422 sizeof(reloc.presumed_offset)))
3430 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3431 struct drm_file *file_priv,
3432 struct drm_i915_gem_exec_object2 *entry,
3433 struct drm_i915_gem_relocation_entry *relocs)
3437 for (i = 0; i < entry->relocation_count; i++) {
3438 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3447 i915_gem_execbuffer_relocate(struct drm_device *dev,
3448 struct drm_file *file,
3449 struct drm_gem_object **object_list,
3450 struct drm_i915_gem_exec_object2 *exec_list,
3455 for (i = 0; i < count; i++) {
3456 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3457 obj->base.pending_read_domains = 0;
3458 obj->base.pending_write_domain = 0;
3459 ret = i915_gem_execbuffer_relocate_object(obj, file,
3469 i915_gem_execbuffer_reserve(struct drm_device *dev,
3470 struct drm_file *file,
3471 struct drm_gem_object **object_list,
3472 struct drm_i915_gem_exec_object2 *exec_list,
3475 struct drm_i915_private *dev_priv = dev->dev_private;
3478 /* attempt to pin all of the buffers into the GTT */
3479 for (retry = 0; retry < 2; retry++) {
3481 for (i = 0; i < count; i++) {
3482 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3483 struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3485 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3486 obj->tiling_mode != I915_TILING_NONE;
3488 /* Check fence reg constraints and rebind if necessary */
3490 !i915_gem_object_fence_offset_ok(&obj->base,
3491 obj->tiling_mode)) {
3492 ret = i915_gem_object_unbind(&obj->base);
3497 ret = i915_gem_object_pin(&obj->base, entry->alignment);
3502 * Pre-965 chips need a fence register set up in order
3503 * to properly handle blits to/from tiled surfaces.
3506 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3508 i915_gem_object_unpin(&obj->base);
3512 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3515 entry->offset = obj->gtt_offset;
3519 i915_gem_object_unpin(object_list[i]);
3524 if (ret != -ENOSPC || retry)
3527 ret = i915_gem_evict_everything(dev);
3536 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3537 struct drm_file *file,
3538 struct drm_gem_object **object_list,
3539 struct drm_i915_gem_exec_object2 *exec_list,
3542 struct drm_i915_gem_relocation_entry *reloc;
3545 for (i = 0; i < count; i++) {
3546 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3547 obj->in_execbuffer = false;
3550 mutex_unlock(&dev->struct_mutex);
3553 for (i = 0; i < count; i++)
3554 total += exec_list[i].relocation_count;
3556 reloc = drm_malloc_ab(total, sizeof(*reloc));
3557 if (reloc == NULL) {
3558 mutex_lock(&dev->struct_mutex);
3563 for (i = 0; i < count; i++) {
3564 struct drm_i915_gem_relocation_entry __user *user_relocs;
3566 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3568 if (copy_from_user(reloc+total, user_relocs,
3569 exec_list[i].relocation_count *
3572 mutex_lock(&dev->struct_mutex);
3576 total += exec_list[i].relocation_count;
3579 ret = i915_mutex_lock_interruptible(dev);
3581 mutex_lock(&dev->struct_mutex);
3585 ret = i915_gem_execbuffer_reserve(dev, file,
3586 object_list, exec_list,
3592 for (i = 0; i < count; i++) {
3593 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3594 obj->base.pending_read_domains = 0;
3595 obj->base.pending_write_domain = 0;
3596 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3602 total += exec_list[i].relocation_count;
3605 /* Leave the user relocations as are, this is the painfully slow path,
3606 * and we want to avoid the complication of dropping the lock whilst
3607 * having buffers reserved in the aperture and so causing spurious
3608 * ENOSPC for random operations.
3612 drm_free_large(reloc);
3617 i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3618 struct drm_file *file,
3619 struct intel_ring_buffer *ring,
3620 struct drm_gem_object **objects,
3623 struct drm_i915_private *dev_priv = dev->dev_private;
3626 /* Zero the global flush/invalidate flags. These
3627 * will be modified as new domains are computed
3630 dev->invalidate_domains = 0;
3631 dev->flush_domains = 0;
3632 dev_priv->mm.flush_rings = 0;
3633 for (i = 0; i < count; i++)
3634 i915_gem_object_set_to_gpu_domain(objects[i], ring);
3636 if (dev->invalidate_domains | dev->flush_domains) {
3638 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3640 dev->invalidate_domains,
3641 dev->flush_domains);
3643 i915_gem_flush(dev, file,
3644 dev->invalidate_domains,
3646 dev_priv->mm.flush_rings);
3649 for (i = 0; i < count; i++) {
3650 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3651 /* XXX replace with semaphores */
3652 if (obj->ring && ring != obj->ring) {
3653 ret = i915_gem_object_wait_rendering(&obj->base, true);
3662 /* Throttle our rendering by waiting until the ring has completed our requests
3663 * emitted over 20 msec ago.
3665 * Note that if we were to use the current jiffies each time around the loop,
3666 * we wouldn't escape the function with any frames outstanding if the time to
3667 * render a frame was over 20ms.
3669 * This should get us reasonable parallelism between CPU and GPU but also
3670 * relatively low latency when blocking on a particular request to finish.
3673 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3675 struct drm_i915_private *dev_priv = dev->dev_private;
3676 struct drm_i915_file_private *file_priv = file->driver_priv;
3677 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3678 struct drm_i915_gem_request *request;
3679 struct intel_ring_buffer *ring = NULL;
3683 spin_lock(&file_priv->mm.lock);
3684 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3685 if (time_after_eq(request->emitted_jiffies, recent_enough))
3688 ring = request->ring;
3689 seqno = request->seqno;
3691 spin_unlock(&file_priv->mm.lock);
3697 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3698 /* And wait for the seqno passing without holding any locks and
3699 * causing extra latency for others. This is safe as the irq
3700 * generation is designed to be run atomically and so is
3703 ring->user_irq_get(dev, ring);
3704 ret = wait_event_interruptible(ring->irq_queue,
3705 i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3706 || atomic_read(&dev_priv->mm.wedged));
3707 ring->user_irq_put(dev, ring);
3709 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3714 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3720 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3721 uint64_t exec_offset)
3723 uint32_t exec_start, exec_len;
3725 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3726 exec_len = (uint32_t) exec->batch_len;
3728 if ((exec_start | exec_len) & 0x7)
3738 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3743 for (i = 0; i < count; i++) {
3744 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3745 int length; /* limited by fault_in_pages_readable() */
3747 /* First check for malicious input causing overflow */
3748 if (exec[i].relocation_count >
3749 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3752 length = exec[i].relocation_count *
3753 sizeof(struct drm_i915_gem_relocation_entry);
3754 if (!access_ok(VERIFY_READ, ptr, length))
3757 /* we may also need to update the presumed offsets */
3758 if (!access_ok(VERIFY_WRITE, ptr, length))
3761 if (fault_in_pages_readable(ptr, length))
3769 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3770 struct drm_file *file,
3771 struct drm_i915_gem_execbuffer2 *args,
3772 struct drm_i915_gem_exec_object2 *exec_list)
3774 drm_i915_private_t *dev_priv = dev->dev_private;
3775 struct drm_gem_object **object_list = NULL;
3776 struct drm_gem_object *batch_obj;
3777 struct drm_i915_gem_object *obj_priv;
3778 struct drm_clip_rect *cliprects = NULL;
3779 struct drm_i915_gem_request *request = NULL;
3781 uint64_t exec_offset;
3783 struct intel_ring_buffer *ring = NULL;
3785 ret = i915_gem_check_is_wedged(dev);
3789 ret = validate_exec_list(exec_list, args->buffer_count);
3794 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3795 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3797 switch (args->flags & I915_EXEC_RING_MASK) {
3798 case I915_EXEC_DEFAULT:
3799 case I915_EXEC_RENDER:
3800 ring = &dev_priv->render_ring;
3803 if (!HAS_BSD(dev)) {
3804 DRM_ERROR("execbuf with invalid ring (BSD)\n");
3807 ring = &dev_priv->bsd_ring;
3810 if (!HAS_BLT(dev)) {
3811 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3814 ring = &dev_priv->blt_ring;
3817 DRM_ERROR("execbuf with unknown ring: %d\n",
3818 (int)(args->flags & I915_EXEC_RING_MASK));
3822 if (args->buffer_count < 1) {
3823 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3826 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3827 if (object_list == NULL) {
3828 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3829 args->buffer_count);
3834 if (args->num_cliprects != 0) {
3835 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3837 if (cliprects == NULL) {
3842 ret = copy_from_user(cliprects,
3843 (struct drm_clip_rect __user *)
3844 (uintptr_t) args->cliprects_ptr,
3845 sizeof(*cliprects) * args->num_cliprects);
3847 DRM_ERROR("copy %d cliprects failed: %d\n",
3848 args->num_cliprects, ret);
3854 request = kzalloc(sizeof(*request), GFP_KERNEL);
3855 if (request == NULL) {
3860 ret = i915_mutex_lock_interruptible(dev);
3864 if (dev_priv->mm.suspended) {
3865 mutex_unlock(&dev->struct_mutex);
3870 /* Look up object handles */
3871 for (i = 0; i < args->buffer_count; i++) {
3872 object_list[i] = drm_gem_object_lookup(dev, file,
3873 exec_list[i].handle);
3874 if (object_list[i] == NULL) {
3875 DRM_ERROR("Invalid object handle %d at index %d\n",
3876 exec_list[i].handle, i);
3877 /* prevent error path from reading uninitialized data */
3878 args->buffer_count = i + 1;
3883 obj_priv = to_intel_bo(object_list[i]);
3884 if (obj_priv->in_execbuffer) {
3885 DRM_ERROR("Object %p appears more than once in object list\n",
3887 /* prevent error path from reading uninitialized data */
3888 args->buffer_count = i + 1;
3892 obj_priv->in_execbuffer = true;
3895 /* Move the objects en-masse into the GTT, evicting if necessary. */
3896 ret = i915_gem_execbuffer_reserve(dev, file,
3897 object_list, exec_list,
3898 args->buffer_count);
3902 /* The objects are in their final locations, apply the relocations. */
3903 ret = i915_gem_execbuffer_relocate(dev, file,
3904 object_list, exec_list,
3905 args->buffer_count);
3907 if (ret == -EFAULT) {
3908 ret = i915_gem_execbuffer_relocate_slow(dev, file,
3911 args->buffer_count);
3912 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3918 /* Set the pending read domains for the batch buffer to COMMAND */
3919 batch_obj = object_list[args->buffer_count-1];
3920 if (batch_obj->pending_write_domain) {
3921 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3925 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3927 /* Sanity check the batch buffer */
3928 exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3929 ret = i915_gem_check_execbuffer(args, exec_offset);
3931 DRM_ERROR("execbuf with invalid offset/length\n");
3935 ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3936 object_list, args->buffer_count);
3940 for (i = 0; i < args->buffer_count; i++) {
3941 struct drm_gem_object *obj = object_list[i];
3942 uint32_t old_write_domain = obj->write_domain;
3943 obj->write_domain = obj->pending_write_domain;
3944 trace_i915_gem_object_change_domain(obj,
3950 for (i = 0; i < args->buffer_count; i++) {
3951 i915_gem_object_check_coherency(object_list[i],
3952 exec_list[i].handle);
3957 i915_gem_dump_object(batch_obj,
3963 /* Check for any pending flips. As we only maintain a flip queue depth
3964 * of 1, we can simply insert a WAIT for the next display flip prior
3965 * to executing the batch and avoid stalling the CPU.
3968 for (i = 0; i < args->buffer_count; i++) {
3969 if (object_list[i]->write_domain)
3970 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3973 int plane, flip_mask;
3975 for (plane = 0; flips >> plane; plane++) {
3976 if (((flips >> plane) & 1) == 0)
3980 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3982 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3984 intel_ring_begin(dev, ring, 2);
3985 intel_ring_emit(dev, ring,
3986 MI_WAIT_FOR_EVENT | flip_mask);
3987 intel_ring_emit(dev, ring, MI_NOOP);
3988 intel_ring_advance(dev, ring);
3992 /* Exec the batchbuffer */
3993 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3994 cliprects, exec_offset);
3996 DRM_ERROR("dispatch failed %d\n", ret);
4001 * Ensure that the commands in the batch buffer are
4002 * finished before the interrupt fires
4004 i915_retire_commands(dev, ring);
4006 for (i = 0; i < args->buffer_count; i++) {
4007 struct drm_gem_object *obj = object_list[i];
4009 i915_gem_object_move_to_active(obj, ring);
4010 if (obj->write_domain)
4011 list_move_tail(&to_intel_bo(obj)->gpu_write_list,
4012 &ring->gpu_write_list);
4015 i915_add_request(dev, file, request, ring);
4019 for (i = 0; i < args->buffer_count; i++) {
4020 if (object_list[i]) {
4021 obj_priv = to_intel_bo(object_list[i]);
4022 obj_priv->in_execbuffer = false;
4024 drm_gem_object_unreference(object_list[i]);
4027 mutex_unlock(&dev->struct_mutex);
4030 drm_free_large(object_list);
4038 * Legacy execbuffer just creates an exec2 list from the original exec object
4039 * list array and passes it to the real function.
4042 i915_gem_execbuffer(struct drm_device *dev, void *data,
4043 struct drm_file *file_priv)
4045 struct drm_i915_gem_execbuffer *args = data;
4046 struct drm_i915_gem_execbuffer2 exec2;
4047 struct drm_i915_gem_exec_object *exec_list = NULL;
4048 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4052 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4053 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4056 if (args->buffer_count < 1) {
4057 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4061 /* Copy in the exec list from userland */
4062 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4063 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4064 if (exec_list == NULL || exec2_list == NULL) {
4065 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4066 args->buffer_count);
4067 drm_free_large(exec_list);
4068 drm_free_large(exec2_list);
4071 ret = copy_from_user(exec_list,
4072 (struct drm_i915_relocation_entry __user *)
4073 (uintptr_t) args->buffers_ptr,
4074 sizeof(*exec_list) * args->buffer_count);
4076 DRM_ERROR("copy %d exec entries failed %d\n",
4077 args->buffer_count, ret);
4078 drm_free_large(exec_list);
4079 drm_free_large(exec2_list);
4083 for (i = 0; i < args->buffer_count; i++) {
4084 exec2_list[i].handle = exec_list[i].handle;
4085 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4086 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4087 exec2_list[i].alignment = exec_list[i].alignment;
4088 exec2_list[i].offset = exec_list[i].offset;
4089 if (INTEL_INFO(dev)->gen < 4)
4090 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4092 exec2_list[i].flags = 0;
4095 exec2.buffers_ptr = args->buffers_ptr;
4096 exec2.buffer_count = args->buffer_count;
4097 exec2.batch_start_offset = args->batch_start_offset;
4098 exec2.batch_len = args->batch_len;
4099 exec2.DR1 = args->DR1;
4100 exec2.DR4 = args->DR4;
4101 exec2.num_cliprects = args->num_cliprects;
4102 exec2.cliprects_ptr = args->cliprects_ptr;
4103 exec2.flags = I915_EXEC_RENDER;
4105 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4107 /* Copy the new buffer offsets back to the user's exec list. */
4108 for (i = 0; i < args->buffer_count; i++)
4109 exec_list[i].offset = exec2_list[i].offset;
4110 /* ... and back out to userspace */
4111 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4112 (uintptr_t) args->buffers_ptr,
4114 sizeof(*exec_list) * args->buffer_count);
4117 DRM_ERROR("failed to copy %d exec entries "
4118 "back to user (%d)\n",
4119 args->buffer_count, ret);
4123 drm_free_large(exec_list);
4124 drm_free_large(exec2_list);
4129 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4130 struct drm_file *file_priv)
4132 struct drm_i915_gem_execbuffer2 *args = data;
4133 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4137 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4138 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4141 if (args->buffer_count < 1) {
4142 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4146 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4147 if (exec2_list == NULL) {
4148 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4149 args->buffer_count);
4152 ret = copy_from_user(exec2_list,
4153 (struct drm_i915_relocation_entry __user *)
4154 (uintptr_t) args->buffers_ptr,
4155 sizeof(*exec2_list) * args->buffer_count);
4157 DRM_ERROR("copy %d exec entries failed %d\n",
4158 args->buffer_count, ret);
4159 drm_free_large(exec2_list);
4163 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4165 /* Copy the new buffer offsets back to the user's exec list. */
4166 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4167 (uintptr_t) args->buffers_ptr,
4169 sizeof(*exec2_list) * args->buffer_count);
4172 DRM_ERROR("failed to copy %d exec entries "
4173 "back to user (%d)\n",
4174 args->buffer_count, ret);
4178 drm_free_large(exec2_list);
4183 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4185 struct drm_device *dev = obj->dev;
4186 struct drm_i915_private *dev_priv = dev->dev_private;
4187 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4190 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4191 WARN_ON(i915_verify_lists(dev));
4193 if (obj_priv->gtt_space != NULL) {
4195 alignment = i915_gem_get_gtt_alignment(obj);
4196 if (obj_priv->gtt_offset & (alignment - 1)) {
4197 WARN(obj_priv->pin_count,
4198 "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n",
4199 obj_priv->gtt_offset, alignment);
4200 ret = i915_gem_object_unbind(obj);
4206 if (obj_priv->gtt_space == NULL) {
4207 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4212 obj_priv->pin_count++;
4214 /* If the object is not active and not pending a flush,
4215 * remove it from the inactive list
4217 if (obj_priv->pin_count == 1) {
4218 i915_gem_info_add_pin(dev_priv, obj->size);
4219 if (!obj_priv->active)
4220 list_move_tail(&obj_priv->mm_list,
4221 &dev_priv->mm.pinned_list);
4224 WARN_ON(i915_verify_lists(dev));
4229 i915_gem_object_unpin(struct drm_gem_object *obj)
4231 struct drm_device *dev = obj->dev;
4232 drm_i915_private_t *dev_priv = dev->dev_private;
4233 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4235 WARN_ON(i915_verify_lists(dev));
4236 obj_priv->pin_count--;
4237 BUG_ON(obj_priv->pin_count < 0);
4238 BUG_ON(obj_priv->gtt_space == NULL);
4240 /* If the object is no longer pinned, and is
4241 * neither active nor being flushed, then stick it on
4244 if (obj_priv->pin_count == 0) {
4245 if (!obj_priv->active)
4246 list_move_tail(&obj_priv->mm_list,
4247 &dev_priv->mm.inactive_list);
4248 i915_gem_info_remove_pin(dev_priv, obj->size);
4250 WARN_ON(i915_verify_lists(dev));
4254 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4255 struct drm_file *file_priv)
4257 struct drm_i915_gem_pin *args = data;
4258 struct drm_gem_object *obj;
4259 struct drm_i915_gem_object *obj_priv;
4262 ret = i915_mutex_lock_interruptible(dev);
4266 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4271 obj_priv = to_intel_bo(obj);
4273 if (obj_priv->madv != I915_MADV_WILLNEED) {
4274 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4279 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4280 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4286 obj_priv->user_pin_count++;
4287 obj_priv->pin_filp = file_priv;
4288 if (obj_priv->user_pin_count == 1) {
4289 ret = i915_gem_object_pin(obj, args->alignment);
4294 /* XXX - flush the CPU caches for pinned objects
4295 * as the X server doesn't manage domains yet
4297 i915_gem_object_flush_cpu_write_domain(obj);
4298 args->offset = obj_priv->gtt_offset;
4300 drm_gem_object_unreference(obj);
4302 mutex_unlock(&dev->struct_mutex);
4307 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4308 struct drm_file *file_priv)
4310 struct drm_i915_gem_pin *args = data;
4311 struct drm_gem_object *obj;
4312 struct drm_i915_gem_object *obj_priv;
4315 ret = i915_mutex_lock_interruptible(dev);
4319 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4324 obj_priv = to_intel_bo(obj);
4326 if (obj_priv->pin_filp != file_priv) {
4327 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4332 obj_priv->user_pin_count--;
4333 if (obj_priv->user_pin_count == 0) {
4334 obj_priv->pin_filp = NULL;
4335 i915_gem_object_unpin(obj);
4339 drm_gem_object_unreference(obj);
4341 mutex_unlock(&dev->struct_mutex);
4346 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4347 struct drm_file *file_priv)
4349 struct drm_i915_gem_busy *args = data;
4350 struct drm_gem_object *obj;
4351 struct drm_i915_gem_object *obj_priv;
4354 ret = i915_mutex_lock_interruptible(dev);
4358 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4363 obj_priv = to_intel_bo(obj);
4365 /* Count all active objects as busy, even if they are currently not used
4366 * by the gpu. Users of this interface expect objects to eventually
4367 * become non-busy without any further actions, therefore emit any
4368 * necessary flushes here.
4370 args->busy = obj_priv->active;
4372 /* Unconditionally flush objects, even when the gpu still uses this
4373 * object. Userspace calling this function indicates that it wants to
4374 * use this buffer rather sooner than later, so issuing the required
4375 * flush earlier is beneficial.
4377 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4378 i915_gem_flush_ring(dev, file_priv,
4380 0, obj->write_domain);
4382 /* Update the active list for the hardware's current position.
4383 * Otherwise this only updates on a delayed timer or when irqs
4384 * are actually unmasked, and our working set ends up being
4385 * larger than required.
4387 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4389 args->busy = obj_priv->active;
4392 drm_gem_object_unreference(obj);
4394 mutex_unlock(&dev->struct_mutex);
4399 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4400 struct drm_file *file_priv)
4402 return i915_gem_ring_throttle(dev, file_priv);
4406 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4407 struct drm_file *file_priv)
4409 struct drm_i915_gem_madvise *args = data;
4410 struct drm_gem_object *obj;
4411 struct drm_i915_gem_object *obj_priv;
4414 switch (args->madv) {
4415 case I915_MADV_DONTNEED:
4416 case I915_MADV_WILLNEED:
4422 ret = i915_mutex_lock_interruptible(dev);
4426 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4431 obj_priv = to_intel_bo(obj);
4433 if (obj_priv->pin_count) {
4438 if (obj_priv->madv != __I915_MADV_PURGED)
4439 obj_priv->madv = args->madv;
4441 /* if the object is no longer bound, discard its backing storage */
4442 if (i915_gem_object_is_purgeable(obj_priv) &&
4443 obj_priv->gtt_space == NULL)
4444 i915_gem_object_truncate(obj);
4446 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4449 drm_gem_object_unreference(obj);
4451 mutex_unlock(&dev->struct_mutex);
4455 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4458 struct drm_i915_private *dev_priv = dev->dev_private;
4459 struct drm_i915_gem_object *obj;
4461 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4465 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4470 i915_gem_info_add_obj(dev_priv, size);
4472 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4473 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4475 obj->agp_type = AGP_USER_MEMORY;
4476 obj->base.driver_private = NULL;
4477 obj->fence_reg = I915_FENCE_REG_NONE;
4478 INIT_LIST_HEAD(&obj->mm_list);
4479 INIT_LIST_HEAD(&obj->ring_list);
4480 INIT_LIST_HEAD(&obj->gpu_write_list);
4481 obj->madv = I915_MADV_WILLNEED;
4486 int i915_gem_init_object(struct drm_gem_object *obj)
4493 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4495 struct drm_device *dev = obj->dev;
4496 drm_i915_private_t *dev_priv = dev->dev_private;
4497 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4500 ret = i915_gem_object_unbind(obj);
4501 if (ret == -ERESTARTSYS) {
4502 list_move(&obj_priv->mm_list,
4503 &dev_priv->mm.deferred_free_list);
4507 if (obj_priv->mmap_offset)
4508 i915_gem_free_mmap_offset(obj);
4510 drm_gem_object_release(obj);
4511 i915_gem_info_remove_obj(dev_priv, obj->size);
4513 kfree(obj_priv->page_cpu_valid);
4514 kfree(obj_priv->bit_17);
4518 void i915_gem_free_object(struct drm_gem_object *obj)
4520 struct drm_device *dev = obj->dev;
4521 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4523 trace_i915_gem_object_destroy(obj);
4525 while (obj_priv->pin_count > 0)
4526 i915_gem_object_unpin(obj);
4528 if (obj_priv->phys_obj)
4529 i915_gem_detach_phys_object(dev, obj);
4531 i915_gem_free_object_tail(obj);
4535 i915_gem_idle(struct drm_device *dev)
4537 drm_i915_private_t *dev_priv = dev->dev_private;
4540 mutex_lock(&dev->struct_mutex);
4542 if (dev_priv->mm.suspended) {
4543 mutex_unlock(&dev->struct_mutex);
4547 ret = i915_gpu_idle(dev);
4549 mutex_unlock(&dev->struct_mutex);
4553 /* Under UMS, be paranoid and evict. */
4554 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4555 ret = i915_gem_evict_inactive(dev);
4557 mutex_unlock(&dev->struct_mutex);
4562 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4563 * We need to replace this with a semaphore, or something.
4564 * And not confound mm.suspended!
4566 dev_priv->mm.suspended = 1;
4567 del_timer_sync(&dev_priv->hangcheck_timer);
4569 i915_kernel_lost_context(dev);
4570 i915_gem_cleanup_ringbuffer(dev);
4572 mutex_unlock(&dev->struct_mutex);
4574 /* Cancel the retire work handler, which should be idle now. */
4575 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4581 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4582 * over cache flushing.
4585 i915_gem_init_pipe_control(struct drm_device *dev)
4587 drm_i915_private_t *dev_priv = dev->dev_private;
4588 struct drm_gem_object *obj;
4589 struct drm_i915_gem_object *obj_priv;
4592 obj = i915_gem_alloc_object(dev, 4096);
4594 DRM_ERROR("Failed to allocate seqno page\n");
4598 obj_priv = to_intel_bo(obj);
4599 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4601 ret = i915_gem_object_pin(obj, 4096);
4605 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4606 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4607 if (dev_priv->seqno_page == NULL)
4610 dev_priv->seqno_obj = obj;
4611 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4616 i915_gem_object_unpin(obj);
4618 drm_gem_object_unreference(obj);
4625 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4627 drm_i915_private_t *dev_priv = dev->dev_private;
4628 struct drm_gem_object *obj;
4629 struct drm_i915_gem_object *obj_priv;
4631 obj = dev_priv->seqno_obj;
4632 obj_priv = to_intel_bo(obj);
4633 kunmap(obj_priv->pages[0]);
4634 i915_gem_object_unpin(obj);
4635 drm_gem_object_unreference(obj);
4636 dev_priv->seqno_obj = NULL;
4638 dev_priv->seqno_page = NULL;
4642 i915_gem_init_ringbuffer(struct drm_device *dev)
4644 drm_i915_private_t *dev_priv = dev->dev_private;
4647 if (HAS_PIPE_CONTROL(dev)) {
4648 ret = i915_gem_init_pipe_control(dev);
4653 ret = intel_init_render_ring_buffer(dev);
4655 goto cleanup_pipe_control;
4658 ret = intel_init_bsd_ring_buffer(dev);
4660 goto cleanup_render_ring;
4664 ret = intel_init_blt_ring_buffer(dev);
4666 goto cleanup_bsd_ring;
4669 dev_priv->next_seqno = 1;
4674 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4675 cleanup_render_ring:
4676 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4677 cleanup_pipe_control:
4678 if (HAS_PIPE_CONTROL(dev))
4679 i915_gem_cleanup_pipe_control(dev);
4684 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4686 drm_i915_private_t *dev_priv = dev->dev_private;
4688 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4689 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4690 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
4691 if (HAS_PIPE_CONTROL(dev))
4692 i915_gem_cleanup_pipe_control(dev);
4696 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4697 struct drm_file *file_priv)
4699 drm_i915_private_t *dev_priv = dev->dev_private;
4702 if (drm_core_check_feature(dev, DRIVER_MODESET))
4705 if (atomic_read(&dev_priv->mm.wedged)) {
4706 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4707 atomic_set(&dev_priv->mm.wedged, 0);
4710 mutex_lock(&dev->struct_mutex);
4711 dev_priv->mm.suspended = 0;
4713 ret = i915_gem_init_ringbuffer(dev);
4715 mutex_unlock(&dev->struct_mutex);
4719 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4720 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4721 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4722 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4723 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4724 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4725 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4726 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4727 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4728 mutex_unlock(&dev->struct_mutex);
4730 ret = drm_irq_install(dev);
4732 goto cleanup_ringbuffer;
4737 mutex_lock(&dev->struct_mutex);
4738 i915_gem_cleanup_ringbuffer(dev);
4739 dev_priv->mm.suspended = 1;
4740 mutex_unlock(&dev->struct_mutex);
4746 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4747 struct drm_file *file_priv)
4749 if (drm_core_check_feature(dev, DRIVER_MODESET))
4752 drm_irq_uninstall(dev);
4753 return i915_gem_idle(dev);
4757 i915_gem_lastclose(struct drm_device *dev)
4761 if (drm_core_check_feature(dev, DRIVER_MODESET))
4764 ret = i915_gem_idle(dev);
4766 DRM_ERROR("failed to idle hardware: %d\n", ret);
4770 init_ring_lists(struct intel_ring_buffer *ring)
4772 INIT_LIST_HEAD(&ring->active_list);
4773 INIT_LIST_HEAD(&ring->request_list);
4774 INIT_LIST_HEAD(&ring->gpu_write_list);
4778 i915_gem_load(struct drm_device *dev)
4781 drm_i915_private_t *dev_priv = dev->dev_private;
4783 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4784 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4785 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4786 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4787 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4788 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4789 init_ring_lists(&dev_priv->render_ring);
4790 init_ring_lists(&dev_priv->bsd_ring);
4791 init_ring_lists(&dev_priv->blt_ring);
4792 for (i = 0; i < 16; i++)
4793 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4794 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4795 i915_gem_retire_work_handler);
4796 init_completion(&dev_priv->error_completion);
4797 spin_lock(&shrink_list_lock);
4798 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4799 spin_unlock(&shrink_list_lock);
4801 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4803 u32 tmp = I915_READ(MI_ARB_STATE);
4804 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4805 /* arb state is a masked write, so set bit + bit in mask */
4806 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4807 I915_WRITE(MI_ARB_STATE, tmp);
4811 /* Old X drivers will take 0-2 for front, back, depth buffers */
4812 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4813 dev_priv->fence_reg_start = 3;
4815 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4816 dev_priv->num_fence_regs = 16;
4818 dev_priv->num_fence_regs = 8;
4820 /* Initialize fence registers to zero */
4821 switch (INTEL_INFO(dev)->gen) {
4823 for (i = 0; i < 16; i++)
4824 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4828 for (i = 0; i < 16; i++)
4829 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4832 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4833 for (i = 0; i < 8; i++)
4834 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4836 for (i = 0; i < 8; i++)
4837 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4840 i915_gem_detect_bit_6_swizzle(dev);
4841 init_waitqueue_head(&dev_priv->pending_flip_queue);
4845 * Create a physically contiguous memory object for this object
4846 * e.g. for cursor + overlay regs
4848 static int i915_gem_init_phys_object(struct drm_device *dev,
4849 int id, int size, int align)
4851 drm_i915_private_t *dev_priv = dev->dev_private;
4852 struct drm_i915_gem_phys_object *phys_obj;
4855 if (dev_priv->mm.phys_objs[id - 1] || !size)
4858 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4864 phys_obj->handle = drm_pci_alloc(dev, size, align);
4865 if (!phys_obj->handle) {
4870 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4873 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4881 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4883 drm_i915_private_t *dev_priv = dev->dev_private;
4884 struct drm_i915_gem_phys_object *phys_obj;
4886 if (!dev_priv->mm.phys_objs[id - 1])
4889 phys_obj = dev_priv->mm.phys_objs[id - 1];
4890 if (phys_obj->cur_obj) {
4891 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4895 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4897 drm_pci_free(dev, phys_obj->handle);
4899 dev_priv->mm.phys_objs[id - 1] = NULL;
4902 void i915_gem_free_all_phys_object(struct drm_device *dev)
4906 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4907 i915_gem_free_phys_object(dev, i);
4910 void i915_gem_detach_phys_object(struct drm_device *dev,
4911 struct drm_gem_object *obj)
4913 struct drm_i915_gem_object *obj_priv;
4918 obj_priv = to_intel_bo(obj);
4919 if (!obj_priv->phys_obj)
4922 ret = i915_gem_object_get_pages(obj, 0);
4926 page_count = obj->size / PAGE_SIZE;
4928 for (i = 0; i < page_count; i++) {
4929 char *dst = kmap_atomic(obj_priv->pages[i]);
4930 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4932 memcpy(dst, src, PAGE_SIZE);
4935 drm_clflush_pages(obj_priv->pages, page_count);
4936 drm_agp_chipset_flush(dev);
4938 i915_gem_object_put_pages(obj);
4940 obj_priv->phys_obj->cur_obj = NULL;
4941 obj_priv->phys_obj = NULL;
4945 i915_gem_attach_phys_object(struct drm_device *dev,
4946 struct drm_gem_object *obj,
4950 drm_i915_private_t *dev_priv = dev->dev_private;
4951 struct drm_i915_gem_object *obj_priv;
4956 if (id > I915_MAX_PHYS_OBJECT)
4959 obj_priv = to_intel_bo(obj);
4961 if (obj_priv->phys_obj) {
4962 if (obj_priv->phys_obj->id == id)
4964 i915_gem_detach_phys_object(dev, obj);
4967 /* create a new object */
4968 if (!dev_priv->mm.phys_objs[id - 1]) {
4969 ret = i915_gem_init_phys_object(dev, id,
4972 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4977 /* bind to the object */
4978 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4979 obj_priv->phys_obj->cur_obj = obj;
4981 ret = i915_gem_object_get_pages(obj, 0);
4983 DRM_ERROR("failed to get page list\n");
4987 page_count = obj->size / PAGE_SIZE;
4989 for (i = 0; i < page_count; i++) {
4990 char *src = kmap_atomic(obj_priv->pages[i]);
4991 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4993 memcpy(dst, src, PAGE_SIZE);
4997 i915_gem_object_put_pages(obj);
5005 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5006 struct drm_i915_gem_pwrite *args,
5007 struct drm_file *file_priv)
5009 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
5010 void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
5011 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
5013 DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
5015 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5016 unsigned long unwritten;
5018 /* The physical object once assigned is fixed for the lifetime
5019 * of the obj, so we can safely drop the lock and continue
5022 mutex_unlock(&dev->struct_mutex);
5023 unwritten = copy_from_user(vaddr, user_data, args->size);
5024 mutex_lock(&dev->struct_mutex);
5029 drm_agp_chipset_flush(dev);
5033 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5035 struct drm_i915_file_private *file_priv = file->driver_priv;
5037 /* Clean up our request list when the client is going away, so that
5038 * later retire_requests won't dereference our soon-to-be-gone
5041 spin_lock(&file_priv->mm.lock);
5042 while (!list_empty(&file_priv->mm.request_list)) {
5043 struct drm_i915_gem_request *request;
5045 request = list_first_entry(&file_priv->mm.request_list,
5046 struct drm_i915_gem_request,
5048 list_del(&request->client_list);
5049 request->file_priv = NULL;
5051 spin_unlock(&file_priv->mm.lock);
5055 i915_gpu_is_active(struct drm_device *dev)
5057 drm_i915_private_t *dev_priv = dev->dev_private;
5060 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5061 list_empty(&dev_priv->mm.active_list);
5063 return !lists_empty;
5067 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
5069 drm_i915_private_t *dev_priv, *next_dev;
5070 struct drm_i915_gem_object *obj_priv, *next_obj;
5072 int would_deadlock = 1;
5074 /* "fast-path" to count number of available objects */
5075 if (nr_to_scan == 0) {
5076 spin_lock(&shrink_list_lock);
5077 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5078 struct drm_device *dev = dev_priv->dev;
5080 if (mutex_trylock(&dev->struct_mutex)) {
5081 list_for_each_entry(obj_priv,
5082 &dev_priv->mm.inactive_list,
5085 mutex_unlock(&dev->struct_mutex);
5088 spin_unlock(&shrink_list_lock);
5090 return (cnt / 100) * sysctl_vfs_cache_pressure;
5093 spin_lock(&shrink_list_lock);
5096 /* first scan for clean buffers */
5097 list_for_each_entry_safe(dev_priv, next_dev,
5098 &shrink_list, mm.shrink_list) {
5099 struct drm_device *dev = dev_priv->dev;
5101 if (! mutex_trylock(&dev->struct_mutex))
5104 spin_unlock(&shrink_list_lock);
5105 i915_gem_retire_requests(dev);
5107 list_for_each_entry_safe(obj_priv, next_obj,
5108 &dev_priv->mm.inactive_list,
5110 if (i915_gem_object_is_purgeable(obj_priv)) {
5111 i915_gem_object_unbind(&obj_priv->base);
5112 if (--nr_to_scan <= 0)
5117 spin_lock(&shrink_list_lock);
5118 mutex_unlock(&dev->struct_mutex);
5122 if (nr_to_scan <= 0)
5126 /* second pass, evict/count anything still on the inactive list */
5127 list_for_each_entry_safe(dev_priv, next_dev,
5128 &shrink_list, mm.shrink_list) {
5129 struct drm_device *dev = dev_priv->dev;
5131 if (! mutex_trylock(&dev->struct_mutex))
5134 spin_unlock(&shrink_list_lock);
5136 list_for_each_entry_safe(obj_priv, next_obj,
5137 &dev_priv->mm.inactive_list,
5139 if (nr_to_scan > 0) {
5140 i915_gem_object_unbind(&obj_priv->base);
5146 spin_lock(&shrink_list_lock);
5147 mutex_unlock(&dev->struct_mutex);
5156 * We are desperate for pages, so as a last resort, wait
5157 * for the GPU to finish and discard whatever we can.
5158 * This has a dramatic impact to reduce the number of
5159 * OOM-killer events whilst running the GPU aggressively.
5161 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5162 struct drm_device *dev = dev_priv->dev;
5164 if (!mutex_trylock(&dev->struct_mutex))
5167 spin_unlock(&shrink_list_lock);
5169 if (i915_gpu_is_active(dev)) {
5174 spin_lock(&shrink_list_lock);
5175 mutex_unlock(&dev->struct_mutex);
5182 spin_unlock(&shrink_list_lock);
5187 return (cnt / 100) * sysctl_vfs_cache_pressure;
5192 static struct shrinker shrinker = {
5193 .shrink = i915_gem_shrink,
5194 .seeks = DEFAULT_SEEKS,
5198 i915_gem_shrinker_init(void)
5200 register_shrinker(&shrinker);
5204 i915_gem_shrinker_exit(void)
5206 unregister_shrinker(&shrinker);