#include "intel_drv.h"
#include "intel_frontbuffer.h"
-#define DBG_USE_CPU_RELOC 0 /* -1 force GTT relocs; 1 force CPU relocs */
+enum {
+ FORCE_CPU_RELOC = 1,
+ FORCE_GTT_RELOC,
+ FORCE_GPU_RELOC,
+#define DBG_FORCE_RELOC 0 /* choose one of the above! */
+};
+
+#define __EXEC_OBJECT_HAS_REF BIT(31)
+#define __EXEC_OBJECT_HAS_PIN BIT(30)
+#define __EXEC_OBJECT_HAS_FENCE BIT(29)
+#define __EXEC_OBJECT_NEEDS_MAP BIT(28)
+#define __EXEC_OBJECT_NEEDS_BIAS BIT(27)
+#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 27) /* all of the above */
+#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
-#define __EXEC_OBJECT_HAS_PIN (1<<31)
-#define __EXEC_OBJECT_HAS_FENCE (1<<30)
-#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
-#define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
-#define __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
+#define __EXEC_HAS_RELOC BIT(31)
+#define __EXEC_VALIDATED BIT(30)
+#define UPDATE PIN_OFFSET_FIXED
#define BATCH_OFFSET_BIAS (256*1024)
-struct i915_execbuffer_params {
- struct drm_device *dev;
- struct drm_file *file;
- struct i915_vma *batch;
- u32 dispatch_flags;
- u32 args_batch_start_offset;
- struct intel_engine_cs *engine;
- struct i915_gem_context *ctx;
- struct drm_i915_gem_request *request;
-};
+#define __I915_EXEC_ILLEGAL_FLAGS \
+ (__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
-struct eb_vmas {
- struct drm_i915_private *i915;
- struct list_head vmas;
- int and;
- union {
- struct i915_vma *lut[0];
- struct hlist_head buckets[0];
- };
+/**
+ * DOC: User command execution
+ *
+ * Userspace submits commands to be executed on the GPU as an instruction
+ * stream within a GEM object we call a batchbuffer. This instructions may
+ * refer to other GEM objects containing auxiliary state such as kernels,
+ * samplers, render targets and even secondary batchbuffers. Userspace does
+ * not know where in the GPU memory these objects reside and so before the
+ * batchbuffer is passed to the GPU for execution, those addresses in the
+ * batchbuffer and auxiliary objects are updated. This is known as relocation,
+ * or patching. To try and avoid having to relocate each object on the next
+ * execution, userspace is told the location of those objects in this pass,
+ * but this remains just a hint as the kernel may choose a new location for
+ * any object in the future.
+ *
+ * Processing an execbuf ioctl is conceptually split up into a few phases.
+ *
+ * 1. Validation - Ensure all the pointers, handles and flags are valid.
+ * 2. Reservation - Assign GPU address space for every object
+ * 3. Relocation - Update any addresses to point to the final locations
+ * 4. Serialisation - Order the request with respect to its dependencies
+ * 5. Construction - Construct a request to execute the batchbuffer
+ * 6. Submission (at some point in the future execution)
+ *
+ * Reserving resources for the execbuf is the most complicated phase. We
+ * neither want to have to migrate the object in the address space, nor do
+ * we want to have to update any relocations pointing to this object. Ideally,
+ * we want to leave the object where it is and for all the existing relocations
+ * to match. If the object is given a new address, or if userspace thinks the
+ * object is elsewhere, we have to parse all the relocation entries and update
+ * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
+ * all the target addresses in all of its objects match the value in the
+ * relocation entries and that they all match the presumed offsets given by the
+ * list of execbuffer objects. Using this knowledge, we know that if we haven't
+ * moved any buffers, all the relocation entries are valid and we can skip
+ * the update. (If userspace is wrong, the likely outcome is an impromptu GPU
+ * hang.) The requirement for using I915_EXEC_NO_RELOC are:
+ *
+ * The addresses written in the objects must match the corresponding
+ * reloc.presumed_offset which in turn must match the corresponding
+ * execobject.offset.
+ *
+ * Any render targets written to in the batch must be flagged with
+ * EXEC_OBJECT_WRITE.
+ *
+ * To avoid stalling, execobject.offset should match the current
+ * address of that object within the active context.
+ *
+ * The reservation is done is multiple phases. First we try and keep any
+ * object already bound in its current location - so as long as meets the
+ * constraints imposed by the new execbuffer. Any object left unbound after the
+ * first pass is then fitted into any available idle space. If an object does
+ * not fit, all objects are removed from the reservation and the process rerun
+ * after sorting the objects into a priority order (more difficult to fit
+ * objects are tried first). Failing that, the entire VM is cleared and we try
+ * to fit the execbuf once last time before concluding that it simply will not
+ * fit.
+ *
+ * A small complication to all of this is that we allow userspace not only to
+ * specify an alignment and a size for the object in the address space, but
+ * we also allow userspace to specify the exact offset. This objects are
+ * simpler to place (the location is known a priori) all we have to do is make
+ * sure the space is available.
+ *
+ * Once all the objects are in place, patching up the buried pointers to point
+ * to the final locations is a fairly simple job of walking over the relocation
+ * entry arrays, looking up the right address and rewriting the value into
+ * the object. Simple! ... The relocation entries are stored in user memory
+ * and so to access them we have to copy them into a local buffer. That copy
+ * has to avoid taking any pagefaults as they may lead back to a GEM object
+ * requiring the struct_mutex (i.e. recursive deadlock). So once again we split
+ * the relocation into multiple passes. First we try to do everything within an
+ * atomic context (avoid the pagefaults) which requires that we never wait. If
+ * we detect that we may wait, or if we need to fault, then we have to fallback
+ * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
+ * bells yet?) Dropping the mutex means that we lose all the state we have
+ * built up so far for the execbuf and we must reset any global data. However,
+ * we do leave the objects pinned in their final locations - which is a
+ * potential issue for concurrent execbufs. Once we have left the mutex, we can
+ * allocate and copy all the relocation entries into a large array at our
+ * leisure, reacquire the mutex, reclaim all the objects and other state and
+ * then proceed to update any incorrect addresses with the objects.
+ *
+ * As we process the relocation entries, we maintain a record of whether the
+ * object is being written to. Using NORELOC, we expect userspace to provide
+ * this information instead. We also check whether we can skip the relocation
+ * by comparing the expected value inside the relocation entry with the target's
+ * final address. If they differ, we have to map the current object and rewrite
+ * the 4 or 8 byte pointer within.
+ *
+ * Serialising an execbuf is quite simple according to the rules of the GEM
+ * ABI. Execution within each context is ordered by the order of submission.
+ * Writes to any GEM object are in order of submission and are exclusive. Reads
+ * from a GEM object are unordered with respect to other reads, but ordered by
+ * writes. A write submitted after a read cannot occur before the read, and
+ * similarly any read submitted after a write cannot occur before the write.
+ * Writes are ordered between engines such that only one write occurs at any
+ * time (completing any reads beforehand) - using semaphores where available
+ * and CPU serialisation otherwise. Other GEM access obey the same rules, any
+ * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
+ * reads before starting, and any read (either using set-domain or pread) must
+ * flush all GPU writes before starting. (Note we only employ a barrier before,
+ * we currently rely on userspace not concurrently starting a new execution
+ * whilst reading or writing to an object. This may be an advantage or not
+ * depending on how much you trust userspace not to shoot themselves in the
+ * foot.) Serialisation may just result in the request being inserted into
+ * a DAG awaiting its turn, but most simple is to wait on the CPU until
+ * all dependencies are resolved.
+ *
+ * After all of that, is just a matter of closing the request and handing it to
+ * the hardware (well, leaving it in a queue to be executed). However, we also
+ * offer the ability for batchbuffers to be run with elevated privileges so
+ * that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
+ * Before any batch is given extra privileges we first must check that it
+ * contains no nefarious instructions, we check that each instruction is from
+ * our whitelist and all registers are also from an allowed list. We first
+ * copy the user's batchbuffer to a shadow (so that the user doesn't have
+ * access to it, either by the CPU or GPU as we scan it) and then parse each
+ * instruction. If everything is ok, we set a flag telling the hardware to run
+ * the batchbuffer in trusted mode, otherwise the ioctl is rejected.
+ */
+
+struct i915_execbuffer {
+ struct drm_i915_private *i915; /** i915 backpointer */
+ struct drm_file *file; /** per-file lookup tables and limits */
+ struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
+ struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
+
+ struct intel_engine_cs *engine; /** engine to queue the request to */
+ struct i915_gem_context *ctx; /** context for building the request */
+ struct i915_address_space *vm; /** GTT and vma for the request */
+
+ struct drm_i915_gem_request *request; /** our request to build */
+ struct i915_vma *batch; /** identity of the batch obj/vma */
+
+ /** actual size of execobj[] as we may extend it for the cmdparser */
+ unsigned int buffer_count;
+
+ /** list of vma not yet bound during reservation phase */
+ struct list_head unbound;
+
+ /** list of vma that have execobj.relocation_count */
+ struct list_head relocs;
+
+ /**
+ * Track the most recently used object for relocations, as we
+ * frequently have to perform multiple relocations within the same
+ * obj/page
+ */
+ struct reloc_cache {
+ struct drm_mm_node node; /** temporary GTT binding */
+ unsigned long vaddr; /** Current kmap address */
+ unsigned long page; /** Currently mapped page index */
+ unsigned int gen; /** Cached value of INTEL_GEN */
+ bool use_64bit_reloc : 1;
+ bool has_llc : 1;
+ bool has_fence : 1;
+ bool needs_unfenced : 1;
+
+ struct drm_i915_gem_request *rq;
+ u32 *rq_cmd;
+ unsigned int rq_size;
+ } reloc_cache;
+
+ u64 invalid_flags; /** Set of execobj.flags that are invalid */
+ u32 context_flags; /** Set of execobj.flags to insert from the ctx */
+
+ u32 batch_start_offset; /** Location within object of batch */
+ u32 batch_len; /** Length of batch within object */
+ u32 batch_flags; /** Flags composed for emit_bb_start() */
+
+ /**
+ * Indicate either the size of the hastable used to resolve
+ * relocation handles, or if negative that we are using a direct
+ * index into the execobj[].
+ */
+ int lut_size;
+ struct hlist_head *buckets; /** ht for relocation handles */
};
-static struct eb_vmas *
-eb_create(struct drm_i915_private *i915,
- struct drm_i915_gem_execbuffer2 *args)
+/*
+ * As an alternative to creating a hashtable of handle-to-vma for a batch,
+ * we used the last available reserved field in the execobject[] and stash
+ * a link from the execobj to its vma.
+ */
+#define __exec_to_vma(ee) (ee)->rsvd2
+#define exec_to_vma(ee) u64_to_ptr(struct i915_vma, __exec_to_vma(ee))
+
+/*
+ * Used to convert any address to canonical form.
+ * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
+ * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
+ * addresses to be in a canonical form:
+ * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
+ * canonical form [63:48] == [47]."
+ */
+#define GEN8_HIGH_ADDRESS_BIT 47
+static inline u64 gen8_canonical_addr(u64 address)
{
- struct eb_vmas *eb = NULL;
-
- if (args->flags & I915_EXEC_HANDLE_LUT) {
- unsigned size = args->buffer_count;
- size *= sizeof(struct i915_vma *);
- size += sizeof(struct eb_vmas);
- eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
- }
-
- if (eb == NULL) {
- unsigned size = args->buffer_count;
- unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
- BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
- while (count > 2*size)
- count >>= 1;
- eb = kzalloc(count*sizeof(struct hlist_head) +
- sizeof(struct eb_vmas),
- GFP_TEMPORARY);
- if (eb == NULL)
- return eb;
-
- eb->and = count - 1;
- } else
- eb->and = -args->buffer_count;
+ return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
+}
- eb->i915 = i915;
- INIT_LIST_HEAD(&eb->vmas);
- return eb;
+static inline u64 gen8_noncanonical_addr(u64 address)
+{
+ return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0);
}
-static void
-eb_reset(struct eb_vmas *eb)
+static int eb_create(struct i915_execbuffer *eb)
{
- if (eb->and >= 0)
- memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
+ if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
+ unsigned int size = 1 + ilog2(eb->buffer_count);
+
+ /*
+ * Without a 1:1 association between relocation handles and
+ * the execobject[] index, we instead create a hashtable.
+ * We size it dynamically based on available memory, starting
+ * first with 1:1 assocative hash and scaling back until
+ * the allocation succeeds.
+ *
+ * Later on we use a positive lut_size to indicate we are
+ * using this hashtable, and a negative value to indicate a
+ * direct lookup.
+ */
+ do {
+ unsigned int flags;
+
+ /* While we can still reduce the allocation size, don't
+ * raise a warning and allow the allocation to fail.
+ * On the last pass though, we want to try as hard
+ * as possible to perform the allocation and warn
+ * if it fails.
+ */
+ flags = GFP_TEMPORARY;
+ if (size > 1)
+ flags |= __GFP_NORETRY | __GFP_NOWARN;
+
+ eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
+ flags);
+ if (eb->buckets)
+ break;
+ } while (--size);
+
+ if (unlikely(!size))
+ return -ENOMEM;
+
+ eb->lut_size = size;
+ } else {
+ eb->lut_size = -eb->buffer_count;
+ }
+
+ return 0;
}
-static struct i915_vma *
-eb_get_batch(struct eb_vmas *eb)
+static bool
+eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
+ const struct i915_vma *vma)
{
- struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
+ if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+ return true;
+
+ if (vma->node.size < entry->pad_to_size)
+ return true;
+
+ if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
+ return true;
+
+ if (entry->flags & EXEC_OBJECT_PINNED &&
+ vma->node.start != entry->offset)
+ return true;
+
+ if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
+ vma->node.start < BATCH_OFFSET_BIAS)
+ return true;
+
+ if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
+ (vma->node.start + vma->node.size - 1) >> 32)
+ return true;
+
+ return false;
+}
+
+static inline void
+eb_pin_vma(struct i915_execbuffer *eb,
+ struct drm_i915_gem_exec_object2 *entry,
+ struct i915_vma *vma)
+{
+ u64 flags;
+
+ if (vma->node.size)
+ flags = vma->node.start;
+ else
+ flags = entry->offset & PIN_OFFSET_MASK;
+
+ flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
+ if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_GTT))
+ flags |= PIN_GLOBAL;
+
+ if (unlikely(i915_vma_pin(vma, 0, 0, flags)))
+ return;
+
+ if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+ if (unlikely(i915_vma_get_fence(vma))) {
+ i915_vma_unpin(vma);
+ return;
+ }
+
+ if (i915_vma_pin_fence(vma))
+ entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+ }
+
+ entry->flags |= __EXEC_OBJECT_HAS_PIN;
+}
+
+static inline void
+__eb_unreserve_vma(struct i915_vma *vma,
+ const struct drm_i915_gem_exec_object2 *entry)
+{
+ GEM_BUG_ON(!(entry->flags & __EXEC_OBJECT_HAS_PIN));
+
+ if (unlikely(entry->flags & __EXEC_OBJECT_HAS_FENCE))
+ i915_vma_unpin_fence(vma);
+
+ __i915_vma_unpin(vma);
+}
+
+static inline void
+eb_unreserve_vma(struct i915_vma *vma,
+ struct drm_i915_gem_exec_object2 *entry)
+{
+ if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+ return;
+
+ __eb_unreserve_vma(vma, entry);
+ entry->flags &= ~__EXEC_OBJECT_RESERVED;
+}
+
+static int
+eb_validate_vma(struct i915_execbuffer *eb,
+ struct drm_i915_gem_exec_object2 *entry,
+ struct i915_vma *vma)
+{
+ if (unlikely(entry->flags & eb->invalid_flags))
+ return -EINVAL;
+
+ if (unlikely(entry->alignment && !is_power_of_2(entry->alignment)))
+ return -EINVAL;
/*
- * SNA is doing fancy tricks with compressing batch buffers, which leads
- * to negative relocation deltas. Usually that works out ok since the
- * relocate address is still positive, except when the batch is placed
- * very low in the GTT. Ensure this doesn't happen.
- *
- * Note that actual hangs have only been observed on gen7, but for
- * paranoia do it everywhere.
+ * Offset can be used as input (EXEC_OBJECT_PINNED), reject
+ * any non-page-aligned or non-canonical addresses.
*/
- if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
- vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
+ if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
+ entry->offset != gen8_canonical_addr(entry->offset & PAGE_MASK)))
+ return -EINVAL;
- return vma;
+ /* pad_to_size was once a reserved field, so sanitize it */
+ if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
+ if (unlikely(offset_in_page(entry->pad_to_size)))
+ return -EINVAL;
+ } else {
+ entry->pad_to_size = 0;
+ }
+
+ if (unlikely(vma->exec_entry)) {
+ DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
+ entry->handle, (int)(entry - eb->exec));
+ return -EINVAL;
+ }
+
+ /*
+ * From drm_mm perspective address space is continuous,
+ * so from this point we're always using non-canonical
+ * form internally.
+ */
+ entry->offset = gen8_noncanonical_addr(entry->offset);
+
+ return 0;
}
static int
-eb_lookup_vmas(struct eb_vmas *eb,
- struct drm_i915_gem_exec_object2 *exec,
- const struct drm_i915_gem_execbuffer2 *args,
- struct i915_address_space *vm,
- struct drm_file *file)
+eb_add_vma(struct i915_execbuffer *eb,
+ struct drm_i915_gem_exec_object2 *entry,
+ struct i915_vma *vma)
{
- struct drm_i915_gem_object *obj;
- struct list_head objects;
- int i, ret;
+ int err;
- INIT_LIST_HEAD(&objects);
- spin_lock(&file->table_lock);
- /* Grab a reference to the object and release the lock so we can lookup
- * or create the VMA without using GFP_ATOMIC */
- for (i = 0; i < args->buffer_count; i++) {
- obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
- if (obj == NULL) {
- spin_unlock(&file->table_lock);
- DRM_DEBUG("Invalid object handle %d at index %d\n",
- exec[i].handle, i);
- ret = -ENOENT;
- goto err;
+ GEM_BUG_ON(i915_vma_is_closed(vma));
+
+ if (!(eb->args->flags & __EXEC_VALIDATED)) {
+ err = eb_validate_vma(eb, entry, vma);
+ if (unlikely(err))
+ return err;
+ }
+
+ if (eb->lut_size > 0) {
+ vma->exec_handle = entry->handle;
+ hlist_add_head(&vma->exec_node,
+ &eb->buckets[hash_32(entry->handle,
+ eb->lut_size)]);
+ }
+
+ if (entry->relocation_count)
+ list_add_tail(&vma->reloc_link, &eb->relocs);
+
+ if (!eb->reloc_cache.has_fence) {
+ entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
+ } else {
+ if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
+ eb->reloc_cache.needs_unfenced) &&
+ i915_gem_object_is_tiled(vma->obj))
+ entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
+ }
+
+ if (!(entry->flags & EXEC_OBJECT_PINNED))
+ entry->flags |= eb->context_flags;
+
+ /*
+ * Stash a pointer from the vma to execobj, so we can query its flags,
+ * size, alignment etc as provided by the user. Also we stash a pointer
+ * to the vma inside the execobj so that we can use a direct lookup
+ * to find the right target VMA when doing relocations.
+ */
+ vma->exec_entry = entry;
+ __exec_to_vma(entry) = (uintptr_t)vma;
+
+ err = 0;
+ eb_pin_vma(eb, entry, vma);
+ if (eb_vma_misplaced(entry, vma)) {
+ eb_unreserve_vma(vma, entry);
+
+ list_add_tail(&vma->exec_link, &eb->unbound);
+ if (drm_mm_node_allocated(&vma->node))
+ err = i915_vma_unbind(vma);
+ } else {
+ if (entry->offset != vma->node.start) {
+ entry->offset = vma->node.start | UPDATE;
+ eb->args->flags |= __EXEC_HAS_RELOC;
}
+ }
+ return err;
+}
+
+static inline int use_cpu_reloc(const struct reloc_cache *cache,
+ const struct drm_i915_gem_object *obj)
+{
+ if (!i915_gem_object_has_struct_page(obj))
+ return false;
+
+ if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
+ return true;
+
+ if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
+ return false;
+
+ return (cache->has_llc ||
+ obj->cache_dirty ||
+ obj->cache_level != I915_CACHE_NONE);
+}
+
+static int eb_reserve_vma(const struct i915_execbuffer *eb,
+ struct i915_vma *vma)
+{
+ struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+ u64 flags;
+ int err;
+
+ flags = PIN_USER | PIN_NONBLOCK;
+ if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
+ flags |= PIN_GLOBAL;
- if (!list_empty(&obj->obj_exec_link)) {
- spin_unlock(&file->table_lock);
- DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
- obj, exec[i].handle, i);
- ret = -EINVAL;
+ /*
+ * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
+ * limit address to the first 4GBs for unflagged objects.
+ */
+ if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
+ flags |= PIN_ZONE_4G;
+
+ if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+ flags |= PIN_MAPPABLE;
+
+ if (entry->flags & EXEC_OBJECT_PINNED) {
+ flags |= entry->offset | PIN_OFFSET_FIXED;
+ flags &= ~PIN_NONBLOCK; /* force overlapping PINNED checks */
+ } else if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) {
+ flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+ }
+
+ err = i915_vma_pin(vma, entry->pad_to_size, entry->alignment, flags);
+ if (err)
+ return err;
+
+ if (entry->offset != vma->node.start) {
+ entry->offset = vma->node.start | UPDATE;
+ eb->args->flags |= __EXEC_HAS_RELOC;
+ }
+
+ if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+ err = i915_vma_get_fence(vma);
+ if (unlikely(err)) {
+ i915_vma_unpin(vma);
+ return err;
+ }
+
+ if (i915_vma_pin_fence(vma))
+ entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+ }
+
+ entry->flags |= __EXEC_OBJECT_HAS_PIN;
+ GEM_BUG_ON(eb_vma_misplaced(entry, vma));
+
+ return 0;
+}
+
+static int eb_reserve(struct i915_execbuffer *eb)
+{
+ const unsigned int count = eb->buffer_count;
+ struct list_head last;
+ struct i915_vma *vma;
+ unsigned int i, pass;
+ int err;
+
+ /*
+ * Attempt to pin all of the buffers into the GTT.
+ * This is done in 3 phases:
+ *
+ * 1a. Unbind all objects that do not match the GTT constraints for
+ * the execbuffer (fenceable, mappable, alignment etc).
+ * 1b. Increment pin count for already bound objects.
+ * 2. Bind new objects.
+ * 3. Decrement pin count.
+ *
+ * This avoid unnecessary unbinding of later objects in order to make
+ * room for the earlier objects *unless* we need to defragment.
+ */
+
+ pass = 0;
+ err = 0;
+ do {
+ list_for_each_entry(vma, &eb->unbound, exec_link) {
+ err = eb_reserve_vma(eb, vma);
+ if (err)
+ break;
+ }
+ if (err != -ENOSPC)
+ return err;
+
+ /* Resort *all* the objects into priority order */
+ INIT_LIST_HEAD(&eb->unbound);
+ INIT_LIST_HEAD(&last);
+ for (i = 0; i < count; i++) {
+ struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+
+ if (entry->flags & EXEC_OBJECT_PINNED &&
+ entry->flags & __EXEC_OBJECT_HAS_PIN)
+ continue;
+
+ vma = exec_to_vma(entry);
+ eb_unreserve_vma(vma, entry);
+
+ if (entry->flags & EXEC_OBJECT_PINNED)
+ list_add(&vma->exec_link, &eb->unbound);
+ else if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+ list_add_tail(&vma->exec_link, &eb->unbound);
+ else
+ list_add_tail(&vma->exec_link, &last);
+ }
+ list_splice_tail(&last, &eb->unbound);
+
+ switch (pass++) {
+ case 0:
+ break;
+
+ case 1:
+ /* Too fragmented, unbind everything and retry */
+ err = i915_gem_evict_vm(eb->vm);
+ if (err)
+ return err;
+ break;
+
+ default:
+ return -ENOSPC;
+ }
+ } while (1);
+}
+
+static inline struct hlist_head *
+ht_head(const struct i915_gem_context_vma_lut *lut, u32 handle)
+{
+ return &lut->ht[hash_32(handle, lut->ht_bits)];
+}
+
+static inline bool
+ht_needs_resize(const struct i915_gem_context_vma_lut *lut)
+{
+ return (4*lut->ht_count > 3*lut->ht_size ||
+ 4*lut->ht_count + 1 < lut->ht_size);
+}
+
+static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
+{
+ if (eb->args->flags & I915_EXEC_BATCH_FIRST)
+ return 0;
+ else
+ return eb->buffer_count - 1;
+}
+
+static int eb_select_context(struct i915_execbuffer *eb)
+{
+ struct i915_gem_context *ctx;
+
+ ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
+ if (unlikely(IS_ERR(ctx)))
+ return PTR_ERR(ctx);
+
+ if (unlikely(i915_gem_context_is_banned(ctx))) {
+ DRM_DEBUG("Context %u tried to submit while banned\n",
+ ctx->user_handle);
+ return -EIO;
+ }
+
+ eb->ctx = i915_gem_context_get(ctx);
+ eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
+
+ eb->context_flags = 0;
+ if (ctx->flags & CONTEXT_NO_ZEROMAP)
+ eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
+
+ return 0;
+}
+
+static int eb_lookup_vmas(struct i915_execbuffer *eb)
+{
+#define INTERMEDIATE BIT(0)
+ const unsigned int count = eb->buffer_count;
+ struct i915_gem_context_vma_lut *lut = &eb->ctx->vma_lut;
+ struct i915_vma *vma;
+ struct idr *idr;
+ unsigned int i;
+ int slow_pass = -1;
+ int err;
+
+ INIT_LIST_HEAD(&eb->relocs);
+ INIT_LIST_HEAD(&eb->unbound);
+
+ if (unlikely(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS))
+ flush_work(&lut->resize);
+ GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
+
+ for (i = 0; i < count; i++) {
+ __exec_to_vma(&eb->exec[i]) = 0;
+
+ hlist_for_each_entry(vma,
+ ht_head(lut, eb->exec[i].handle),
+ ctx_node) {
+ if (vma->ctx_handle != eb->exec[i].handle)
+ continue;
+
+ err = eb_add_vma(eb, &eb->exec[i], vma);
+ if (unlikely(err))
+ return err;
+
+ goto next_vma;
+ }
+
+ if (slow_pass < 0)
+ slow_pass = i;
+next_vma: ;
+ }
+
+ if (slow_pass < 0)
+ goto out;
+
+ spin_lock(&eb->file->table_lock);
+ /*
+ * Grab a reference to the object and release the lock so we can lookup
+ * or create the VMA without using GFP_ATOMIC
+ */
+ idr = &eb->file->object_idr;
+ for (i = slow_pass; i < count; i++) {
+ struct drm_i915_gem_object *obj;
+
+ if (__exec_to_vma(&eb->exec[i]))
+ continue;
+
+ obj = to_intel_bo(idr_find(idr, eb->exec[i].handle));
+ if (unlikely(!obj)) {
+ spin_unlock(&eb->file->table_lock);
+ DRM_DEBUG("Invalid object handle %d at index %d\n",
+ eb->exec[i].handle, i);
+ err = -ENOENT;
goto err;
}
- i915_gem_object_get(obj);
- list_add_tail(&obj->obj_exec_link, &objects);
+ __exec_to_vma(&eb->exec[i]) = INTERMEDIATE | (uintptr_t)obj;
}
- spin_unlock(&file->table_lock);
+ spin_unlock(&eb->file->table_lock);
- i = 0;
- while (!list_empty(&objects)) {
- struct i915_vma *vma;
+ for (i = slow_pass; i < count; i++) {
+ struct drm_i915_gem_object *obj;
- obj = list_first_entry(&objects,
- struct drm_i915_gem_object,
- obj_exec_link);
+ if (!(__exec_to_vma(&eb->exec[i]) & INTERMEDIATE))
+ continue;
/*
* NOTE: We can leak any vmas created here when something fails
* from the (obj, vm) we don't run the risk of creating
* duplicated vmas for the same vm.
*/
- vma = i915_vma_instance(obj, vm, NULL);
+ obj = u64_to_ptr(typeof(*obj),
+ __exec_to_vma(&eb->exec[i]) & ~INTERMEDIATE);
+ vma = i915_vma_instance(obj, eb->vm, NULL);
if (unlikely(IS_ERR(vma))) {
DRM_DEBUG("Failed to lookup VMA\n");
- ret = PTR_ERR(vma);
+ err = PTR_ERR(vma);
goto err;
}
- /* Transfer ownership from the objects list to the vmas list. */
- list_add_tail(&vma->exec_list, &eb->vmas);
- list_del_init(&obj->obj_exec_link);
+ /* First come, first served */
+ if (!vma->ctx) {
+ vma->ctx = eb->ctx;
+ vma->ctx_handle = eb->exec[i].handle;
+ hlist_add_head(&vma->ctx_node,
+ ht_head(lut, eb->exec[i].handle));
+ lut->ht_count++;
+ lut->ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
+ if (i915_vma_is_ggtt(vma)) {
+ GEM_BUG_ON(obj->vma_hashed);
+ obj->vma_hashed = vma;
+ }
- vma->exec_entry = &exec[i];
- if (eb->and < 0) {
- eb->lut[i] = vma;
- } else {
- uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
- vma->exec_handle = handle;
- hlist_add_head(&vma->exec_node,
- &eb->buckets[handle & eb->and]);
+ i915_vma_get(vma);
}
- ++i;
- }
- return 0;
+ err = eb_add_vma(eb, &eb->exec[i], vma);
+ if (unlikely(err))
+ goto err;
+ /* Only after we validated the user didn't use our bits */
+ if (vma->ctx != eb->ctx) {
+ i915_vma_get(vma);
+ eb->exec[i].flags |= __EXEC_OBJECT_HAS_REF;
+ }
+ }
-err:
- while (!list_empty(&objects)) {
- obj = list_first_entry(&objects,
- struct drm_i915_gem_object,
- obj_exec_link);
- list_del_init(&obj->obj_exec_link);
- i915_gem_object_put(obj);
+ if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
+ if (ht_needs_resize(lut))
+ queue_work(system_highpri_wq, &lut->resize);
+ else
+ lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
}
+
+out:
+ /* take note of the batch buffer before we might reorder the lists */
+ i = eb_batch_index(eb);
+ eb->batch = exec_to_vma(&eb->exec[i]);
+
/*
- * Objects already transfered to the vmas list will be unreferenced by
- * eb_destroy.
+ * SNA is doing fancy tricks with compressing batch buffers, which leads
+ * to negative relocation deltas. Usually that works out ok since the
+ * relocate address is still positive, except when the batch is placed
+ * very low in the GTT. Ensure this doesn't happen.
+ *
+ * Note that actual hangs have only been observed on gen7, but for
+ * paranoia do it everywhere.
*/
+ if (!(eb->exec[i].flags & EXEC_OBJECT_PINNED))
+ eb->exec[i].flags |= __EXEC_OBJECT_NEEDS_BIAS;
+ if (eb->reloc_cache.has_fence)
+ eb->exec[i].flags |= EXEC_OBJECT_NEEDS_FENCE;
- return ret;
+ eb->args->flags |= __EXEC_VALIDATED;
+ return eb_reserve(eb);
+
+err:
+ for (i = slow_pass; i < count; i++) {
+ if (__exec_to_vma(&eb->exec[i]) & INTERMEDIATE)
+ __exec_to_vma(&eb->exec[i]) = 0;
+ }
+ lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+ return err;
+#undef INTERMEDIATE
}
-static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
+static struct i915_vma *
+eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
{
- if (eb->and < 0) {
- if (handle >= -eb->and)
+ if (eb->lut_size < 0) {
+ if (handle >= -eb->lut_size)
return NULL;
- return eb->lut[handle];
+ return exec_to_vma(&eb->exec[handle]);
} else {
struct hlist_head *head;
struct i915_vma *vma;
- head = &eb->buckets[handle & eb->and];
+ head = &eb->buckets[hash_32(handle, eb->lut_size)];
hlist_for_each_entry(vma, head, exec_node) {
if (vma->exec_handle == handle)
return vma;
}
}
-static void
-i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
+static void eb_release_vmas(const struct i915_execbuffer *eb)
{
- struct drm_i915_gem_exec_object2 *entry;
-
- if (!drm_mm_node_allocated(&vma->node))
- return;
+ const unsigned int count = eb->buffer_count;
+ unsigned int i;
- entry = vma->exec_entry;
+ for (i = 0; i < count; i++) {
+ struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+ struct i915_vma *vma = exec_to_vma(entry);
- if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
- i915_vma_unpin_fence(vma);
+ if (!vma)
+ continue;
- if (entry->flags & __EXEC_OBJECT_HAS_PIN)
- __i915_vma_unpin(vma);
+ GEM_BUG_ON(vma->exec_entry != entry);
+ vma->exec_entry = NULL;
+ __exec_to_vma(entry) = 0;
- entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
-}
+ if (entry->flags & __EXEC_OBJECT_HAS_PIN)
+ __eb_unreserve_vma(vma, entry);
-static void eb_destroy(struct eb_vmas *eb)
-{
- while (!list_empty(&eb->vmas)) {
- struct i915_vma *vma;
+ if (entry->flags & __EXEC_OBJECT_HAS_REF)
+ i915_vma_put(vma);
- vma = list_first_entry(&eb->vmas,
- struct i915_vma,
- exec_list);
- list_del_init(&vma->exec_list);
- i915_gem_execbuffer_unreserve_vma(vma);
- vma->exec_entry = NULL;
- i915_vma_put(vma);
+ entry->flags &=
+ ~(__EXEC_OBJECT_RESERVED | __EXEC_OBJECT_HAS_REF);
}
- kfree(eb);
}
-static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
+static void eb_reset_vmas(const struct i915_execbuffer *eb)
{
- if (!i915_gem_object_has_struct_page(obj))
- return false;
-
- if (DBG_USE_CPU_RELOC)
- return DBG_USE_CPU_RELOC > 0;
-
- return (HAS_LLC(to_i915(obj->base.dev)) ||
- obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
- obj->cache_level != I915_CACHE_NONE);
+ eb_release_vmas(eb);
+ if (eb->lut_size > 0)
+ memset(eb->buckets, 0,
+ sizeof(struct hlist_head) << eb->lut_size);
}
-/* Used to convert any address to canonical form.
- * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
- * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
- * addresses to be in a canonical form:
- * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
- * canonical form [63:48] == [47]."
- */
-#define GEN8_HIGH_ADDRESS_BIT 47
-static inline uint64_t gen8_canonical_addr(uint64_t address)
+static void eb_destroy(const struct i915_execbuffer *eb)
{
- return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
-}
+ GEM_BUG_ON(eb->reloc_cache.rq);
-static inline uint64_t gen8_noncanonical_addr(uint64_t address)
-{
- return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
+ if (eb->lut_size > 0)
+ kfree(eb->buckets);
}
-static inline uint64_t
+static inline u64
relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
- uint64_t target_offset)
+ const struct i915_vma *target)
{
- return gen8_canonical_addr((int)reloc->delta + target_offset);
+ return gen8_canonical_addr((int)reloc->delta + target->node.start);
}
-struct reloc_cache {
- struct drm_i915_private *i915;
- struct drm_mm_node node;
- unsigned long vaddr;
- unsigned int page;
- bool use_64bit_reloc;
-};
-
static void reloc_cache_init(struct reloc_cache *cache,
struct drm_i915_private *i915)
{
cache->page = -1;
cache->vaddr = 0;
- cache->i915 = i915;
/* Must be a variable in the struct to allow GCC to unroll. */
+ cache->gen = INTEL_GEN(i915);
+ cache->has_llc = HAS_LLC(i915);
cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
+ cache->has_fence = cache->gen < 4;
+ cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
cache->node.allocated = false;
+ cache->rq = NULL;
+ cache->rq_size = 0;
}
static inline void *unmask_page(unsigned long p)
#define KMAP 0x4 /* after CLFLUSH_FLAGS */
-static void reloc_cache_fini(struct reloc_cache *cache)
+static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
+{
+ struct drm_i915_private *i915 =
+ container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
+ return &i915->ggtt;
+}
+
+static void reloc_gpu_flush(struct reloc_cache *cache)
+{
+ GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32));
+ cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
+ i915_gem_object_unpin_map(cache->rq->batch->obj);
+ i915_gem_chipset_flush(cache->rq->i915);
+
+ __i915_add_request(cache->rq, true);
+ cache->rq = NULL;
+}
+
+static void reloc_cache_reset(struct reloc_cache *cache)
{
void *vaddr;
+ if (cache->rq)
+ reloc_gpu_flush(cache);
+
if (!cache->vaddr)
return;
wmb();
io_mapping_unmap_atomic((void __iomem *)vaddr);
if (cache->node.allocated) {
- struct i915_ggtt *ggtt = &cache->i915->ggtt;
+ struct i915_ggtt *ggtt = cache_to_ggtt(cache);
ggtt->base.clear_range(&ggtt->base,
cache->node.start,
i915_vma_unpin((struct i915_vma *)cache->node.mm);
}
}
+
+ cache->vaddr = 0;
+ cache->page = -1;
}
static void *reloc_kmap(struct drm_i915_gem_object *obj,
struct reloc_cache *cache,
- int page)
+ unsigned long page)
{
void *vaddr;
kunmap_atomic(unmask_page(cache->vaddr));
} else {
unsigned int flushes;
- int ret;
+ int err;
- ret = i915_gem_obj_prepare_shmem_write(obj, &flushes);
- if (ret)
- return ERR_PTR(ret);
+ err = i915_gem_obj_prepare_shmem_write(obj, &flushes);
+ if (err)
+ return ERR_PTR(err);
BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
static void *reloc_iomap(struct drm_i915_gem_object *obj,
struct reloc_cache *cache,
- int page)
+ unsigned long page)
{
- struct i915_ggtt *ggtt = &cache->i915->ggtt;
+ struct i915_ggtt *ggtt = cache_to_ggtt(cache);
unsigned long offset;
void *vaddr;
io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
} else {
struct i915_vma *vma;
- int ret;
+ int err;
- if (use_cpu_reloc(obj))
+ if (use_cpu_reloc(cache, obj))
return NULL;
- ret = i915_gem_object_set_to_gtt_domain(obj, true);
- if (ret)
- return ERR_PTR(ret);
+ err = i915_gem_object_set_to_gtt_domain(obj, true);
+ if (err)
+ return ERR_PTR(err);
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
PIN_MAPPABLE | PIN_NONBLOCK);
if (IS_ERR(vma)) {
memset(&cache->node, 0, sizeof(cache->node));
- ret = drm_mm_insert_node_in_range
+ err = drm_mm_insert_node_in_range
(&ggtt->base.mm, &cache->node,
PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
0, ggtt->mappable_end,
DRM_MM_INSERT_LOW);
- if (ret) /* no inactive aperture space, use cpu reloc */
+ if (err) /* no inactive aperture space, use cpu reloc */
return NULL;
} else {
- ret = i915_vma_put_fence(vma);
- if (ret) {
+ err = i915_vma_put_fence(vma);
+ if (err) {
i915_vma_unpin(vma);
- return ERR_PTR(ret);
+ return ERR_PTR(err);
}
cache->node.start = vma->node.start;
offset += page << PAGE_SHIFT;
}
- vaddr = (void __force *) io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
+ vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->mappable,
+ offset);
cache->page = page;
cache->vaddr = (unsigned long)vaddr;
static void *reloc_vaddr(struct drm_i915_gem_object *obj,
struct reloc_cache *cache,
- int page)
+ unsigned long page)
{
void *vaddr;
*addr = value;
- /* Writes to the same cacheline are serialised by the CPU
+ /*
+ * Writes to the same cacheline are serialised by the CPU
* (including clflush). On the write path, we only require
* that it hits memory in an orderly fashion and place
* mb barriers at the start and end of the relocation phase
*addr = value;
}
-static int
-relocate_entry(struct drm_i915_gem_object *obj,
+static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
+ struct i915_vma *vma,
+ unsigned int len)
+{
+ struct reloc_cache *cache = &eb->reloc_cache;
+ struct drm_i915_gem_object *obj;
+ struct drm_i915_gem_request *rq;
+ struct i915_vma *batch;
+ u32 *cmd;
+ int err;
+
+ GEM_BUG_ON(vma->obj->base.write_domain & I915_GEM_DOMAIN_CPU);
+
+ obj = i915_gem_batch_pool_get(&eb->engine->batch_pool, PAGE_SIZE);
+ if (IS_ERR(obj))
+ return PTR_ERR(obj);
+
+ cmd = i915_gem_object_pin_map(obj,
+ cache->has_llc ? I915_MAP_WB : I915_MAP_WC);
+ i915_gem_object_unpin_pages(obj);
+ if (IS_ERR(cmd))
+ return PTR_ERR(cmd);
+
+ err = i915_gem_object_set_to_wc_domain(obj, false);
+ if (err)
+ goto err_unmap;
+
+ batch = i915_vma_instance(obj, vma->vm, NULL);
+ if (IS_ERR(batch)) {
+ err = PTR_ERR(batch);
+ goto err_unmap;
+ }
+
+ err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
+ if (err)
+ goto err_unmap;
+
+ rq = i915_gem_request_alloc(eb->engine, eb->ctx);
+ if (IS_ERR(rq)) {
+ err = PTR_ERR(rq);
+ goto err_unpin;
+ }
+
+ err = i915_gem_request_await_object(rq, vma->obj, true);
+ if (err)
+ goto err_request;
+
+ err = eb->engine->emit_flush(rq, EMIT_INVALIDATE);
+ if (err)
+ goto err_request;
+
+ err = i915_switch_context(rq);
+ if (err)
+ goto err_request;
+
+ err = eb->engine->emit_bb_start(rq,
+ batch->node.start, PAGE_SIZE,
+ cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
+ if (err)
+ goto err_request;
+
+ GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true));
+ i915_vma_move_to_active(batch, rq, 0);
+ reservation_object_lock(batch->resv, NULL);
+ reservation_object_add_excl_fence(batch->resv, &rq->fence);
+ reservation_object_unlock(batch->resv);
+ i915_vma_unpin(batch);
+
+ i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ reservation_object_lock(vma->resv, NULL);
+ reservation_object_add_excl_fence(vma->resv, &rq->fence);
+ reservation_object_unlock(vma->resv);
+
+ rq->batch = batch;
+
+ cache->rq = rq;
+ cache->rq_cmd = cmd;
+ cache->rq_size = 0;
+
+ /* Return with batch mapping (cmd) still pinned */
+ return 0;
+
+err_request:
+ i915_add_request(rq);
+err_unpin:
+ i915_vma_unpin(batch);
+err_unmap:
+ i915_gem_object_unpin_map(obj);
+ return err;
+}
+
+static u32 *reloc_gpu(struct i915_execbuffer *eb,
+ struct i915_vma *vma,
+ unsigned int len)
+{
+ struct reloc_cache *cache = &eb->reloc_cache;
+ u32 *cmd;
+
+ if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))
+ reloc_gpu_flush(cache);
+
+ if (unlikely(!cache->rq)) {
+ int err;
+
+ err = __reloc_gpu_alloc(eb, vma, len);
+ if (unlikely(err))
+ return ERR_PTR(err);
+ }
+
+ cmd = cache->rq_cmd + cache->rq_size;
+ cache->rq_size += len;
+
+ return cmd;
+}
+
+static u64
+relocate_entry(struct i915_vma *vma,
const struct drm_i915_gem_relocation_entry *reloc,
- struct reloc_cache *cache,
- u64 target_offset)
+ struct i915_execbuffer *eb,
+ const struct i915_vma *target)
{
u64 offset = reloc->offset;
- bool wide = cache->use_64bit_reloc;
+ u64 target_offset = relocation_target(reloc, target);
+ bool wide = eb->reloc_cache.use_64bit_reloc;
void *vaddr;
- target_offset = relocation_target(reloc, target_offset);
+ if (!eb->reloc_cache.vaddr &&
+ (DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
+ !reservation_object_test_signaled_rcu(vma->resv, true))) {
+ const unsigned int gen = eb->reloc_cache.gen;
+ unsigned int len;
+ u32 *batch;
+ u64 addr;
+
+ if (wide)
+ len = offset & 7 ? 8 : 5;
+ else if (gen >= 4)
+ len = 4;
+ else if (gen >= 3)
+ len = 3;
+ else /* On gen2 MI_STORE_DWORD_IMM uses a physical address */
+ goto repeat;
+
+ batch = reloc_gpu(eb, vma, len);
+ if (IS_ERR(batch))
+ goto repeat;
+
+ addr = gen8_canonical_addr(vma->node.start + offset);
+ if (wide) {
+ if (offset & 7) {
+ *batch++ = MI_STORE_DWORD_IMM_GEN4;
+ *batch++ = lower_32_bits(addr);
+ *batch++ = upper_32_bits(addr);
+ *batch++ = lower_32_bits(target_offset);
+
+ addr = gen8_canonical_addr(addr + 4);
+
+ *batch++ = MI_STORE_DWORD_IMM_GEN4;
+ *batch++ = lower_32_bits(addr);
+ *batch++ = upper_32_bits(addr);
+ *batch++ = upper_32_bits(target_offset);
+ } else {
+ *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1;
+ *batch++ = lower_32_bits(addr);
+ *batch++ = upper_32_bits(addr);
+ *batch++ = lower_32_bits(target_offset);
+ *batch++ = upper_32_bits(target_offset);
+ }
+ } else if (gen >= 6) {
+ *batch++ = MI_STORE_DWORD_IMM_GEN4;
+ *batch++ = 0;
+ *batch++ = addr;
+ *batch++ = target_offset;
+ } else if (gen >= 4) {
+ *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+ *batch++ = 0;
+ *batch++ = addr;
+ *batch++ = target_offset;
+ } else {
+ *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
+ *batch++ = addr;
+ *batch++ = target_offset;
+ }
+
+ goto out;
+ }
+
repeat:
- vaddr = reloc_vaddr(obj, cache, offset >> PAGE_SHIFT);
+ vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT);
if (IS_ERR(vaddr))
return PTR_ERR(vaddr);
clflush_write32(vaddr + offset_in_page(offset),
lower_32_bits(target_offset),
- cache->vaddr);
+ eb->reloc_cache.vaddr);
if (wide) {
offset += sizeof(u32);
goto repeat;
}
- return 0;
+out:
+ return target->node.start | UPDATE;
}
-static int
-i915_gem_execbuffer_relocate_entry(struct i915_vma *vma,
- struct eb_vmas *eb,
- struct drm_i915_gem_relocation_entry *reloc,
- struct reloc_cache *cache)
+static u64
+eb_relocate_entry(struct i915_execbuffer *eb,
+ struct i915_vma *vma,
+ const struct drm_i915_gem_relocation_entry *reloc)
{
- struct drm_i915_gem_object *obj = vma->obj;
- struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
- struct drm_gem_object *target_obj;
- struct drm_i915_gem_object *target_i915_obj;
- struct i915_vma *target_vma;
- uint64_t target_offset;
- int ret;
+ struct i915_vma *target;
+ int err;
/* we've already hold a reference to all valid objects */
- target_vma = eb_get_vma(eb, reloc->target_handle);
- if (unlikely(target_vma == NULL))
+ target = eb_get_vma(eb, reloc->target_handle);
+ if (unlikely(!target))
return -ENOENT;
- target_i915_obj = target_vma->obj;
- target_obj = &target_vma->obj->base;
-
- target_offset = gen8_canonical_addr(target_vma->node.start);
-
- /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
- * pipe_control writes because the gpu doesn't properly redirect them
- * through the ppgtt for non_secure batchbuffers. */
- if (unlikely(IS_GEN6(dev_priv) &&
- reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
- ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
- PIN_GLOBAL);
- if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
- return ret;
- }
/* Validate that the target is in a valid r/w GPU domain */
if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
DRM_DEBUG("reloc with multiple write domains: "
- "obj %p target %d offset %d "
+ "target %d offset %d "
"read %08x write %08x",
- obj, reloc->target_handle,
+ reloc->target_handle,
(int) reloc->offset,
reloc->read_domains,
reloc->write_domain);
if (unlikely((reloc->write_domain | reloc->read_domains)
& ~I915_GEM_GPU_DOMAINS)) {
DRM_DEBUG("reloc with read/write non-GPU domains: "
- "obj %p target %d offset %d "
+ "target %d offset %d "
"read %08x write %08x",
- obj, reloc->target_handle,
+ reloc->target_handle,
(int) reloc->offset,
reloc->read_domains,
reloc->write_domain);
return -EINVAL;
}
- target_obj->pending_read_domains |= reloc->read_domains;
- target_obj->pending_write_domain |= reloc->write_domain;
+ if (reloc->write_domain) {
+ target->exec_entry->flags |= EXEC_OBJECT_WRITE;
+
+ /*
+ * Sandybridge PPGTT errata: We need a global gtt mapping
+ * for MI and pipe_control writes because the gpu doesn't
+ * properly redirect them through the ppgtt for non_secure
+ * batchbuffers.
+ */
+ if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
+ IS_GEN6(eb->i915)) {
+ err = i915_vma_bind(target, target->obj->cache_level,
+ PIN_GLOBAL);
+ if (WARN_ONCE(err,
+ "Unexpected failure to bind target VMA!"))
+ return err;
+ }
+ }
- /* If the relocation already has the right value in it, no
+ /*
+ * If the relocation already has the right value in it, no
* more work needs to be done.
*/
- if (target_offset == reloc->presumed_offset)
+ if (!DBG_FORCE_RELOC &&
+ gen8_canonical_addr(target->node.start) == reloc->presumed_offset)
return 0;
/* Check that the relocation address is valid... */
if (unlikely(reloc->offset >
- obj->base.size - (cache->use_64bit_reloc ? 8 : 4))) {
+ vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
DRM_DEBUG("Relocation beyond object bounds: "
- "obj %p target %d offset %d size %d.\n",
- obj, reloc->target_handle,
- (int) reloc->offset,
- (int) obj->base.size);
+ "target %d offset %d size %d.\n",
+ reloc->target_handle,
+ (int)reloc->offset,
+ (int)vma->size);
return -EINVAL;
}
if (unlikely(reloc->offset & 3)) {
DRM_DEBUG("Relocation not 4-byte aligned: "
- "obj %p target %d offset %d.\n",
- obj, reloc->target_handle,
- (int) reloc->offset);
+ "target %d offset %d.\n",
+ reloc->target_handle,
+ (int)reloc->offset);
return -EINVAL;
}
*/
vma->exec_entry->flags &= ~EXEC_OBJECT_ASYNC;
- ret = relocate_entry(obj, reloc, cache, target_offset);
- if (ret)
- return ret;
-
/* and update the user's relocation entry */
- reloc->presumed_offset = target_offset;
- return 0;
+ return relocate_entry(vma, reloc, eb, target);
}
-static int
-i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
- struct eb_vmas *eb)
+static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
{
#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
- struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
- struct drm_i915_gem_relocation_entry __user *user_relocs;
- struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
- struct reloc_cache cache;
- int remain, ret = 0;
-
- user_relocs = u64_to_user_ptr(entry->relocs_ptr);
- reloc_cache_init(&cache, eb->i915);
+ struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
+ struct drm_i915_gem_relocation_entry __user *urelocs;
+ const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+ unsigned int remain;
+ urelocs = u64_to_user_ptr(entry->relocs_ptr);
remain = entry->relocation_count;
- while (remain) {
- struct drm_i915_gem_relocation_entry *r = stack_reloc;
- unsigned long unwritten;
- unsigned int count;
+ if (unlikely(remain > N_RELOC(ULONG_MAX)))
+ return -EINVAL;
- count = min_t(unsigned int, remain, ARRAY_SIZE(stack_reloc));
- remain -= count;
+ /*
+ * We must check that the entire relocation array is safe
+ * to read. However, if the array is not writable the user loses
+ * the updated relocation values.
+ */
+ if (unlikely(!access_ok(VERIFY_READ, urelocs, remain*sizeof(*urelocs))))
+ return -EFAULT;
+
+ do {
+ struct drm_i915_gem_relocation_entry *r = stack;
+ unsigned int count =
+ min_t(unsigned int, remain, ARRAY_SIZE(stack));
+ unsigned int copied;
- /* This is the fast path and we cannot handle a pagefault
+ /*
+ * This is the fast path and we cannot handle a pagefault
* whilst holding the struct mutex lest the user pass in the
* relocations contained within a mmaped bo. For in such a case
* we, the page fault handler would call i915_gem_fault() and
* this is bad and so lockdep complains vehemently.
*/
pagefault_disable();
- unwritten = __copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]));
+ copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
pagefault_enable();
- if (unlikely(unwritten)) {
- ret = -EFAULT;
+ if (unlikely(copied)) {
+ remain = -EFAULT;
goto out;
}
+ remain -= count;
do {
- u64 offset = r->presumed_offset;
+ u64 offset = eb_relocate_entry(eb, vma, r);
- ret = i915_gem_execbuffer_relocate_entry(vma, eb, r, &cache);
- if (ret)
+ if (likely(offset == 0)) {
+ } else if ((s64)offset < 0) {
+ remain = (int)offset;
goto out;
-
- if (r->presumed_offset != offset) {
- pagefault_disable();
- unwritten = __put_user(r->presumed_offset,
- &user_relocs->presumed_offset);
- pagefault_enable();
- if (unlikely(unwritten)) {
- /* Note that reporting an error now
- * leaves everything in an inconsistent
- * state as we have *already* changed
- * the relocation value inside the
- * object. As we have not changed the
- * reloc.presumed_offset or will not
- * change the execobject.offset, on the
- * call we may not rewrite the value
- * inside the object, leaving it
- * dangling and causing a GPU hang.
- */
- ret = -EFAULT;
- goto out;
- }
+ } else {
+ /*
+ * Note that reporting an error now
+ * leaves everything in an inconsistent
+ * state as we have *already* changed
+ * the relocation value inside the
+ * object. As we have not changed the
+ * reloc.presumed_offset or will not
+ * change the execobject.offset, on the
+ * call we may not rewrite the value
+ * inside the object, leaving it
+ * dangling and causing a GPU hang. Unless
+ * userspace dynamically rebuilds the
+ * relocations on each execbuf rather than
+ * presume a static tree.
+ *
+ * We did previously check if the relocations
+ * were writable (access_ok), an error now
+ * would be a strange race with mprotect,
+ * having already demonstrated that we
+ * can read from this userspace address.
+ */
+ offset = gen8_canonical_addr(offset & ~UPDATE);
+ __put_user(offset,
+ &urelocs[r-stack].presumed_offset);
}
-
- user_relocs++;
- r++;
- } while (--count);
- }
-
+ } while (r++, --count);
+ urelocs += ARRAY_SIZE(stack);
+ } while (remain);
out:
- reloc_cache_fini(&cache);
- return ret;
-#undef N_RELOC
+ reloc_cache_reset(&eb->reloc_cache);
+ return remain;
}
static int
-i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
- struct eb_vmas *eb,
- struct drm_i915_gem_relocation_entry *relocs)
+eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
{
const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
- struct reloc_cache cache;
- int i, ret = 0;
+ struct drm_i915_gem_relocation_entry *relocs =
+ u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
+ unsigned int i;
+ int err;
- reloc_cache_init(&cache, eb->i915);
for (i = 0; i < entry->relocation_count; i++) {
- ret = i915_gem_execbuffer_relocate_entry(vma, eb, &relocs[i], &cache);
- if (ret)
- break;
- }
- reloc_cache_fini(&cache);
-
- return ret;
-}
-
-static int
-i915_gem_execbuffer_relocate(struct eb_vmas *eb)
-{
- struct i915_vma *vma;
- int ret = 0;
+ u64 offset = eb_relocate_entry(eb, vma, &relocs[i]);
- list_for_each_entry(vma, &eb->vmas, exec_list) {
- ret = i915_gem_execbuffer_relocate_vma(vma, eb);
- if (ret)
- break;
+ if ((s64)offset < 0) {
+ err = (int)offset;
+ goto err;
+ }
}
-
- return ret;
-}
-
-static bool only_mappable_for_reloc(unsigned int flags)
-{
- return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
- __EXEC_OBJECT_NEEDS_MAP;
+ err = 0;
+err:
+ reloc_cache_reset(&eb->reloc_cache);
+ return err;
}
-static int
-i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
- struct intel_engine_cs *engine,
- bool *need_reloc)
+static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
{
- struct drm_i915_gem_object *obj = vma->obj;
- struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
- uint64_t flags;
- int ret;
-
- flags = PIN_USER;
- if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
- flags |= PIN_GLOBAL;
-
- if (!drm_mm_node_allocated(&vma->node)) {
- /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
- * limit address to the first 4GBs for unflagged objects.
- */
- if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
- flags |= PIN_ZONE_4G;
- if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
- flags |= PIN_GLOBAL | PIN_MAPPABLE;
- if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
- flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
- if (entry->flags & EXEC_OBJECT_PINNED)
- flags |= entry->offset | PIN_OFFSET_FIXED;
- if ((flags & PIN_MAPPABLE) == 0)
- flags |= PIN_HIGH;
- }
-
- ret = i915_vma_pin(vma,
- entry->pad_to_size,
- entry->alignment,
- flags);
- if ((ret == -ENOSPC || ret == -E2BIG) &&
- only_mappable_for_reloc(entry->flags))
- ret = i915_vma_pin(vma,
- entry->pad_to_size,
- entry->alignment,
- flags & ~PIN_MAPPABLE);
- if (ret)
- return ret;
-
- entry->flags |= __EXEC_OBJECT_HAS_PIN;
+ const char __user *addr, *end;
+ unsigned long size;
+ char __maybe_unused c;
- if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
- ret = i915_vma_get_fence(vma);
- if (ret)
- return ret;
+ size = entry->relocation_count;
+ if (size == 0)
+ return 0;
- if (i915_vma_pin_fence(vma))
- entry->flags |= __EXEC_OBJECT_HAS_FENCE;
- }
+ if (size > N_RELOC(ULONG_MAX))
+ return -EINVAL;
- if (entry->offset != vma->node.start) {
- entry->offset = vma->node.start;
- *need_reloc = true;
- }
+ addr = u64_to_user_ptr(entry->relocs_ptr);
+ size *= sizeof(struct drm_i915_gem_relocation_entry);
+ if (!access_ok(VERIFY_READ, addr, size))
+ return -EFAULT;
- if (entry->flags & EXEC_OBJECT_WRITE) {
- obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
- obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
+ end = addr + size;
+ for (; addr < end; addr += PAGE_SIZE) {
+ int err = __get_user(c, addr);
+ if (err)
+ return err;
}
-
- return 0;
+ return __get_user(c, end - 1);
}
-static bool
-need_reloc_mappable(struct i915_vma *vma)
+static int eb_copy_relocations(const struct i915_execbuffer *eb)
{
- struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-
- if (entry->relocation_count == 0)
- return false;
-
- if (!i915_vma_is_ggtt(vma))
- return false;
-
- /* See also use_cpu_reloc() */
- if (HAS_LLC(to_i915(vma->obj->base.dev)))
- return false;
+ const unsigned int count = eb->buffer_count;
+ unsigned int i;
+ int err;
- if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
- return false;
+ for (i = 0; i < count; i++) {
+ const unsigned int nreloc = eb->exec[i].relocation_count;
+ struct drm_i915_gem_relocation_entry __user *urelocs;
+ struct drm_i915_gem_relocation_entry *relocs;
+ unsigned long size;
+ unsigned long copied;
- return true;
-}
+ if (nreloc == 0)
+ continue;
-static bool
-eb_vma_misplaced(struct i915_vma *vma)
-{
- struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+ err = check_relocations(&eb->exec[i]);
+ if (err)
+ goto err;
- WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
- !i915_vma_is_ggtt(vma));
+ urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
+ size = nreloc * sizeof(*relocs);
- if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
- return true;
+ relocs = kvmalloc_array(size, 1, GFP_TEMPORARY);
+ if (!relocs) {
+ kvfree(relocs);
+ err = -ENOMEM;
+ goto err;
+ }
- if (vma->node.size < entry->pad_to_size)
- return true;
+ /* copy_from_user is limited to < 4GiB */
+ copied = 0;
+ do {
+ unsigned int len =
+ min_t(u64, BIT_ULL(31), size - copied);
+
+ if (__copy_from_user((char *)relocs + copied,
+ (char *)urelocs + copied,
+ len)) {
+ kvfree(relocs);
+ err = -EFAULT;
+ goto err;
+ }
- if (entry->flags & EXEC_OBJECT_PINNED &&
- vma->node.start != entry->offset)
- return true;
+ copied += len;
+ } while (copied < size);
- if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
- vma->node.start < BATCH_OFFSET_BIAS)
- return true;
+ /*
+ * As we do not update the known relocation offsets after
+ * relocating (due to the complexities in lock handling),
+ * we need to mark them as invalid now so that we force the
+ * relocation processing next time. Just in case the target
+ * object is evicted and then rebound into its old
+ * presumed_offset before the next execbuffer - if that
+ * happened we would make the mistake of assuming that the
+ * relocations were valid.
+ */
+ user_access_begin();
+ for (copied = 0; copied < nreloc; copied++)
+ unsafe_put_user(-1,
+ &urelocs[copied].presumed_offset,
+ end_user);
+end_user:
+ user_access_end();
- /* avoid costly ping-pong once a batch bo ended up non-mappable */
- if (entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
- !i915_vma_is_map_and_fenceable(vma))
- return !only_mappable_for_reloc(entry->flags);
+ eb->exec[i].relocs_ptr = (uintptr_t)relocs;
+ }
- if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
- (vma->node.start + vma->node.size - 1) >> 32)
- return true;
+ return 0;
- return false;
+err:
+ while (i--) {
+ struct drm_i915_gem_relocation_entry *relocs =
+ u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
+ if (eb->exec[i].relocation_count)
+ kvfree(relocs);
+ }
+ return err;
}
-static int
-i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
- struct list_head *vmas,
- struct i915_gem_context *ctx,
- bool *need_relocs)
+static int eb_prefault_relocations(const struct i915_execbuffer *eb)
{
- struct drm_i915_gem_object *obj;
- struct i915_vma *vma;
- struct i915_address_space *vm;
- struct list_head ordered_vmas;
- struct list_head pinned_vmas;
- bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
- bool needs_unfenced_map = INTEL_INFO(engine->i915)->unfenced_needs_alignment;
- int retry;
-
- vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
-
- INIT_LIST_HEAD(&ordered_vmas);
- INIT_LIST_HEAD(&pinned_vmas);
- while (!list_empty(vmas)) {
- struct drm_i915_gem_exec_object2 *entry;
- bool need_fence, need_mappable;
-
- vma = list_first_entry(vmas, struct i915_vma, exec_list);
- obj = vma->obj;
- entry = vma->exec_entry;
-
- if (ctx->flags & CONTEXT_NO_ZEROMAP)
- entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
-
- if (!has_fenced_gpu_access)
- entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
- need_fence =
- (entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
- needs_unfenced_map) &&
- i915_gem_object_is_tiled(obj);
- need_mappable = need_fence || need_reloc_mappable(vma);
-
- if (entry->flags & EXEC_OBJECT_PINNED)
- list_move_tail(&vma->exec_list, &pinned_vmas);
- else if (need_mappable) {
- entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
- list_move(&vma->exec_list, &ordered_vmas);
- } else
- list_move_tail(&vma->exec_list, &ordered_vmas);
-
- obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
- obj->base.pending_write_domain = 0;
- }
- list_splice(&ordered_vmas, vmas);
- list_splice(&pinned_vmas, vmas);
-
- /* Attempt to pin all of the buffers into the GTT.
- * This is done in 3 phases:
- *
- * 1a. Unbind all objects that do not match the GTT constraints for
- * the execbuffer (fenceable, mappable, alignment etc).
- * 1b. Increment pin count for already bound objects.
- * 2. Bind new objects.
- * 3. Decrement pin count.
- *
- * This avoid unnecessary unbinding of later objects in order to make
- * room for the earlier objects *unless* we need to defragment.
- */
- retry = 0;
- do {
- int ret = 0;
-
- /* Unbind any ill-fitting objects or pin. */
- list_for_each_entry(vma, vmas, exec_list) {
- if (!drm_mm_node_allocated(&vma->node))
- continue;
+ const unsigned int count = eb->buffer_count;
+ unsigned int i;
- if (eb_vma_misplaced(vma))
- ret = i915_vma_unbind(vma);
- else
- ret = i915_gem_execbuffer_reserve_vma(vma,
- engine,
- need_relocs);
- if (ret)
- goto err;
- }
-
- /* Bind fresh objects */
- list_for_each_entry(vma, vmas, exec_list) {
- if (drm_mm_node_allocated(&vma->node))
- continue;
-
- ret = i915_gem_execbuffer_reserve_vma(vma, engine,
- need_relocs);
- if (ret)
- goto err;
- }
+ if (unlikely(i915.prefault_disable))
+ return 0;
-err:
- if (ret != -ENOSPC || retry++)
- return ret;
+ for (i = 0; i < count; i++) {
+ int err;
- /* Decrement pin count for bound objects */
- list_for_each_entry(vma, vmas, exec_list)
- i915_gem_execbuffer_unreserve_vma(vma);
+ err = check_relocations(&eb->exec[i]);
+ if (err)
+ return err;
+ }
- ret = i915_gem_evict_vm(vm, true);
- if (ret)
- return ret;
- } while (1);
+ return 0;
}
-static int
-i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
- struct drm_i915_gem_execbuffer2 *args,
- struct drm_file *file,
- struct intel_engine_cs *engine,
- struct eb_vmas *eb,
- struct drm_i915_gem_exec_object2 *exec,
- struct i915_gem_context *ctx)
+static noinline int eb_relocate_slow(struct i915_execbuffer *eb)
{
- struct drm_i915_gem_relocation_entry *reloc;
- struct i915_address_space *vm;
+ struct drm_device *dev = &eb->i915->drm;
+ bool have_copy = false;
struct i915_vma *vma;
- bool need_relocs;
- int *reloc_offset;
- int i, total, ret;
- unsigned count = args->buffer_count;
-
- vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
+ int err = 0;
- /* We may process another execbuffer during the unlock... */
- while (!list_empty(&eb->vmas)) {
- vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
- list_del_init(&vma->exec_list);
- i915_gem_execbuffer_unreserve_vma(vma);
- i915_vma_put(vma);
+repeat:
+ if (signal_pending(current)) {
+ err = -ERESTARTSYS;
+ goto out;
}
+ /* We may process another execbuffer during the unlock... */
+ eb_reset_vmas(eb);
mutex_unlock(&dev->struct_mutex);
- total = 0;
- for (i = 0; i < count; i++)
- total += exec[i].relocation_count;
-
- reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
- reloc = drm_malloc_ab(total, sizeof(*reloc));
- if (reloc == NULL || reloc_offset == NULL) {
- drm_free_large(reloc);
- drm_free_large(reloc_offset);
+ /*
+ * We take 3 passes through the slowpatch.
+ *
+ * 1 - we try to just prefault all the user relocation entries and
+ * then attempt to reuse the atomic pagefault disabled fast path again.
+ *
+ * 2 - we copy the user entries to a local buffer here outside of the
+ * local and allow ourselves to wait upon any rendering before
+ * relocations
+ *
+ * 3 - we already have a local copy of the relocation entries, but
+ * were interrupted (EAGAIN) whilst waiting for the objects, try again.
+ */
+ if (!err) {
+ err = eb_prefault_relocations(eb);
+ } else if (!have_copy) {
+ err = eb_copy_relocations(eb);
+ have_copy = err == 0;
+ } else {
+ cond_resched();
+ err = 0;
+ }
+ if (err) {
mutex_lock(&dev->struct_mutex);
- return -ENOMEM;
+ goto out;
}
- total = 0;
- for (i = 0; i < count; i++) {
- struct drm_i915_gem_relocation_entry __user *user_relocs;
- u64 invalid_offset = (u64)-1;
- int j;
-
- user_relocs = u64_to_user_ptr(exec[i].relocs_ptr);
-
- if (copy_from_user(reloc+total, user_relocs,
- exec[i].relocation_count * sizeof(*reloc))) {
- ret = -EFAULT;
- mutex_lock(&dev->struct_mutex);
- goto err;
- }
-
- /* As we do not update the known relocation offsets after
- * relocating (due to the complexities in lock handling),
- * we need to mark them as invalid now so that we force the
- * relocation processing next time. Just in case the target
- * object is evicted and then rebound into its old
- * presumed_offset before the next execbuffer - if that
- * happened we would make the mistake of assuming that the
- * relocations were valid.
- */
- for (j = 0; j < exec[i].relocation_count; j++) {
- if (__copy_to_user(&user_relocs[j].presumed_offset,
- &invalid_offset,
- sizeof(invalid_offset))) {
- ret = -EFAULT;
- mutex_lock(&dev->struct_mutex);
- goto err;
- }
- }
-
- reloc_offset[i] = total;
- total += exec[i].relocation_count;
- }
+ /* A frequent cause for EAGAIN are currently unavailable client pages */
+ flush_workqueue(eb->i915->mm.userptr_wq);
- ret = i915_mutex_lock_interruptible(dev);
- if (ret) {
+ err = i915_mutex_lock_interruptible(dev);
+ if (err) {
mutex_lock(&dev->struct_mutex);
- goto err;
+ goto out;
}
/* reacquire the objects */
- eb_reset(eb);
- ret = eb_lookup_vmas(eb, exec, args, vm, file);
- if (ret)
- goto err;
-
- need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
- ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
- &need_relocs);
- if (ret)
+ err = eb_lookup_vmas(eb);
+ if (err)
goto err;
- list_for_each_entry(vma, &eb->vmas, exec_list) {
- int offset = vma->exec_entry - exec;
- ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
- reloc + reloc_offset[offset]);
- if (ret)
- goto err;
+ list_for_each_entry(vma, &eb->relocs, reloc_link) {
+ if (!have_copy) {
+ pagefault_disable();
+ err = eb_relocate_vma(eb, vma);
+ pagefault_enable();
+ if (err)
+ goto repeat;
+ } else {
+ err = eb_relocate_vma_slow(eb, vma);
+ if (err)
+ goto err;
+ }
}
- /* Leave the user relocations as are, this is the painfully slow path,
+ /*
+ * Leave the user relocations as are, this is the painfully slow path,
* and we want to avoid the complication of dropping the lock whilst
* having buffers reserved in the aperture and so causing spurious
* ENOSPC for random operations.
*/
err:
- drm_free_large(reloc);
- drm_free_large(reloc_offset);
- return ret;
-}
+ if (err == -EAGAIN)
+ goto repeat;
-static int
-i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
- struct list_head *vmas)
-{
- struct i915_vma *vma;
- int ret;
+out:
+ if (have_copy) {
+ const unsigned int count = eb->buffer_count;
+ unsigned int i;
- list_for_each_entry(vma, vmas, exec_list) {
- struct drm_i915_gem_object *obj = vma->obj;
+ for (i = 0; i < count; i++) {
+ const struct drm_i915_gem_exec_object2 *entry =
+ &eb->exec[i];
+ struct drm_i915_gem_relocation_entry *relocs;
- if (vma->exec_entry->flags & EXEC_OBJECT_ASYNC)
- continue;
+ if (!entry->relocation_count)
+ continue;
- if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) {
- i915_gem_clflush_object(obj, 0);
- obj->base.write_domain = 0;
+ relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
+ kvfree(relocs);
}
-
- ret = i915_gem_request_await_object
- (req, obj, obj->base.pending_write_domain);
- if (ret)
- return ret;
}
- /* Unconditionally flush any chipset caches (for streaming writes). */
- i915_gem_chipset_flush(req->engine->i915);
-
- /* Unconditionally invalidate GPU caches and TLBs. */
- return req->engine->emit_flush(req, EMIT_INVALIDATE);
+ return err;
}
-static bool
-i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
+static int eb_relocate(struct i915_execbuffer *eb)
{
- if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
- return false;
+ if (eb_lookup_vmas(eb))
+ goto slow;
- /* Kernel clipping was a DRI1 misfeature */
- if (exec->num_cliprects || exec->cliprects_ptr)
- return false;
+ /* The objects are in their final locations, apply the relocations. */
+ if (eb->args->flags & __EXEC_HAS_RELOC) {
+ struct i915_vma *vma;
- if (exec->DR4 == 0xffffffff) {
- DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
- exec->DR4 = 0;
+ list_for_each_entry(vma, &eb->relocs, reloc_link) {
+ if (eb_relocate_vma(eb, vma))
+ goto slow;
+ }
}
- if (exec->DR1 || exec->DR4)
- return false;
- if ((exec->batch_start_offset | exec->batch_len) & 0x7)
- return false;
+ return 0;
- return true;
+slow:
+ return eb_relocate_slow(eb);
}
-static int
-validate_exec_list(struct drm_device *dev,
- struct drm_i915_gem_exec_object2 *exec,
- int count)
+static void eb_export_fence(struct i915_vma *vma,
+ struct drm_i915_gem_request *req,
+ unsigned int flags)
{
- unsigned relocs_total = 0;
- unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
- unsigned invalid_flags;
- int i;
+ struct reservation_object *resv = vma->resv;
- /* INTERNAL flags must not overlap with external ones */
- BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
+ /*
+ * Ignore errors from failing to allocate the new fence, we can't
+ * handle an error right now. Worst case should be missed
+ * synchronisation leading to rendering corruption.
+ */
+ reservation_object_lock(resv, NULL);
+ if (flags & EXEC_OBJECT_WRITE)
+ reservation_object_add_excl_fence(resv, &req->fence);
+ else if (reservation_object_reserve_shared(resv) == 0)
+ reservation_object_add_shared_fence(resv, &req->fence);
+ reservation_object_unlock(resv);
+}
- invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
- if (USES_FULL_PPGTT(dev))
- invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
+static int eb_move_to_gpu(struct i915_execbuffer *eb)
+{
+ const unsigned int count = eb->buffer_count;
+ unsigned int i;
+ int err;
for (i = 0; i < count; i++) {
- char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
- int length; /* limited by fault_in_pages_readable() */
+ const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+ struct i915_vma *vma = exec_to_vma(entry);
+ struct drm_i915_gem_object *obj = vma->obj;
- if (exec[i].flags & invalid_flags)
- return -EINVAL;
+ if (entry->flags & EXEC_OBJECT_CAPTURE) {
+ struct i915_gem_capture_list *capture;
- /* Offset can be used as input (EXEC_OBJECT_PINNED), reject
- * any non-page-aligned or non-canonical addresses.
- */
- if (exec[i].flags & EXEC_OBJECT_PINNED) {
- if (exec[i].offset !=
- gen8_canonical_addr(exec[i].offset & PAGE_MASK))
- return -EINVAL;
+ capture = kmalloc(sizeof(*capture), GFP_KERNEL);
+ if (unlikely(!capture))
+ return -ENOMEM;
+
+ capture->next = eb->request->capture_list;
+ capture->vma = vma;
+ eb->request->capture_list = capture;
}
- /* From drm_mm perspective address space is continuous,
- * so from this point we're always using non-canonical
- * form internally.
- */
- exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
+ if (entry->flags & EXEC_OBJECT_ASYNC)
+ goto skip_flushes;
- if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
- return -EINVAL;
+ if (unlikely(obj->cache_dirty && !obj->cache_coherent))
+ i915_gem_clflush_object(obj, 0);
- /* pad_to_size was once a reserved field, so sanitize it */
- if (exec[i].flags & EXEC_OBJECT_PAD_TO_SIZE) {
- if (offset_in_page(exec[i].pad_to_size))
- return -EINVAL;
- } else {
- exec[i].pad_to_size = 0;
- }
+ err = i915_gem_request_await_object
+ (eb->request, obj, entry->flags & EXEC_OBJECT_WRITE);
+ if (err)
+ return err;
- /* First check for malicious input causing overflow in
- * the worst case where we need to allocate the entire
- * relocation tree as a single array.
- */
- if (exec[i].relocation_count > relocs_max - relocs_total)
- return -EINVAL;
- relocs_total += exec[i].relocation_count;
+skip_flushes:
+ i915_vma_move_to_active(vma, eb->request, entry->flags);
+ __eb_unreserve_vma(vma, entry);
+ vma->exec_entry = NULL;
+ }
- length = exec[i].relocation_count *
- sizeof(struct drm_i915_gem_relocation_entry);
- /*
- * We must check that the entire relocation array is safe
- * to read, but since we may need to update the presumed
- * offsets during execution, check for full write access.
- */
- if (!access_ok(VERIFY_WRITE, ptr, length))
- return -EFAULT;
+ for (i = 0; i < count; i++) {
+ const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+ struct i915_vma *vma = exec_to_vma(entry);
- if (likely(!i915.prefault_disable)) {
- if (fault_in_pages_readable(ptr, length))
- return -EFAULT;
- }
+ eb_export_fence(vma, eb->request, entry->flags);
+ if (unlikely(entry->flags & __EXEC_OBJECT_HAS_REF))
+ i915_vma_put(vma);
}
+ eb->exec = NULL;
- return 0;
+ /* Unconditionally flush any chipset caches (for streaming writes). */
+ i915_gem_chipset_flush(eb->i915);
+
+ /* Unconditionally invalidate GPU caches and TLBs. */
+ return eb->engine->emit_flush(eb->request, EMIT_INVALIDATE);
}
-static struct i915_gem_context *
-i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
- struct intel_engine_cs *engine, const u32 ctx_id)
+static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
{
- struct i915_gem_context *ctx;
+ if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
+ return false;
- ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
- if (IS_ERR(ctx))
- return ctx;
+ /* Kernel clipping was a DRI1 misfeature */
+ if (exec->num_cliprects || exec->cliprects_ptr)
+ return false;
- if (i915_gem_context_is_banned(ctx)) {
- DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
- return ERR_PTR(-EIO);
+ if (exec->DR4 == 0xffffffff) {
+ DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
+ exec->DR4 = 0;
}
+ if (exec->DR1 || exec->DR4)
+ return false;
- return ctx;
-}
+ if ((exec->batch_start_offset | exec->batch_len) & 0x7)
+ return false;
-static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
-{
- return !(obj->cache_level == I915_CACHE_NONE ||
- obj->cache_level == I915_CACHE_WT);
+ return true;
}
void i915_vma_move_to_active(struct i915_vma *vma,
lockdep_assert_held(&req->i915->drm.struct_mutex);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
- /* Add a reference if we're newly entering the active list.
+ /*
+ * Add a reference if we're newly entering the active list.
* The order in which we add operations to the retirement queue is
* vital here: mark_active adds to the start of the callback list,
* such that subsequent callbacks are called first. Therefore we
i915_gem_active_set(&vma->last_read[idx], req);
list_move_tail(&vma->vm_link, &vma->vm->active_list);
+ obj->base.write_domain = 0;
if (flags & EXEC_OBJECT_WRITE) {
+ obj->base.write_domain = I915_GEM_DOMAIN_RENDER;
+
if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
i915_gem_active_set(&obj->frontbuffer_write, req);
- /* update for the implicit flush after a batch */
- obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
- if (!obj->cache_dirty && gpu_write_needs_clflush(obj))
- obj->cache_dirty = true;
+ obj->base.read_domains = 0;
}
+ obj->base.read_domains |= I915_GEM_GPU_DOMAINS;
if (flags & EXEC_OBJECT_NEEDS_FENCE)
i915_gem_active_set(&vma->last_fence, req);
}
-static void eb_export_fence(struct drm_i915_gem_object *obj,
- struct drm_i915_gem_request *req,
- unsigned int flags)
-{
- struct reservation_object *resv = obj->resv;
-
- /* Ignore errors from failing to allocate the new fence, we can't
- * handle an error right now. Worst case should be missed
- * synchronisation leading to rendering corruption.
- */
- reservation_object_lock(resv, NULL);
- if (flags & EXEC_OBJECT_WRITE)
- reservation_object_add_excl_fence(resv, &req->fence);
- else if (reservation_object_reserve_shared(resv) == 0)
- reservation_object_add_shared_fence(resv, &req->fence);
- reservation_object_unlock(resv);
-}
-
-static void
-i915_gem_execbuffer_move_to_active(struct list_head *vmas,
- struct drm_i915_gem_request *req)
-{
- struct i915_vma *vma;
-
- list_for_each_entry(vma, vmas, exec_list) {
- struct drm_i915_gem_object *obj = vma->obj;
-
- obj->base.write_domain = obj->base.pending_write_domain;
- if (obj->base.write_domain)
- vma->exec_entry->flags |= EXEC_OBJECT_WRITE;
- else
- obj->base.pending_read_domains |= obj->base.read_domains;
- obj->base.read_domains = obj->base.pending_read_domains;
-
- i915_vma_move_to_active(vma, req, vma->exec_entry->flags);
- eb_export_fence(obj, req, vma->exec_entry->flags);
- }
-}
-
-static int
-i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
+static int i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
{
u32 *cs;
int i;
return -EINVAL;
}
- cs = intel_ring_begin(req, 4 * 3);
+ cs = intel_ring_begin(req, 4 * 2 + 2);
if (IS_ERR(cs))
return PTR_ERR(cs);
+ *cs++ = MI_LOAD_REGISTER_IMM(4);
for (i = 0; i < 4; i++) {
- *cs++ = MI_LOAD_REGISTER_IMM(1);
*cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
*cs++ = 0;
}
-
+ *cs++ = MI_NOOP;
intel_ring_advance(req, cs);
return 0;
}
-static struct i915_vma *
-i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
- struct drm_i915_gem_exec_object2 *shadow_exec_entry,
- struct drm_i915_gem_object *batch_obj,
- struct eb_vmas *eb,
- u32 batch_start_offset,
- u32 batch_len,
- bool is_master)
+static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
{
struct drm_i915_gem_object *shadow_batch_obj;
struct i915_vma *vma;
- int ret;
+ int err;
- shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
- PAGE_ALIGN(batch_len));
+ shadow_batch_obj = i915_gem_batch_pool_get(&eb->engine->batch_pool,
+ PAGE_ALIGN(eb->batch_len));
if (IS_ERR(shadow_batch_obj))
return ERR_CAST(shadow_batch_obj);
- ret = intel_engine_cmd_parser(engine,
- batch_obj,
+ err = intel_engine_cmd_parser(eb->engine,
+ eb->batch->obj,
shadow_batch_obj,
- batch_start_offset,
- batch_len,
+ eb->batch_start_offset,
+ eb->batch_len,
is_master);
- if (ret) {
- if (ret == -EACCES) /* unhandled chained batch */
+ if (err) {
+ if (err == -EACCES) /* unhandled chained batch */
vma = NULL;
else
- vma = ERR_PTR(ret);
+ vma = ERR_PTR(err);
goto out;
}
if (IS_ERR(vma))
goto out;
- memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
-
- vma->exec_entry = shadow_exec_entry;
- vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
- i915_gem_object_get(shadow_batch_obj);
- list_add_tail(&vma->exec_list, &eb->vmas);
+ vma->exec_entry =
+ memset(&eb->exec[eb->buffer_count++],
+ 0, sizeof(*vma->exec_entry));
+ vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
+ __exec_to_vma(vma->exec_entry) = (uintptr_t)i915_vma_get(vma);
out:
i915_gem_object_unpin_pages(shadow_batch_obj);
}
static void
-add_to_client(struct drm_i915_gem_request *req,
- struct drm_file *file)
+add_to_client(struct drm_i915_gem_request *req, struct drm_file *file)
{
req->file_priv = file->driver_priv;
list_add_tail(&req->client_link, &req->file_priv->mm.request_list);
}
-static int
-execbuf_submit(struct i915_execbuffer_params *params,
- struct drm_i915_gem_execbuffer2 *args,
- struct list_head *vmas)
+static int eb_submit(struct i915_execbuffer *eb)
{
- u64 exec_start, exec_len;
- int ret;
+ int err;
- ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
- if (ret)
- return ret;
+ err = eb_move_to_gpu(eb);
+ if (err)
+ return err;
- ret = i915_switch_context(params->request);
- if (ret)
- return ret;
-
- if (args->flags & I915_EXEC_CONSTANTS_MASK) {
- DRM_DEBUG("I915_EXEC_CONSTANTS_* unsupported\n");
- return -EINVAL;
- }
+ err = i915_switch_context(eb->request);
+ if (err)
+ return err;
- if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
- ret = i915_reset_gen7_sol_offsets(params->request);
- if (ret)
- return ret;
+ if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
+ err = i915_reset_gen7_sol_offsets(eb->request);
+ if (err)
+ return err;
}
- exec_len = args->batch_len;
- exec_start = params->batch->node.start +
- params->args_batch_start_offset;
-
- if (exec_len == 0)
- exec_len = params->batch->size - params->args_batch_start_offset;
-
- ret = params->engine->emit_bb_start(params->request,
- exec_start, exec_len,
- params->dispatch_flags);
- if (ret)
- return ret;
-
- i915_gem_execbuffer_move_to_active(vmas, params->request);
+ err = eb->engine->emit_bb_start(eb->request,
+ eb->batch->node.start +
+ eb->batch_start_offset,
+ eb->batch_len,
+ eb->batch_flags);
+ if (err)
+ return err;
return 0;
}
}
static int
-i915_gem_do_execbuffer(struct drm_device *dev, void *data,
+i915_gem_do_execbuffer(struct drm_device *dev,
struct drm_file *file,
struct drm_i915_gem_execbuffer2 *args,
struct drm_i915_gem_exec_object2 *exec)
{
- struct drm_i915_private *dev_priv = to_i915(dev);
- struct i915_ggtt *ggtt = &dev_priv->ggtt;
- struct eb_vmas *eb;
- struct drm_i915_gem_exec_object2 shadow_exec_entry;
- struct intel_engine_cs *engine;
- struct i915_gem_context *ctx;
- struct i915_address_space *vm;
- struct i915_execbuffer_params params_master; /* XXX: will be removed later */
- struct i915_execbuffer_params *params = ¶ms_master;
- const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
- u32 dispatch_flags;
+ struct i915_execbuffer eb;
struct dma_fence *in_fence = NULL;
struct sync_file *out_fence = NULL;
int out_fence_fd = -1;
- int ret;
- bool need_relocs;
-
- if (!i915_gem_check_execbuffer(args))
- return -EINVAL;
-
- ret = validate_exec_list(dev, exec, args->buffer_count);
- if (ret)
- return ret;
-
- dispatch_flags = 0;
+ int err;
+
+ BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
+ ~__EXEC_OBJECT_UNKNOWN_FLAGS);
+
+ eb.i915 = to_i915(dev);
+ eb.file = file;
+ eb.args = args;
+ if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
+ args->flags |= __EXEC_HAS_RELOC;
+ eb.exec = exec;
+ eb.ctx = NULL;
+ eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
+ if (USES_FULL_PPGTT(eb.i915))
+ eb.invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
+ reloc_cache_init(&eb.reloc_cache, eb.i915);
+
+ eb.buffer_count = args->buffer_count;
+ eb.batch_start_offset = args->batch_start_offset;
+ eb.batch_len = args->batch_len;
+
+ eb.batch_flags = 0;
if (args->flags & I915_EXEC_SECURE) {
if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
return -EPERM;
- dispatch_flags |= I915_DISPATCH_SECURE;
+ eb.batch_flags |= I915_DISPATCH_SECURE;
}
if (args->flags & I915_EXEC_IS_PINNED)
- dispatch_flags |= I915_DISPATCH_PINNED;
-
- engine = eb_select_engine(dev_priv, file, args);
- if (!engine)
- return -EINVAL;
+ eb.batch_flags |= I915_DISPATCH_PINNED;
- if (args->buffer_count < 1) {
- DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
+ eb.engine = eb_select_engine(eb.i915, file, args);
+ if (!eb.engine)
return -EINVAL;
- }
if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
- if (!HAS_RESOURCE_STREAMER(dev_priv)) {
+ if (!HAS_RESOURCE_STREAMER(eb.i915)) {
DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
return -EINVAL;
}
- if (engine->id != RCS) {
+ if (eb.engine->id != RCS) {
DRM_DEBUG("RS is not available on %s\n",
- engine->name);
+ eb.engine->name);
return -EINVAL;
}
- dispatch_flags |= I915_DISPATCH_RS;
+ eb.batch_flags |= I915_DISPATCH_RS;
}
if (args->flags & I915_EXEC_FENCE_IN) {
if (args->flags & I915_EXEC_FENCE_OUT) {
out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
if (out_fence_fd < 0) {
- ret = out_fence_fd;
+ err = out_fence_fd;
goto err_in_fence;
}
}
- /* Take a local wakeref for preparing to dispatch the execbuf as
+ err = eb_create(&eb);
+ if (err)
+ goto err_out_fence;
+
+ GEM_BUG_ON(!eb.lut_size);
+
+ /*
+ * Take a local wakeref for preparing to dispatch the execbuf as
* we expect to access the hardware fairly frequently in the
* process. Upon first dispatch, we acquire another prolonged
* wakeref that we hold until the GPU has been idle for at least
* 100ms.
*/
- intel_runtime_pm_get(dev_priv);
-
- ret = i915_mutex_lock_interruptible(dev);
- if (ret)
- goto pre_mutex_err;
-
- ctx = i915_gem_validate_context(dev, file, engine, ctx_id);
- if (IS_ERR(ctx)) {
- mutex_unlock(&dev->struct_mutex);
- ret = PTR_ERR(ctx);
- goto pre_mutex_err;
- }
-
- i915_gem_context_get(ctx);
-
- if (ctx->ppgtt)
- vm = &ctx->ppgtt->base;
- else
- vm = &ggtt->base;
-
- memset(¶ms_master, 0x00, sizeof(params_master));
+ intel_runtime_pm_get(eb.i915);
+ err = i915_mutex_lock_interruptible(dev);
+ if (err)
+ goto err_rpm;
- eb = eb_create(dev_priv, args);
- if (eb == NULL) {
- i915_gem_context_put(ctx);
- mutex_unlock(&dev->struct_mutex);
- ret = -ENOMEM;
- goto pre_mutex_err;
- }
-
- /* Look up object handles */
- ret = eb_lookup_vmas(eb, exec, args, vm, file);
- if (ret)
- goto err;
-
- /* take note of the batch buffer before we might reorder the lists */
- params->batch = eb_get_batch(eb);
-
- /* Move the objects en-masse into the GTT, evicting if necessary. */
- need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
- ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
- &need_relocs);
- if (ret)
- goto err;
+ err = eb_select_context(&eb);
+ if (unlikely(err))
+ goto err_unlock;
- /* The objects are in their final locations, apply the relocations. */
- if (need_relocs)
- ret = i915_gem_execbuffer_relocate(eb);
- if (ret) {
- if (ret == -EFAULT) {
- ret = i915_gem_execbuffer_relocate_slow(dev, args, file,
- engine,
- eb, exec, ctx);
- BUG_ON(!mutex_is_locked(&dev->struct_mutex));
- }
- if (ret)
- goto err;
+ err = eb_relocate(&eb);
+ if (err) {
+ /*
+ * If the user expects the execobject.offset and
+ * reloc.presumed_offset to be an exact match,
+ * as for using NO_RELOC, then we cannot update
+ * the execobject.offset until we have completed
+ * relocation.
+ */
+ args->flags &= ~__EXEC_HAS_RELOC;
+ goto err_vma;
}
- /* Set the pending read domains for the batch buffer to COMMAND */
- if (params->batch->obj->base.pending_write_domain) {
+ if (unlikely(eb.batch->exec_entry->flags & EXEC_OBJECT_WRITE)) {
DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
- ret = -EINVAL;
- goto err;
+ err = -EINVAL;
+ goto err_vma;
}
- if (args->batch_start_offset > params->batch->size ||
- args->batch_len > params->batch->size - args->batch_start_offset) {
+ if (eb.batch_start_offset > eb.batch->size ||
+ eb.batch_len > eb.batch->size - eb.batch_start_offset) {
DRM_DEBUG("Attempting to use out-of-bounds batch\n");
- ret = -EINVAL;
- goto err;
+ err = -EINVAL;
+ goto err_vma;
}
- params->args_batch_start_offset = args->batch_start_offset;
- if (engine->needs_cmd_parser && args->batch_len) {
+ if (eb.engine->needs_cmd_parser && eb.batch_len) {
struct i915_vma *vma;
- vma = i915_gem_execbuffer_parse(engine, &shadow_exec_entry,
- params->batch->obj,
- eb,
- args->batch_start_offset,
- args->batch_len,
- drm_is_current_master(file));
+ vma = eb_parse(&eb, drm_is_current_master(file));
if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err;
+ err = PTR_ERR(vma);
+ goto err_vma;
}
if (vma) {
* specifically don't want that set on batches the
* command parser has accepted.
*/
- dispatch_flags |= I915_DISPATCH_SECURE;
- params->args_batch_start_offset = 0;
- params->batch = vma;
+ eb.batch_flags |= I915_DISPATCH_SECURE;
+ eb.batch_start_offset = 0;
+ eb.batch = vma;
}
}
- params->batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
+ if (eb.batch_len == 0)
+ eb.batch_len = eb.batch->size - eb.batch_start_offset;
- /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
+ /*
+ * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
* batch" bit. Hence we need to pin secure batches into the global gtt.
* hsw should have this fixed, but bdw mucks it up again. */
- if (dispatch_flags & I915_DISPATCH_SECURE) {
- struct drm_i915_gem_object *obj = params->batch->obj;
+ if (eb.batch_flags & I915_DISPATCH_SECURE) {
struct i915_vma *vma;
/*
* fitting due to fragmentation.
* So this is actually safe.
*/
- vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+ vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0);
if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto err;
+ err = PTR_ERR(vma);
+ goto err_vma;
}
- params->batch = vma;
+ eb.batch = vma;
}
+ /* All GPU relocation batches must be submitted prior to the user rq */
+ GEM_BUG_ON(eb.reloc_cache.rq);
+
/* Allocate a request for this batch buffer nice and early. */
- params->request = i915_gem_request_alloc(engine, ctx);
- if (IS_ERR(params->request)) {
- ret = PTR_ERR(params->request);
+ eb.request = i915_gem_request_alloc(eb.engine, eb.ctx);
+ if (IS_ERR(eb.request)) {
+ err = PTR_ERR(eb.request);
goto err_batch_unpin;
}
if (in_fence) {
- ret = i915_gem_request_await_dma_fence(params->request,
- in_fence);
- if (ret < 0)
+ err = i915_gem_request_await_dma_fence(eb.request, in_fence);
+ if (err < 0)
goto err_request;
}
if (out_fence_fd != -1) {
- out_fence = sync_file_create(¶ms->request->fence);
+ out_fence = sync_file_create(&eb.request->fence);
if (!out_fence) {
- ret = -ENOMEM;
+ err = -ENOMEM;
goto err_request;
}
}
- /* Whilst this request exists, batch_obj will be on the
+ /*
+ * Whilst this request exists, batch_obj will be on the
* active_list, and so will hold the active reference. Only when this
* request is retired will the the batch_obj be moved onto the
* inactive_list and lose its active reference. Hence we do not need
* to explicitly hold another reference here.
*/
- params->request->batch = params->batch;
-
- /*
- * Save assorted stuff away to pass through to *_submission().
- * NB: This data should be 'persistent' and not local as it will
- * kept around beyond the duration of the IOCTL once the GPU
- * scheduler arrives.
- */
- params->dev = dev;
- params->file = file;
- params->engine = engine;
- params->dispatch_flags = dispatch_flags;
- params->ctx = ctx;
+ eb.request->batch = eb.batch;
- trace_i915_gem_request_queue(params->request, dispatch_flags);
-
- ret = execbuf_submit(params, args, &eb->vmas);
+ trace_i915_gem_request_queue(eb.request, eb.batch_flags);
+ err = eb_submit(&eb);
err_request:
- __i915_add_request(params->request, ret == 0);
- add_to_client(params->request, file);
+ __i915_add_request(eb.request, err == 0);
+ add_to_client(eb.request, file);
if (out_fence) {
- if (ret == 0) {
+ if (err == 0) {
fd_install(out_fence_fd, out_fence->file);
args->rsvd2 &= GENMASK_ULL(0, 31); /* keep in-fence */
args->rsvd2 |= (u64)out_fence_fd << 32;
}
err_batch_unpin:
- /*
- * FIXME: We crucially rely upon the active tracking for the (ppgtt)
- * batch vma for correctness. For less ugly and less fragility this
- * needs to be adjusted to also track the ggtt batch vma properly as
- * active.
- */
- if (dispatch_flags & I915_DISPATCH_SECURE)
- i915_vma_unpin(params->batch);
-err:
- /* the request owns the ref now */
- i915_gem_context_put(ctx);
- eb_destroy(eb);
-
+ if (eb.batch_flags & I915_DISPATCH_SECURE)
+ i915_vma_unpin(eb.batch);
+err_vma:
+ if (eb.exec)
+ eb_release_vmas(&eb);
+ i915_gem_context_put(eb.ctx);
+err_unlock:
mutex_unlock(&dev->struct_mutex);
-
-pre_mutex_err:
- /* intel_gpu_busy should also get a ref, so it will free when the device
- * is really idle. */
- intel_runtime_pm_put(dev_priv);
+err_rpm:
+ intel_runtime_pm_put(eb.i915);
+ eb_destroy(&eb);
+err_out_fence:
if (out_fence_fd != -1)
put_unused_fd(out_fence_fd);
err_in_fence:
dma_fence_put(in_fence);
- return ret;
+ return err;
}
/*
i915_gem_execbuffer(struct drm_device *dev, void *data,
struct drm_file *file)
{
+ const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
struct drm_i915_gem_execbuffer *args = data;
struct drm_i915_gem_execbuffer2 exec2;
struct drm_i915_gem_exec_object *exec_list = NULL;
struct drm_i915_gem_exec_object2 *exec2_list = NULL;
- int ret, i;
+ unsigned int i;
+ int err;
- if (args->buffer_count < 1) {
- DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
+ if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
+ DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
return -EINVAL;
}
+ exec2.buffers_ptr = args->buffers_ptr;
+ exec2.buffer_count = args->buffer_count;
+ exec2.batch_start_offset = args->batch_start_offset;
+ exec2.batch_len = args->batch_len;
+ exec2.DR1 = args->DR1;
+ exec2.DR4 = args->DR4;
+ exec2.num_cliprects = args->num_cliprects;
+ exec2.cliprects_ptr = args->cliprects_ptr;
+ exec2.flags = I915_EXEC_RENDER;
+ i915_execbuffer2_set_context_id(exec2, 0);
+
+ if (!i915_gem_check_execbuffer(&exec2))
+ return -EINVAL;
+
/* Copy in the exec list from userland */
- exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
- exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
+ exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
+ __GFP_NOWARN | GFP_TEMPORARY);
+ exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+ __GFP_NOWARN | GFP_TEMPORARY);
if (exec_list == NULL || exec2_list == NULL) {
DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
args->buffer_count);
- drm_free_large(exec_list);
- drm_free_large(exec2_list);
+ kvfree(exec_list);
+ kvfree(exec2_list);
return -ENOMEM;
}
- ret = copy_from_user(exec_list,
+ err = copy_from_user(exec_list,
u64_to_user_ptr(args->buffers_ptr),
sizeof(*exec_list) * args->buffer_count);
- if (ret != 0) {
+ if (err) {
DRM_DEBUG("copy %d exec entries failed %d\n",
- args->buffer_count, ret);
- drm_free_large(exec_list);
- drm_free_large(exec2_list);
+ args->buffer_count, err);
+ kvfree(exec_list);
+ kvfree(exec2_list);
return -EFAULT;
}
exec2_list[i].flags = 0;
}
- exec2.buffers_ptr = args->buffers_ptr;
- exec2.buffer_count = args->buffer_count;
- exec2.batch_start_offset = args->batch_start_offset;
- exec2.batch_len = args->batch_len;
- exec2.DR1 = args->DR1;
- exec2.DR4 = args->DR4;
- exec2.num_cliprects = args->num_cliprects;
- exec2.cliprects_ptr = args->cliprects_ptr;
- exec2.flags = I915_EXEC_RENDER;
- i915_execbuffer2_set_context_id(exec2, 0);
-
- ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
- if (!ret) {
+ err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
+ if (exec2.flags & __EXEC_HAS_RELOC) {
struct drm_i915_gem_exec_object __user *user_exec_list =
u64_to_user_ptr(args->buffers_ptr);
/* Copy the new buffer offsets back to the user's exec list. */
for (i = 0; i < args->buffer_count; i++) {
+ if (!(exec2_list[i].offset & UPDATE))
+ continue;
+
exec2_list[i].offset =
- gen8_canonical_addr(exec2_list[i].offset);
- ret = __copy_to_user(&user_exec_list[i].offset,
- &exec2_list[i].offset,
- sizeof(user_exec_list[i].offset));
- if (ret) {
- ret = -EFAULT;
- DRM_DEBUG("failed to copy %d exec entries "
- "back to user (%d)\n",
- args->buffer_count, ret);
+ gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
+ exec2_list[i].offset &= PIN_OFFSET_MASK;
+ if (__copy_to_user(&user_exec_list[i].offset,
+ &exec2_list[i].offset,
+ sizeof(user_exec_list[i].offset)))
break;
- }
}
}
- drm_free_large(exec_list);
- drm_free_large(exec2_list);
- return ret;
+ kvfree(exec_list);
+ kvfree(exec2_list);
+ return err;
}
int
i915_gem_execbuffer2(struct drm_device *dev, void *data,
struct drm_file *file)
{
+ const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
struct drm_i915_gem_execbuffer2 *args = data;
- struct drm_i915_gem_exec_object2 *exec2_list = NULL;
- int ret;
+ struct drm_i915_gem_exec_object2 *exec2_list;
+ int err;
- if (args->buffer_count < 1 ||
- args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
+ if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
return -EINVAL;
}
- exec2_list = drm_malloc_gfp(args->buffer_count,
- sizeof(*exec2_list),
- GFP_TEMPORARY);
+ if (!i915_gem_check_execbuffer(args))
+ return -EINVAL;
+
+ /* Allocate an extra slot for use by the command parser */
+ exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+ __GFP_NOWARN | GFP_TEMPORARY);
if (exec2_list == NULL) {
DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
args->buffer_count);
return -ENOMEM;
}
- ret = copy_from_user(exec2_list,
- u64_to_user_ptr(args->buffers_ptr),
- sizeof(*exec2_list) * args->buffer_count);
- if (ret != 0) {
- DRM_DEBUG("copy %d exec entries failed %d\n",
- args->buffer_count, ret);
- drm_free_large(exec2_list);
+ if (copy_from_user(exec2_list,
+ u64_to_user_ptr(args->buffers_ptr),
+ sizeof(*exec2_list) * args->buffer_count)) {
+ DRM_DEBUG("copy %d exec entries failed\n", args->buffer_count);
+ kvfree(exec2_list);
return -EFAULT;
}
- ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
- if (!ret) {
- /* Copy the new buffer offsets back to the user's exec list. */
+ err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
+
+ /*
+ * Now that we have begun execution of the batchbuffer, we ignore
+ * any new error after this point. Also given that we have already
+ * updated the associated relocations, we try to write out the current
+ * object locations irrespective of any error.
+ */
+ if (args->flags & __EXEC_HAS_RELOC) {
struct drm_i915_gem_exec_object2 __user *user_exec_list =
- u64_to_user_ptr(args->buffers_ptr);
- int i;
+ u64_to_user_ptr(args->buffers_ptr);
+ unsigned int i;
+ /* Copy the new buffer offsets back to the user's exec list. */
+ user_access_begin();
for (i = 0; i < args->buffer_count; i++) {
+ if (!(exec2_list[i].offset & UPDATE))
+ continue;
+
exec2_list[i].offset =
- gen8_canonical_addr(exec2_list[i].offset);
- ret = __copy_to_user(&user_exec_list[i].offset,
- &exec2_list[i].offset,
- sizeof(user_exec_list[i].offset));
- if (ret) {
- ret = -EFAULT;
- DRM_DEBUG("failed to copy %d exec entries "
- "back to user\n",
- args->buffer_count);
- break;
- }
+ gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
+ unsafe_put_user(exec2_list[i].offset,
+ &user_exec_list[i].offset,
+ end_user);
}
+end_user:
+ user_access_end();
}
- drm_free_large(exec2_list);
- return ret;
+ args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
+ kvfree(exec2_list);
+ return err;
}
projects / karo-tx-linux.git / blobdiff