drm/i915: Make i915_vma_pin() small and inline

Not only is i915_vma_pin() called for every single object on every single
execbuf, it is usually a simple increment as the VMA is already bound for
execution by the GPU. Rearrange the tests for unbound and pin_count
overflow so that we can do the increment and test very cheaply and
compact enough to inline the operation into execbuf. The trick used is
to note that we can check for an overflow bit (keeping space available
for it inside the flags) at the same time as checking the binding bits.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1470324762-2545-17-git-send-email-chris@chris-wilson.co.uk

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 3ade87b1a57576811de4338301b99870919f6e93..f9e45ad9aee1032c2daa6cc74d61309339fcc50f 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3705,23 +3705,19 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
        obj->map_and_fenceable = mappable && fenceable;
 }
 
-int
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+int __i915_vma_do_pin(struct i915_vma *vma,
+                     u64 size, u64 alignment, u64 flags)
 {
-       unsigned int bound;
+       unsigned int bound = vma->flags;
        int ret;
 
        GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
        GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
 
-       bound = vma->flags;
-       if (WARN_ON((bound & I915_VMA_PIN_MASK) == I915_VMA_PIN_MASK))
-               return -EBUSY;
-
-       /* Pin early to prevent the shrinker/eviction logic from destroying
-        * our vma as we insert and bind.
-        */
-       __i915_vma_pin(vma);
+       if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
+               ret = -EBUSY;
+               goto err;
+       }
 
        if ((bound & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)) == 0) {
                ret = i915_vma_insert(vma, size, alignment, flags);

diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 5134c0433abdecca7dd78cf4c6845b5413438f0f..cc56206a160066bc79cb0ca2d0897c0233895cda 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -194,13 +194,15 @@ struct i915_vma {
         * bits with absolutely no headroom. So use 4 bits.
         */
 #define I915_VMA_PIN_MASK 0xf
+#define I915_VMA_PIN_OVERFLOW  BIT(5)
 
        /** Flags and address space this VMA is bound to */
-#define I915_VMA_GLOBAL_BIND   BIT(5)
-#define I915_VMA_LOCAL_BIND    BIT(6)
+#define I915_VMA_GLOBAL_BIND   BIT(6)
+#define I915_VMA_LOCAL_BIND    BIT(7)
+#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
 
-#define I915_VMA_GGTT  BIT(7)
-#define I915_VMA_CLOSED BIT(8)
+#define I915_VMA_GGTT  BIT(8)
+#define I915_VMA_CLOSED BIT(9)
 
        unsigned int active;
        struct i915_gem_active last_read[I915_NUM_ENGINES];
@@ -620,20 +622,39 @@ i915_ggtt_view_equal(const struct i915_ggtt_view *a,
        return true;
 }
 
-int __must_check
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
 /* Flags used by pin/bind&friends. */
-#define PIN_MAPPABLE           BIT(0)
-#define PIN_NONBLOCK           BIT(1)
-#define PIN_GLOBAL             BIT(2)
-#define PIN_OFFSET_BIAS                BIT(3)
-#define PIN_USER               BIT(4)
-#define PIN_UPDATE             BIT(5)
-#define PIN_ZONE_4G            BIT(6)
-#define PIN_HIGH               BIT(7)
-#define PIN_OFFSET_FIXED       BIT(8)
+#define PIN_NONBLOCK           BIT(0)
+#define PIN_MAPPABLE           BIT(1)
+#define PIN_ZONE_4G            BIT(2)
+
+#define PIN_MBZ                        BIT(5) /* I915_VMA_PIN_OVERFLOW */
+#define PIN_GLOBAL             BIT(6) /* I915_VMA_GLOBAL_BIND */
+#define PIN_USER               BIT(7) /* I915_VMA_LOCAL_BIND */
+#define PIN_UPDATE             BIT(8)
+
+#define PIN_HIGH               BIT(9)
+#define PIN_OFFSET_BIAS                BIT(10)
+#define PIN_OFFSET_FIXED       BIT(11)
 #define PIN_OFFSET_MASK                (~4095)
 
+int __i915_vma_do_pin(struct i915_vma *vma,
+                     u64 size, u64 alignment, u64 flags);
+static inline int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+{
+       BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
+       BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
+       BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
+
+       /* Pin early to prevent the shrinker/eviction logic from destroying
+        * our vma as we insert and bind.
+        */
+       if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
+               return 0;
+
+       return __i915_vma_do_pin(vma, size, alignment, flags);
+}
+
 static inline int i915_vma_pin_count(const struct i915_vma *vma)
 {
        return vma->flags & I915_VMA_PIN_MASK;
@@ -647,7 +668,7 @@ static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
 static inline void __i915_vma_pin(struct i915_vma *vma)
 {
        vma->flags++;
-       GEM_BUG_ON(!i915_vma_is_pinned(vma));
+       GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
 }
 
 static inline void __i915_vma_unpin(struct i915_vma *vma)