drm/i915: Store the reset counter when constructing a request

As the request is only valid during the same global reset epoch, we can
record the current reset_counter when constructing the request and reuse
it when waiting upon that request in future. This removes a very hairy
atomic check serialised by the struct_mutex at the time of waiting and
allows us to transfer those waits to a central dispatcher for all
waiters and all requests.

PS: With per-engine resets, we obviously cannot assume a global reset
epoch for the requests - a per-engine epoch makes the most sense. The
challenge then is how to handle checking in the waiter for when to break
the wait, as the fine-grained reset may also want to requeue the
request (i.e. the assumption that just because the epoch changes the
request is completed may be broken - or we just avoid breaking that
assumption with the fine-grained resets).

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: http://patchwork.freedesktop.org/patch/msgid/1460565315-7748-7-git-send-email-chris@chris-wilson.co.uk

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 69fb1853d5c33779141bf93b61014bbb3e3ed99d..54648e03db8c6cf07f2b0bab927b644386ad1359 100644 (file)
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2250,6 +2250,7 @@ struct drm_i915_gem_request {
        /** On Which ring this request was generated */
        struct drm_i915_private *i915;
        struct intel_engine_cs *engine;
+       unsigned reset_counter;
 
         /** GEM sequence number associated with the previous request,
          * when the HWS breadcrumb is equal to this the GPU is processing
@@ -3160,7 +3161,6 @@ void __i915_add_request(struct drm_i915_gem_request *req,
 #define i915_add_request_no_flush(req) \
        __i915_add_request(req, NULL, false)
 int __i915_wait_request(struct drm_i915_gem_request *req,
-                       unsigned reset_counter,
                        bool interruptible,
                        s64 *timeout,
                        struct intel_rps_client *rps);

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index defb445f21285c8ec9769fb99e255d391270ff1a..a62a5ec3679a9bda622a1923ee5a40aabfb8109f 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1213,7 +1213,6 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
 /**
  * __i915_wait_request - wait until execution of request has finished
  * @req: duh!
- * @reset_counter: reset sequence associated with the given request
  * @interruptible: do an interruptible wait (normally yes)
  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  *
@@ -1228,7 +1227,6 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
  * errno with remaining time filled in timeout argument.
  */
 int __i915_wait_request(struct drm_i915_gem_request *req,
-                       unsigned reset_counter,
                        bool interruptible,
                        s64 *timeout,
                        struct intel_rps_client *rps)
@@ -1290,7 +1288,7 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
 
                /* We need to check whether any gpu reset happened in between
                 * the caller grabbing the seqno and now ... */
-               if (reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
+               if (req->reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
                        /* ... but upgrade the -EAGAIN to an -EIO if the gpu
                         * is truely gone. */
                        ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
@@ -1460,13 +1458,7 @@ i915_wait_request(struct drm_i915_gem_request *req)
 
        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
 
-       ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
-       if (ret)
-               return ret;
-
-       ret = __i915_wait_request(req,
-                                 i915_reset_counter(&dev_priv->gpu_error),
-                                 interruptible, NULL, NULL);
+       ret = __i915_wait_request(req, interruptible, NULL, NULL);
        if (ret)
                return ret;
 
@@ -1541,7 +1533,6 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
        struct drm_device *dev = obj->base.dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
-       unsigned reset_counter;
        int ret, i, n = 0;
 
        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -1550,12 +1541,6 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
        if (!obj->active)
                return 0;
 
-       ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
-       if (ret)
-               return ret;
-
-       reset_counter = i915_reset_counter(&dev_priv->gpu_error);
-
        if (readonly) {
                struct drm_i915_gem_request *req;
 
@@ -1577,9 +1562,9 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
        }
 
        mutex_unlock(&dev->struct_mutex);
+       ret = 0;
        for (i = 0; ret == 0 && i < n; i++)
-               ret = __i915_wait_request(requests[i], reset_counter, true,
-                                         NULL, rps);
+               ret = __i915_wait_request(requests[i], true, NULL, rps);
        mutex_lock(&dev->struct_mutex);
 
        for (i = 0; i < n; i++) {
@@ -2735,6 +2720,7 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine,
                         struct drm_i915_gem_request **req_out)
 {
        struct drm_i915_private *dev_priv = to_i915(engine->dev);
+       unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
        struct drm_i915_gem_request *req;
        int ret;
 
@@ -2743,6 +2729,11 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine,
 
        *req_out = NULL;
 
+       ret = i915_gem_check_wedge(&dev_priv->gpu_error,
+                                  dev_priv->mm.interruptible);
+       if (ret)
+               return ret;
+
        req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
        if (req == NULL)
                return -ENOMEM;
@@ -2754,6 +2745,7 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine,
        kref_init(&req->ref);
        req->i915 = dev_priv;
        req->engine = engine;
+       req->reset_counter = reset_counter;
        req->ctx  = ctx;
        i915_gem_context_reference(req->ctx);
 
@@ -3132,11 +3124,9 @@ retire:
 int
 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
-       struct drm_i915_private *dev_priv = dev->dev_private;
        struct drm_i915_gem_wait *args = data;
        struct drm_i915_gem_object *obj;
        struct drm_i915_gem_request *req[I915_NUM_ENGINES];
-       unsigned reset_counter;
        int i, n = 0;
        int ret;
 
@@ -3170,7 +3160,6 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
        }
 
        drm_gem_object_unreference(&obj->base);
-       reset_counter = i915_reset_counter(&dev_priv->gpu_error);
 
        for (i = 0; i < I915_NUM_ENGINES; i++) {
                if (obj->last_read_req[i] == NULL)
@@ -3183,7 +3172,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
        for (i = 0; i < n; i++) {
                if (ret == 0)
-                       ret = __i915_wait_request(req[i], reset_counter, true,
+                       ret = __i915_wait_request(req[i], true,
                                                  args->timeout_ns > 0 ? &args->timeout_ns : NULL,
                                                  to_rps_client(file));
                i915_gem_request_unreference__unlocked(req[i]);
@@ -3215,7 +3204,6 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj,
        if (!i915_semaphore_is_enabled(obj->base.dev)) {
                struct drm_i915_private *i915 = to_i915(obj->base.dev);
                ret = __i915_wait_request(from_req,
-                                         i915_reset_counter(&i915->gpu_error),
                                          i915->mm.interruptible,
                                          NULL,
                                          &i915->rps.semaphores);
@@ -4171,7 +4159,6 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
        struct drm_i915_file_private *file_priv = file->driver_priv;
        unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
        struct drm_i915_gem_request *request, *target = NULL;
-       unsigned reset_counter;
        int ret;
 
        ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
@@ -4196,7 +4183,6 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 
                target = request;
        }
-       reset_counter = i915_reset_counter(&dev_priv->gpu_error);
        if (target)
                i915_gem_request_reference(target);
        spin_unlock(&file_priv->mm.lock);
@@ -4204,7 +4190,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
        if (target == NULL)
                return 0;
 
-       ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
+       ret = __i915_wait_request(target, true, NULL, NULL);
        if (ret == 0)
                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 

diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index bebaf75d5348e770ee27af0fc45cbd73dea309df..e6b5938ce6e244064cf2c7a824b4403d8b073e32 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -65,7 +65,6 @@ static void wait_rendering(struct drm_i915_gem_object *obj)
 {
        struct drm_device *dev = obj->base.dev;
        struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
-       unsigned reset_counter;
        int i, n;
 
        if (!obj->active)
@@ -82,12 +81,10 @@ static void wait_rendering(struct drm_i915_gem_object *obj)
                requests[n++] = i915_gem_request_reference(req);
        }
 
-       reset_counter = atomic_read(&to_i915(dev)->gpu_error.reset_counter);
        mutex_unlock(&dev->struct_mutex);
 
        for (i = 0; i < n; i++)
-               __i915_wait_request(requests[i], reset_counter, false,
-                                   NULL, NULL);
+               __i915_wait_request(requests[i], false, NULL, NULL);
 
        mutex_lock(&dev->struct_mutex);
 

diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index ec11c7ef08a209a99c7ea73b3fa787fb18fd730a..8763d953f1dfc56d961ce236063588f52741aecf 100644 (file)
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -11365,7 +11365,6 @@ static void intel_mmio_flip_work_func(struct work_struct *work)
 
        if (mmio_flip->req) {
                WARN_ON(__i915_wait_request(mmio_flip->req,
-                                           mmio_flip->crtc->reset_counter,
                                            false, NULL,
                                            &mmio_flip->i915->rps.mmioflips));
                i915_gem_request_unreference__unlocked(mmio_flip->req);
@@ -13426,9 +13425,6 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,
 
        ret = drm_atomic_helper_prepare_planes(dev, state);
        if (!ret && !async && !i915_reset_in_progress_or_wedged(&dev_priv->gpu_error)) {
-               u32 reset_counter;
-
-               reset_counter = i915_reset_counter(&dev_priv->gpu_error);
                mutex_unlock(&dev->struct_mutex);
 
                for_each_plane_in_state(state, plane, plane_state, i) {
@@ -13439,8 +13435,7 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,
                                continue;
 
                        ret = __i915_wait_request(intel_plane_state->wait_req,
-                                                 reset_counter, true,
-                                                 NULL, NULL);
+                                                 true, NULL, NULL);
 
                        /* Swallow -EIO errors to allow updates during hw lockup. */
                        if (ret == -EIO)

diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 0b5a31dad195975069dd8bae457751a5c3421485..6b0915eefe33a8397d1342874638dc45f01e0dc8 100644 (file)
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -890,16 +890,9 @@ static int logical_ring_prepare(struct drm_i915_gem_request *req, int bytes)
  */
 int intel_logical_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
 {
-       struct drm_i915_private *dev_priv;
        int ret;
 
        WARN_ON(req == NULL);
-       dev_priv = req->i915;
-
-       ret = i915_gem_check_wedge(&dev_priv->gpu_error,
-                                  dev_priv->mm.interruptible);
-       if (ret)
-               return ret;
 
        ret = logical_ring_prepare(req, num_dwords * sizeof(uint32_t));
        if (ret)

diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 83ed4158b53e53e6fdc6ad1d6a876653efe893a2..d9d4a6a41a74900c6cecabc88298a7967ebde624 100644 (file)
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -2364,7 +2364,6 @@ int intel_engine_idle(struct intel_engine_cs *engine)
 
        /* Make sure we do not trigger any retires */
        return __i915_wait_request(req,
-                                  i915_reset_counter(&req->i915->gpu_error),
                                   req->i915->mm.interruptible,
                                   NULL, NULL);
 }
@@ -2495,11 +2494,6 @@ int intel_ring_begin(struct drm_i915_gem_request *req,
        engine = req->engine;
        dev_priv = req->i915;
 
-       ret = i915_gem_check_wedge(&dev_priv->gpu_error,
-                                  dev_priv->mm.interruptible);
-       if (ret)
-               return ret;
-
        ret = __intel_ring_prepare(engine, num_dwords * sizeof(uint32_t));
        if (ret)
                return ret;