blk-mq: simplify __blk_mq_complete_request

[karo-tx-linux.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index ad057fe572a4b7124a53b70a956da1a73f7cfe4f..3a21948c867a0d02cdee98efbab8543278ea84f8 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -350,7 +350,7 @@ void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
        if (rq->tag != -1)
                blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
        if (sched_tag != -1)
-               blk_mq_sched_completed_request(hctx, rq);
+               blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
        blk_mq_sched_restart(hctx);
        blk_queue_exit(q);
 }
@@ -368,6 +368,7 @@ void blk_mq_finish_request(struct request *rq)
 {
        blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
 }
+EXPORT_SYMBOL_GPL(blk_mq_finish_request);
 
 void blk_mq_free_request(struct request *rq)
 {
@@ -405,12 +406,19 @@ static void __blk_mq_complete_request_remote(void *data)
        rq->q->softirq_done_fn(rq);
 }
 
-static void blk_mq_ipi_complete_request(struct request *rq)
+static void __blk_mq_complete_request(struct request *rq)
 {
        struct blk_mq_ctx *ctx = rq->mq_ctx;
        bool shared = false;
        int cpu;
 
+       if (rq->internal_tag != -1)
+               blk_mq_sched_completed_request(rq);
+       if (rq->rq_flags & RQF_STATS) {
+               blk_mq_poll_stats_start(rq->q);
+               blk_stat_add(rq);
+       }
+
        if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
                rq->q->softirq_done_fn(rq);
                return;
@@ -431,26 +439,6 @@ static void blk_mq_ipi_complete_request(struct request *rq)
        put_cpu();
 }
 
-static void blk_mq_stat_add(struct request *rq)
-{
-       if (rq->rq_flags & RQF_STATS) {
-               blk_mq_poll_stats_start(rq->q);
-               blk_stat_add(rq);
-       }
-}
-
-static void __blk_mq_complete_request(struct request *rq)
-{
-       struct request_queue *q = rq->q;
-
-       blk_mq_stat_add(rq);
-
-       if (!q->softirq_done_fn)
-               blk_mq_end_request(rq, rq->errors);
-       else
-               blk_mq_ipi_complete_request(rq);
-}
-
 /**
  * blk_mq_complete_request - end I/O on a request
  * @rq:                the request being processed
@@ -459,16 +447,14 @@ static void __blk_mq_complete_request(struct request *rq)
  *     Ends all I/O on a request. It does not handle partial completions.
  *     The actual completion happens out-of-order, through a IPI handler.
  **/
-void blk_mq_complete_request(struct request *rq, int error)
+void blk_mq_complete_request(struct request *rq)
 {
        struct request_queue *q = rq->q;
 
        if (unlikely(blk_should_fake_timeout(q)))
                return;
-       if (!blk_mark_rq_complete(rq)) {
-               rq->errors = error;
+       if (!blk_mark_rq_complete(rq))
                __blk_mq_complete_request(rq);
-       }
 }
 EXPORT_SYMBOL(blk_mq_complete_request);
 
@@ -1063,8 +1049,8 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list)
         */
        if (!list_empty(list)) {
                /*
-                * If we got a driver tag for the next request already,
-                * free it again.
+                * If an I/O scheduler has been configured and we got a driver
+                * tag for the next request already, free it again.
                 */
                rq = list_first_entry(list, struct request, queuelist);
                blk_mq_put_driver_tag(rq);
@@ -1074,16 +1060,24 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list)
                spin_unlock(&hctx->lock);
 
                /*
-                * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
-                * it's possible the queue is stopped and restarted again
-                * before this. Queue restart will dispatch requests. And since
-                * requests in rq_list aren't added into hctx->dispatch yet,
-                * the requests in rq_list might get lost.
+                * If SCHED_RESTART was set by the caller of this function and
+                * it is no longer set that means that it was cleared by another
+                * thread and hence that a queue rerun is needed.
                 *
-                * blk_mq_run_hw_queue() already checks the STOPPED bit
+                * If TAG_WAITING is set that means that an I/O scheduler has
+                * been configured and another thread is waiting for a driver
+                * tag. To guarantee fairness, do not rerun this hardware queue
+                * but let the other thread grab the driver tag.
                 *
-                * If RESTART or TAG_WAITING is set, then let completion restart
-                * the queue instead of potentially looping here.
+                * If no I/O scheduler has been configured it is possible that
+                * the hardware queue got stopped and restarted before requests
+                * were pushed back onto the dispatch list. Rerun the queue to
+                * avoid starvation. Notes:
+                * - blk_mq_run_hw_queue() checks whether or not a queue has
+                *   been stopped before rerunning a queue.
+                * - Some but not all block drivers stop a queue before
+                *   returning BLK_MQ_RQ_QUEUE_BUSY. Two exceptions are scsi-mq
+                *   and dm-rq.
                 */
                if (!blk_mq_sched_needs_restart(hctx) &&
                    !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state))
@@ -1175,6 +1169,7 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 {
        __blk_mq_delay_run_hw_queue(hctx, async, 0);
 }
+EXPORT_SYMBOL(blk_mq_run_hw_queue);
 
 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
 {
@@ -1411,7 +1406,7 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 
 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
 {
-       init_request_from_bio(rq, bio);
+       blk_init_request_from_bio(rq, bio);
 
        blk_account_io_start(rq, true);
 }