block: remove blk_end_request_cur

[karo-tx-linux.git] / block / blk-core.c

diff --git a/block/blk-core.c b/block/blk-core.c
index 4234332aa23c487354fd412c6cd7ba142c623aaf..728299323f65b27ac782174a486d6fa878edd45a 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -500,6 +500,13 @@ void blk_set_queue_dying(struct request_queue *q)
        queue_flag_set(QUEUE_FLAG_DYING, q);
        spin_unlock_irq(q->queue_lock);
 
+       /*
+        * When queue DYING flag is set, we need to block new req
+        * entering queue, so we call blk_freeze_queue_start() to
+        * prevent I/O from crossing blk_queue_enter().
+        */
+       blk_freeze_queue_start(q);
+
        if (q->mq_ops)
                blk_mq_wake_waiters(q);
        else {
@@ -669,6 +676,15 @@ int blk_queue_enter(struct request_queue *q, bool nowait)
                if (nowait)
                        return -EBUSY;
 
+               /*
+                * read pair of barrier in blk_freeze_queue_start(),
+                * we need to order reading __PERCPU_REF_DEAD flag of
+                * .q_usage_counter and reading .mq_freeze_depth or
+                * queue dying flag, otherwise the following wait may
+                * never return if the two reads are reordered.
+                */
+               smp_rmb();
+
                ret = wait_event_interruptible(q->mq_freeze_wq,
                                !atomic_read(&q->mq_freeze_depth) ||
                                blk_queue_dying(q));
@@ -1133,7 +1149,6 @@ static struct request *__get_request(struct request_list *rl, unsigned int op,
 
        blk_rq_init(q, rq);
        blk_rq_set_rl(rq, rl);
-       blk_rq_set_prio(rq, ioc);
        rq->cmd_flags = op;
        rq->rq_flags = rq_flags;
 
@@ -1620,6 +1635,7 @@ void init_request_from_bio(struct request *req, struct bio *bio)
 
        req->errors = 0;
        req->__sector = bio->bi_iter.bi_sector;
+       blk_rq_set_prio(req, rq_ioc(bio));
        if (ioprio_valid(bio_prio(bio)))
                req->ioprio = bio_prio(bio);
        blk_rq_bio_prep(req->q, req, bio);
@@ -1941,7 +1957,13 @@ generic_make_request_checks(struct bio *bio)
        if (!blkcg_bio_issue_check(q, bio))
                return false;
 
-       trace_block_bio_queue(q, bio);
+       if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
+               trace_block_bio_queue(q, bio);
+               /* Now that enqueuing has been traced, we need to trace
+                * completion as well.
+                */
+               bio_set_flag(bio, BIO_TRACE_COMPLETION);
+       }
        return true;
 
 not_supported:
@@ -2606,6 +2628,8 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
                if (bio_bytes == bio->bi_iter.bi_size)
                        req->bio = bio->bi_next;
 
+               /* Completion has already been traced */
+               bio_clear_flag(bio, BIO_TRACE_COMPLETION);
                req_bio_endio(req, bio, bio_bytes, error);
 
                total_bytes += bio_bytes;
@@ -2833,43 +2857,6 @@ void blk_end_request_all(struct request *rq, int error)
 }
 EXPORT_SYMBOL(blk_end_request_all);
 
-/**
- * blk_end_request_cur - Helper function to finish the current request chunk.
- * @rq: the request to finish the current chunk for
- * @error: %0 for success, < %0 for error
- *
- * Description:
- *     Complete the current consecutively mapped chunk from @rq.
- *
- * Return:
- *     %false - we are done with this request
- *     %true  - still buffers pending for this request
- */
-bool blk_end_request_cur(struct request *rq, int error)
-{
-       return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
-}
-EXPORT_SYMBOL(blk_end_request_cur);
-
-/**
- * blk_end_request_err - Finish a request till the next failure boundary.
- * @rq: the request to finish till the next failure boundary for
- * @error: must be negative errno
- *
- * Description:
- *     Complete @rq till the next failure boundary.
- *
- * Return:
- *     %false - we are done with this request
- *     %true  - still buffers pending for this request
- */
-bool blk_end_request_err(struct request *rq, int error)
-{
-       WARN_ON(error >= 0);
-       return blk_end_request(rq, error, blk_rq_err_bytes(rq));
-}
-EXPORT_SYMBOL_GPL(blk_end_request_err);
-
 /**
  * __blk_end_request - Helper function for drivers to complete the request.
  * @rq:       the request being processed
@@ -2929,26 +2916,6 @@ bool __blk_end_request_cur(struct request *rq, int error)
 }
 EXPORT_SYMBOL(__blk_end_request_cur);
 
-/**
- * __blk_end_request_err - Finish a request till the next failure boundary.
- * @rq: the request to finish till the next failure boundary for
- * @error: must be negative errno
- *
- * Description:
- *     Complete @rq till the next failure boundary.  Must be called
- *     with queue lock held.
- *
- * Return:
- *     %false - we are done with this request
- *     %true  - still buffers pending for this request
- */
-bool __blk_end_request_err(struct request *rq, int error)
-{
-       WARN_ON(error >= 0);
-       return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
-}
-EXPORT_SYMBOL_GPL(__blk_end_request_err);
-
 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
                     struct bio *bio)
 {