block/blk-mq.c

   1 #include <linux/kernel.h>
   2 #include <linux/module.h>
   3 #include <linux/backing-dev.h>
   4 #include <linux/bio.h>
   5 #include <linux/blkdev.h>
   6 #include <linux/mm.h>
   7 #include <linux/init.h>
   8 #include <linux/slab.h>
   9 #include <linux/workqueue.h>
  10 #include <linux/smp.h>
  11 #include <linux/llist.h>
  12 #include <linux/list_sort.h>
  13 #include <linux/cpu.h>
  14 #include <linux/cache.h>
  15 #include <linux/sched/sysctl.h>
  16 #include <linux/delay.h>
  17
  18 #include <trace/events/block.h>
  19
  20 #include <linux/blk-mq.h>
  21 #include "blk.h"
  22 #include "blk-mq.h"
  23 #include "blk-mq-tag.h"
  24
  25 static DEFINE_MUTEX(all_q_mutex);
  26 static LIST_HEAD(all_q_list);
  27
  28 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  29
  30 static struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
  31                                            unsigned int cpu)
  32 {
  33         return per_cpu_ptr(q->queue_ctx, cpu);
  34 }
  35
  36 /*
  37  * This assumes per-cpu software queueing queues. They could be per-node
  38  * as well, for instance. For now this is hardcoded as-is. Note that we don't
  39  * care about preemption, since we know the ctx's are persistent. This does
  40  * mean that we can't rely on ctx always matching the currently running CPU.
  41  */
  42 static struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
  43 {
  44         return __blk_mq_get_ctx(q, get_cpu());
  45 }
  46
  47 static void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
  48 {
  49         put_cpu();
  50 }
  51
  52 /*
  53  * Check if any of the ctx's have pending work in this hardware queue
  54  */
  55 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  56 {
  57         unsigned int i;
  58
  59         for (i = 0; i < hctx->ctx_map.map_size; i++)
  60                 if (hctx->ctx_map.map[i].word)
  61                         return true;
  62
  63         return false;
  64 }
  65
  66 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  67                                               struct blk_mq_ctx *ctx)
  68 {
  69         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  70 }
  71
  72 #define CTX_TO_BIT(hctx, ctx)   \
  73         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  74
  75 /*
  76  * Mark this ctx as having pending work in this hardware queue
  77  */
  78 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  79                                      struct blk_mq_ctx *ctx)
  80 {
  81         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  82
  83         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  84                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  85 }
  86
  87 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  88                                       struct blk_mq_ctx *ctx)
  89 {
  90         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  91
  92         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  93 }
  94
  95 static int blk_mq_queue_enter(struct request_queue *q)
  96 {
  97         int ret;
  98
  99         __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
 100         smp_wmb();
 101         /* we have problems to freeze the queue if it's initializing */
 102         if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
 103                 return 0;
 104
 105         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 106
 107         spin_lock_irq(q->queue_lock);
 108         ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
 109                 !blk_queue_bypass(q) || blk_queue_dying(q),
 110                 *q->queue_lock);
 111         /* inc usage with lock hold to avoid freeze_queue runs here */
 112         if (!ret && !blk_queue_dying(q))
 113                 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
 114         else if (blk_queue_dying(q))
 115                 ret = -ENODEV;
 116         spin_unlock_irq(q->queue_lock);
 117
 118         return ret;
 119 }
 120
 121 static void blk_mq_queue_exit(struct request_queue *q)
 122 {
 123         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 124 }
 125
 126 static void __blk_mq_drain_queue(struct request_queue *q)
 127 {
 128         while (true) {
 129                 s64 count;
 130
 131                 spin_lock_irq(q->queue_lock);
 132                 count = percpu_counter_sum(&q->mq_usage_counter);
 133                 spin_unlock_irq(q->queue_lock);
 134
 135                 if (count == 0)
 136                         break;
 137                 blk_mq_run_queues(q, false);
 138                 msleep(10);
 139         }
 140 }
 141
 142 /*
 143  * Guarantee no request is in use, so we can change any data structure of
 144  * the queue afterward.
 145  */
 146 static void blk_mq_freeze_queue(struct request_queue *q)
 147 {
 148         bool drain;
 149
 150         spin_lock_irq(q->queue_lock);
 151         drain = !q->bypass_depth++;
 152         queue_flag_set(QUEUE_FLAG_BYPASS, q);
 153         spin_unlock_irq(q->queue_lock);
 154
 155         if (drain)
 156                 __blk_mq_drain_queue(q);
 157 }
 158
 159 void blk_mq_drain_queue(struct request_queue *q)
 160 {
 161         __blk_mq_drain_queue(q);
 162 }
 163
 164 static void blk_mq_unfreeze_queue(struct request_queue *q)
 165 {
 166         bool wake = false;
 167
 168         spin_lock_irq(q->queue_lock);
 169         if (!--q->bypass_depth) {
 170                 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
 171                 wake = true;
 172         }
 173         WARN_ON_ONCE(q->bypass_depth < 0);
 174         spin_unlock_irq(q->queue_lock);
 175         if (wake)
 176                 wake_up_all(&q->mq_freeze_wq);
 177 }
 178
 179 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 180 {
 181         return blk_mq_has_free_tags(hctx->tags);
 182 }
 183 EXPORT_SYMBOL(blk_mq_can_queue);
 184
 185 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 186                                struct request *rq, unsigned int rw_flags)
 187 {
 188         if (blk_queue_io_stat(q))
 189                 rw_flags |= REQ_IO_STAT;
 190
 191         INIT_LIST_HEAD(&rq->queuelist);
 192         /* csd/requeue_work/fifo_time is initialized before use */
 193         rq->q = q;
 194         rq->mq_ctx = ctx;
 195         rq->cmd_flags |= rw_flags;
 196         rq->cmd_type = 0;
 197         /* do not touch atomic flags, it needs atomic ops against the timer */
 198         rq->cpu = -1;
 199         rq->__data_len = 0;
 200         rq->__sector = (sector_t) -1;
 201         rq->bio = NULL;
 202         rq->biotail = NULL;
 203         INIT_HLIST_NODE(&rq->hash);
 204         RB_CLEAR_NODE(&rq->rb_node);
 205         memset(&rq->flush, 0, max(sizeof(rq->flush), sizeof(rq->elv)));
 206         rq->rq_disk = NULL;
 207         rq->part = NULL;
 208         rq->start_time = jiffies;
 209 #ifdef CONFIG_BLK_CGROUP
 210         rq->rl = NULL;
 211         set_start_time_ns(rq);
 212         rq->io_start_time_ns = 0;
 213 #endif
 214         rq->nr_phys_segments = 0;
 215 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 216         rq->nr_integrity_segments = 0;
 217 #endif
 218         rq->ioprio = 0;
 219         rq->special = NULL;
 220         /* tag was already set */
 221         rq->errors = 0;
 222         memset(rq->__cmd, 0, sizeof(rq->__cmd));
 223         rq->cmd = rq->__cmd;
 224         rq->cmd_len = BLK_MAX_CDB;
 225
 226         rq->extra_len = 0;
 227         rq->sense_len = 0;
 228         rq->resid_len = 0;
 229         rq->sense = NULL;
 230
 231         rq->deadline = 0;
 232         INIT_LIST_HEAD(&rq->timeout_list);
 233         rq->timeout = 0;
 234         rq->retries = 0;
 235         rq->end_io = NULL;
 236         rq->end_io_data = NULL;
 237         rq->next_rq = NULL;
 238
 239         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 240 }
 241
 242 static struct request *
 243 __blk_mq_alloc_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
 244                 struct blk_mq_ctx *ctx, int rw, gfp_t gfp, bool reserved)
 245 {
 246         struct request *rq;
 247         unsigned int tag;
 248
 249         tag = blk_mq_get_tag(hctx, &ctx->last_tag, gfp, reserved);
 250         if (tag != BLK_MQ_TAG_FAIL) {
 251                 rq = hctx->tags->rqs[tag];
 252
 253                 rq->cmd_flags = 0;
 254                 if (blk_mq_tag_busy(hctx)) {
 255                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 256                         atomic_inc(&hctx->nr_active);
 257                 }
 258
 259                 rq->tag = tag;
 260                 blk_mq_rq_ctx_init(q, ctx, rq, rw);
 261                 return rq;
 262         }
 263
 264         return NULL;
 265 }
 266
 267
 268 static struct request *blk_mq_alloc_request_pinned(struct request_queue *q,
 269                                                    int rw, gfp_t gfp,
 270                                                    bool reserved)
 271 {
 272         struct request *rq;
 273
 274         do {
 275                 struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
 276                 struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue(q, ctx->cpu);
 277
 278                 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, gfp & ~__GFP_WAIT,
 279                                                 reserved);
 280                 if (rq)
 281                         break;
 282
 283                 if (gfp & __GFP_WAIT) {
 284                         __blk_mq_run_hw_queue(hctx);
 285                         blk_mq_put_ctx(ctx);
 286                 } else {
 287                         blk_mq_put_ctx(ctx);
 288                         break;
 289                 }
 290
 291                 blk_mq_wait_for_tags(hctx, reserved);
 292         } while (1);
 293
 294         return rq;
 295 }
 296
 297 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 298                 bool reserved)
 299 {
 300         struct request *rq;
 301
 302         if (blk_mq_queue_enter(q))
 303                 return NULL;
 304
 305         rq = blk_mq_alloc_request_pinned(q, rw, gfp, reserved);
 306         if (rq)
 307                 blk_mq_put_ctx(rq->mq_ctx);
 308         return rq;
 309 }
 310 EXPORT_SYMBOL(blk_mq_alloc_request);
 311
 312 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 313                                   struct blk_mq_ctx *ctx, struct request *rq)
 314 {
 315         const int tag = rq->tag;
 316         struct request_queue *q = rq->q;
 317
 318         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 319                 atomic_dec(&hctx->nr_active);
 320
 321         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 322         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 323         blk_mq_queue_exit(q);
 324 }
 325
 326 void blk_mq_free_request(struct request *rq)
 327 {
 328         struct blk_mq_ctx *ctx = rq->mq_ctx;
 329         struct blk_mq_hw_ctx *hctx;
 330         struct request_queue *q = rq->q;
 331
 332         ctx->rq_completed[rq_is_sync(rq)]++;
 333
 334         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 335         __blk_mq_free_request(hctx, ctx, rq);
 336 }
 337
 338 /*
 339  * Clone all relevant state from a request that has been put on hold in
 340  * the flush state machine into the preallocated flush request that hangs
 341  * off the request queue.
 342  *
 343  * For a driver the flush request should be invisible, that's why we are
 344  * impersonating the original request here.
 345  */
 346 void blk_mq_clone_flush_request(struct request *flush_rq,
 347                 struct request *orig_rq)
 348 {
 349         struct blk_mq_hw_ctx *hctx =
 350                 orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu);
 351
 352         flush_rq->mq_ctx = orig_rq->mq_ctx;
 353         flush_rq->tag = orig_rq->tag;
 354         memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq),
 355                 hctx->cmd_size);
 356 }
 357
 358 inline void __blk_mq_end_io(struct request *rq, int error)
 359 {
 360         blk_account_io_done(rq);
 361
 362         if (rq->end_io) {
 363                 rq->end_io(rq, error);
 364         } else {
 365                 if (unlikely(blk_bidi_rq(rq)))
 366                         blk_mq_free_request(rq->next_rq);
 367                 blk_mq_free_request(rq);
 368         }
 369 }
 370 EXPORT_SYMBOL(__blk_mq_end_io);
 371
 372 void blk_mq_end_io(struct request *rq, int error)
 373 {
 374         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 375                 BUG();
 376         __blk_mq_end_io(rq, error);
 377 }
 378 EXPORT_SYMBOL(blk_mq_end_io);
 379
 380 static void __blk_mq_complete_request_remote(void *data)
 381 {
 382         struct request *rq = data;
 383
 384         rq->q->softirq_done_fn(rq);
 385 }
 386
 387 void __blk_mq_complete_request(struct request *rq)
 388 {
 389         struct blk_mq_ctx *ctx = rq->mq_ctx;
 390         bool shared = false;
 391         int cpu;
 392
 393         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 394                 rq->q->softirq_done_fn(rq);
 395                 return;
 396         }
 397
 398         cpu = get_cpu();
 399         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 400                 shared = cpus_share_cache(cpu, ctx->cpu);
 401
 402         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 403                 rq->csd.func = __blk_mq_complete_request_remote;
 404                 rq->csd.info = rq;
 405                 rq->csd.flags = 0;
 406                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 407         } else {
 408                 rq->q->softirq_done_fn(rq);
 409         }
 410         put_cpu();
 411 }
 412
 413 /**
 414  * blk_mq_complete_request - end I/O on a request
 415  * @rq:         the request being processed
 416  *
 417  * Description:
 418  *      Ends all I/O on a request. It does not handle partial completions.
 419  *      The actual completion happens out-of-order, through a IPI handler.
 420  **/
 421 void blk_mq_complete_request(struct request *rq)
 422 {
 423         struct request_queue *q = rq->q;
 424
 425         if (unlikely(blk_should_fake_timeout(q)))
 426                 return;
 427         if (!blk_mark_rq_complete(rq)) {
 428                 if (q->softirq_done_fn)
 429                         __blk_mq_complete_request(rq);
 430                 else
 431                         blk_mq_end_io(rq, rq->errors);
 432         }
 433 }
 434 EXPORT_SYMBOL(blk_mq_complete_request);
 435
 436 static void blk_mq_start_request(struct request *rq, bool last)
 437 {
 438         struct request_queue *q = rq->q;
 439
 440         trace_block_rq_issue(q, rq);
 441
 442         rq->resid_len = blk_rq_bytes(rq);
 443         if (unlikely(blk_bidi_rq(rq)))
 444                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 445
 446         /*
 447          * Just mark start time and set the started bit. Due to memory
 448          * ordering, we know we'll see the correct deadline as long as
 449          * REQ_ATOMIC_STARTED is seen. Use the default queue timeout,
 450          * unless one has been set in the request.
 451          */
 452         if (!rq->timeout)
 453                 rq->deadline = jiffies + q->rq_timeout;
 454         else
 455                 rq->deadline = jiffies + rq->timeout;
 456
 457         /*
 458          * Mark us as started and clear complete. Complete might have been
 459          * set if requeue raced with timeout, which then marked it as
 460          * complete. So be sure to clear complete again when we start
 461          * the request, otherwise we'll ignore the completion event.
 462          */
 463         set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 464         clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 465
 466         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 467                 /*
 468                  * Make sure space for the drain appears.  We know we can do
 469                  * this because max_hw_segments has been adjusted to be one
 470                  * fewer than the device can handle.
 471                  */
 472                 rq->nr_phys_segments++;
 473         }
 474
 475         /*
 476          * Flag the last request in the series so that drivers know when IO
 477          * should be kicked off, if they don't do it on a per-request basis.
 478          *
 479          * Note: the flag isn't the only condition drivers should do kick off.
 480          * If drive is busy, the last request might not have the bit set.
 481          */
 482         if (last)
 483                 rq->cmd_flags |= REQ_END;
 484 }
 485
 486 static void __blk_mq_requeue_request(struct request *rq)
 487 {
 488         struct request_queue *q = rq->q;
 489
 490         trace_block_rq_requeue(q, rq);
 491         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 492
 493         rq->cmd_flags &= ~REQ_END;
 494
 495         if (q->dma_drain_size && blk_rq_bytes(rq))
 496                 rq->nr_phys_segments--;
 497 }
 498
 499 void blk_mq_requeue_request(struct request *rq)
 500 {
 501         __blk_mq_requeue_request(rq);
 502         blk_clear_rq_complete(rq);
 503
 504         BUG_ON(blk_queued_rq(rq));
 505         blk_mq_add_to_requeue_list(rq, true);
 506 }
 507 EXPORT_SYMBOL(blk_mq_requeue_request);
 508
 509 static void blk_mq_requeue_work(struct work_struct *work)
 510 {
 511         struct request_queue *q =
 512                 container_of(work, struct request_queue, requeue_work);
 513         LIST_HEAD(rq_list);
 514         struct request *rq, *next;
 515         unsigned long flags;
 516
 517         spin_lock_irqsave(&q->requeue_lock, flags);
 518         list_splice_init(&q->requeue_list, &rq_list);
 519         spin_unlock_irqrestore(&q->requeue_lock, flags);
 520
 521         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 522                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 523                         continue;
 524
 525                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 526                 list_del_init(&rq->queuelist);
 527                 blk_mq_insert_request(rq, true, false, false);
 528         }
 529
 530         while (!list_empty(&rq_list)) {
 531                 rq = list_entry(rq_list.next, struct request, queuelist);
 532                 list_del_init(&rq->queuelist);
 533                 blk_mq_insert_request(rq, false, false, false);
 534         }
 535
 536         blk_mq_run_queues(q, false);
 537 }
 538
 539 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 540 {
 541         struct request_queue *q = rq->q;
 542         unsigned long flags;
 543
 544         /*
 545          * We abuse this flag that is otherwise used by the I/O scheduler to
 546          * request head insertation from the workqueue.
 547          */
 548         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 549
 550         spin_lock_irqsave(&q->requeue_lock, flags);
 551         if (at_head) {
 552                 rq->cmd_flags |= REQ_SOFTBARRIER;
 553                 list_add(&rq->queuelist, &q->requeue_list);
 554         } else {
 555                 list_add_tail(&rq->queuelist, &q->requeue_list);
 556         }
 557         spin_unlock_irqrestore(&q->requeue_lock, flags);
 558 }
 559 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 560
 561 void blk_mq_kick_requeue_list(struct request_queue *q)
 562 {
 563         kblockd_schedule_work(&q->requeue_work);
 564 }
 565 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 566
 567 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
 568 {
 569         return tags->rqs[tag];
 570 }
 571 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 572
 573 struct blk_mq_timeout_data {
 574         struct blk_mq_hw_ctx *hctx;
 575         unsigned long *next;
 576         unsigned int *next_set;
 577 };
 578
 579 static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
 580 {
 581         struct blk_mq_timeout_data *data = __data;
 582         struct blk_mq_hw_ctx *hctx = data->hctx;
 583         unsigned int tag;
 584
 585          /* It may not be in flight yet (this is where
 586          * the REQ_ATOMIC_STARTED flag comes in). The requests are
 587          * statically allocated, so we know it's always safe to access the
 588          * memory associated with a bit offset into ->rqs[].
 589          */
 590         tag = 0;
 591         do {
 592                 struct request *rq;
 593
 594                 tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag);
 595                 if (tag >= hctx->tags->nr_tags)
 596                         break;
 597
 598                 rq = blk_mq_tag_to_rq(hctx->tags, tag++);
 599                 if (rq->q != hctx->queue)
 600                         continue;
 601                 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 602                         continue;
 603
 604                 blk_rq_check_expired(rq, data->next, data->next_set);
 605         } while (1);
 606 }
 607
 608 static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
 609                                         unsigned long *next,
 610                                         unsigned int *next_set)
 611 {
 612         struct blk_mq_timeout_data data = {
 613                 .hctx           = hctx,
 614                 .next           = next,
 615                 .next_set       = next_set,
 616         };
 617
 618         /*
 619          * Ask the tagging code to iterate busy requests, so we can
 620          * check them for timeout.
 621          */
 622         blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
 623 }
 624
 625 static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq)
 626 {
 627         struct request_queue *q = rq->q;
 628
 629         /*
 630          * We know that complete is set at this point. If STARTED isn't set
 631          * anymore, then the request isn't active and the "timeout" should
 632          * just be ignored. This can happen due to the bitflag ordering.
 633          * Timeout first checks if STARTED is set, and if it is, assumes
 634          * the request is active. But if we race with completion, then
 635          * we both flags will get cleared. So check here again, and ignore
 636          * a timeout event with a request that isn't active.
 637          */
 638         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 639                 return BLK_EH_NOT_HANDLED;
 640
 641         if (!q->mq_ops->timeout)
 642                 return BLK_EH_RESET_TIMER;
 643
 644         return q->mq_ops->timeout(rq);
 645 }
 646
 647 static void blk_mq_rq_timer(unsigned long data)
 648 {
 649         struct request_queue *q = (struct request_queue *) data;
 650         struct blk_mq_hw_ctx *hctx;
 651         unsigned long next = 0;
 652         int i, next_set = 0;
 653
 654         queue_for_each_hw_ctx(q, hctx, i) {
 655                 /*
 656                  * If not software queues are currently mapped to this
 657                  * hardware queue, there's nothing to check
 658                  */
 659                 if (!hctx->nr_ctx || !hctx->tags)
 660                         continue;
 661
 662                 blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
 663         }
 664
 665         if (next_set) {
 666                 next = blk_rq_timeout(round_jiffies_up(next));
 667                 mod_timer(&q->timeout, next);
 668         } else {
 669                 queue_for_each_hw_ctx(q, hctx, i)
 670                         blk_mq_tag_idle(hctx);
 671         }
 672 }
 673
 674 /*
 675  * Reverse check our software queue for entries that we could potentially
 676  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 677  * too much time checking for merges.
 678  */
 679 static bool blk_mq_attempt_merge(struct request_queue *q,
 680                                  struct blk_mq_ctx *ctx, struct bio *bio)
 681 {
 682         struct request *rq;
 683         int checked = 8;
 684
 685         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 686                 int el_ret;
 687
 688                 if (!checked--)
 689                         break;
 690
 691                 if (!blk_rq_merge_ok(rq, bio))
 692                         continue;
 693
 694                 el_ret = blk_try_merge(rq, bio);
 695                 if (el_ret == ELEVATOR_BACK_MERGE) {
 696                         if (bio_attempt_back_merge(q, rq, bio)) {
 697                                 ctx->rq_merged++;
 698                                 return true;
 699                         }
 700                         break;
 701                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 702                         if (bio_attempt_front_merge(q, rq, bio)) {
 703                                 ctx->rq_merged++;
 704                                 return true;
 705                         }
 706                         break;
 707                 }
 708         }
 709
 710         return false;
 711 }
 712
 713 /*
 714  * Process software queues that have been marked busy, splicing them
 715  * to the for-dispatch
 716  */
 717 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 718 {
 719         struct blk_mq_ctx *ctx;
 720         int i;
 721
 722         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 723                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 724                 unsigned int off, bit;
 725
 726                 if (!bm->word)
 727                         continue;
 728
 729                 bit = 0;
 730                 off = i * hctx->ctx_map.bits_per_word;
 731                 do {
 732                         bit = find_next_bit(&bm->word, bm->depth, bit);
 733                         if (bit >= bm->depth)
 734                                 break;
 735
 736                         ctx = hctx->ctxs[bit + off];
 737                         clear_bit(bit, &bm->word);
 738                         spin_lock(&ctx->lock);
 739                         list_splice_tail_init(&ctx->rq_list, list);
 740                         spin_unlock(&ctx->lock);
 741
 742                         bit++;
 743                 } while (1);
 744         }
 745 }
 746
 747 /*
 748  * Run this hardware queue, pulling any software queues mapped to it in.
 749  * Note that this function currently has various problems around ordering
 750  * of IO. In particular, we'd like FIFO behaviour on handling existing
 751  * items on the hctx->dispatch list. Ignore that for now.
 752  */
 753 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 754 {
 755         struct request_queue *q = hctx->queue;
 756         struct request *rq;
 757         LIST_HEAD(rq_list);
 758         int queued;
 759
 760         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 761
 762         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 763                 return;
 764
 765         hctx->run++;
 766
 767         /*
 768          * Touch any software queue that has pending entries.
 769          */
 770         flush_busy_ctxs(hctx, &rq_list);
 771
 772         /*
 773          * If we have previous entries on our dispatch list, grab them
 774          * and stuff them at the front for more fair dispatch.
 775          */
 776         if (!list_empty_careful(&hctx->dispatch)) {
 777                 spin_lock(&hctx->lock);
 778                 if (!list_empty(&hctx->dispatch))
 779                         list_splice_init(&hctx->dispatch, &rq_list);
 780                 spin_unlock(&hctx->lock);
 781         }
 782
 783         /*
 784          * Now process all the entries, sending them to the driver.
 785          */
 786         queued = 0;
 787         while (!list_empty(&rq_list)) {
 788                 int ret;
 789
 790                 rq = list_first_entry(&rq_list, struct request, queuelist);
 791                 list_del_init(&rq->queuelist);
 792
 793                 blk_mq_start_request(rq, list_empty(&rq_list));
 794
 795                 ret = q->mq_ops->queue_rq(hctx, rq);
 796                 switch (ret) {
 797                 case BLK_MQ_RQ_QUEUE_OK:
 798                         queued++;
 799                         continue;
 800                 case BLK_MQ_RQ_QUEUE_BUSY:
 801                         list_add(&rq->queuelist, &rq_list);
 802                         __blk_mq_requeue_request(rq);
 803                         break;
 804                 default:
 805                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 806                 case BLK_MQ_RQ_QUEUE_ERROR:
 807                         rq->errors = -EIO;
 808                         blk_mq_end_io(rq, rq->errors);
 809                         break;
 810                 }
 811
 812                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 813                         break;
 814         }
 815
 816         if (!queued)
 817                 hctx->dispatched[0]++;
 818         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 819                 hctx->dispatched[ilog2(queued) + 1]++;
 820
 821         /*
 822          * Any items that need requeuing? Stuff them into hctx->dispatch,
 823          * that is where we will continue on next queue run.
 824          */
 825         if (!list_empty(&rq_list)) {
 826                 spin_lock(&hctx->lock);
 827                 list_splice(&rq_list, &hctx->dispatch);
 828                 spin_unlock(&hctx->lock);
 829         }
 830 }
 831
 832 /*
 833  * It'd be great if the workqueue API had a way to pass
 834  * in a mask and had some smarts for more clever placement.
 835  * For now we just round-robin here, switching for every
 836  * BLK_MQ_CPU_WORK_BATCH queued items.
 837  */
 838 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 839 {
 840         int cpu = hctx->next_cpu;
 841
 842         if (--hctx->next_cpu_batch <= 0) {
 843                 int next_cpu;
 844
 845                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 846                 if (next_cpu >= nr_cpu_ids)
 847                         next_cpu = cpumask_first(hctx->cpumask);
 848
 849                 hctx->next_cpu = next_cpu;
 850                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 851         }
 852
 853         return cpu;
 854 }
 855
 856 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 857 {
 858         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 859                 return;
 860
 861         if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
 862                 __blk_mq_run_hw_queue(hctx);
 863         else if (hctx->queue->nr_hw_queues == 1)
 864                 kblockd_schedule_delayed_work(&hctx->run_work, 0);
 865         else {
 866                 unsigned int cpu;
 867
 868                 cpu = blk_mq_hctx_next_cpu(hctx);
 869                 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
 870         }
 871 }
 872
 873 void blk_mq_run_queues(struct request_queue *q, bool async)
 874 {
 875         struct blk_mq_hw_ctx *hctx;
 876         int i;
 877
 878         queue_for_each_hw_ctx(q, hctx, i) {
 879                 if ((!blk_mq_hctx_has_pending(hctx) &&
 880                     list_empty_careful(&hctx->dispatch)) ||
 881                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 882                         continue;
 883
 884                 preempt_disable();
 885                 blk_mq_run_hw_queue(hctx, async);
 886                 preempt_enable();
 887         }
 888 }
 889 EXPORT_SYMBOL(blk_mq_run_queues);
 890
 891 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 892 {
 893         cancel_delayed_work(&hctx->run_work);
 894         cancel_delayed_work(&hctx->delay_work);
 895         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 896 }
 897 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 898
 899 void blk_mq_stop_hw_queues(struct request_queue *q)
 900 {
 901         struct blk_mq_hw_ctx *hctx;
 902         int i;
 903
 904         queue_for_each_hw_ctx(q, hctx, i)
 905                 blk_mq_stop_hw_queue(hctx);
 906 }
 907 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 908
 909 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 910 {
 911         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 912
 913         preempt_disable();
 914         __blk_mq_run_hw_queue(hctx);
 915         preempt_enable();
 916 }
 917 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 918
 919 void blk_mq_start_hw_queues(struct request_queue *q)
 920 {
 921         struct blk_mq_hw_ctx *hctx;
 922         int i;
 923
 924         queue_for_each_hw_ctx(q, hctx, i)
 925                 blk_mq_start_hw_queue(hctx);
 926 }
 927 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 928
 929
 930 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 931 {
 932         struct blk_mq_hw_ctx *hctx;
 933         int i;
 934
 935         queue_for_each_hw_ctx(q, hctx, i) {
 936                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 937                         continue;
 938
 939                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 940                 preempt_disable();
 941                 blk_mq_run_hw_queue(hctx, async);
 942                 preempt_enable();
 943         }
 944 }
 945 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 946
 947 static void blk_mq_run_work_fn(struct work_struct *work)
 948 {
 949         struct blk_mq_hw_ctx *hctx;
 950
 951         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 952
 953         __blk_mq_run_hw_queue(hctx);
 954 }
 955
 956 static void blk_mq_delay_work_fn(struct work_struct *work)
 957 {
 958         struct blk_mq_hw_ctx *hctx;
 959
 960         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 961
 962         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 963                 __blk_mq_run_hw_queue(hctx);
 964 }
 965
 966 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 967 {
 968         unsigned long tmo = msecs_to_jiffies(msecs);
 969
 970         if (hctx->queue->nr_hw_queues == 1)
 971                 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
 972         else {
 973                 unsigned int cpu;
 974
 975                 cpu = blk_mq_hctx_next_cpu(hctx);
 976                 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
 977         }
 978 }
 979 EXPORT_SYMBOL(blk_mq_delay_queue);
 980
 981 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 982                                     struct request *rq, bool at_head)
 983 {
 984         struct blk_mq_ctx *ctx = rq->mq_ctx;
 985
 986         trace_block_rq_insert(hctx->queue, rq);
 987
 988         if (at_head)
 989                 list_add(&rq->queuelist, &ctx->rq_list);
 990         else
 991                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 992
 993         blk_mq_hctx_mark_pending(hctx, ctx);
 994
 995         /*
 996          * We do this early, to ensure we are on the right CPU.
 997          */
 998         blk_add_timer(rq);
 999 }
1000
1001 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
1002                 bool async)
1003 {
1004         struct request_queue *q = rq->q;
1005         struct blk_mq_hw_ctx *hctx;
1006         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
1007
1008         current_ctx = blk_mq_get_ctx(q);
1009         if (!cpu_online(ctx->cpu))
1010                 rq->mq_ctx = ctx = current_ctx;
1011
1012         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1013
1014         if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA) &&
1015             !(rq->cmd_flags & (REQ_FLUSH_SEQ))) {
1016                 blk_insert_flush(rq);
1017         } else {
1018                 spin_lock(&ctx->lock);
1019                 __blk_mq_insert_request(hctx, rq, at_head);
1020                 spin_unlock(&ctx->lock);
1021         }
1022
1023         if (run_queue)
1024                 blk_mq_run_hw_queue(hctx, async);
1025
1026         blk_mq_put_ctx(current_ctx);
1027 }
1028
1029 static void blk_mq_insert_requests(struct request_queue *q,
1030                                      struct blk_mq_ctx *ctx,
1031                                      struct list_head *list,
1032                                      int depth,
1033                                      bool from_schedule)
1034
1035 {
1036         struct blk_mq_hw_ctx *hctx;
1037         struct blk_mq_ctx *current_ctx;
1038
1039         trace_block_unplug(q, depth, !from_schedule);
1040
1041         current_ctx = blk_mq_get_ctx(q);
1042
1043         if (!cpu_online(ctx->cpu))
1044                 ctx = current_ctx;
1045         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1046
1047         /*
1048          * preemption doesn't flush plug list, so it's possible ctx->cpu is
1049          * offline now
1050          */
1051         spin_lock(&ctx->lock);
1052         while (!list_empty(list)) {
1053                 struct request *rq;
1054
1055                 rq = list_first_entry(list, struct request, queuelist);
1056                 list_del_init(&rq->queuelist);
1057                 rq->mq_ctx = ctx;
1058                 __blk_mq_insert_request(hctx, rq, false);
1059         }
1060         spin_unlock(&ctx->lock);
1061
1062         blk_mq_run_hw_queue(hctx, from_schedule);
1063         blk_mq_put_ctx(current_ctx);
1064 }
1065
1066 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1067 {
1068         struct request *rqa = container_of(a, struct request, queuelist);
1069         struct request *rqb = container_of(b, struct request, queuelist);
1070
1071         return !(rqa->mq_ctx < rqb->mq_ctx ||
1072                  (rqa->mq_ctx == rqb->mq_ctx &&
1073                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1074 }
1075
1076 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1077 {
1078         struct blk_mq_ctx *this_ctx;
1079         struct request_queue *this_q;
1080         struct request *rq;
1081         LIST_HEAD(list);
1082         LIST_HEAD(ctx_list);
1083         unsigned int depth;
1084
1085         list_splice_init(&plug->mq_list, &list);
1086
1087         list_sort(NULL, &list, plug_ctx_cmp);
1088
1089         this_q = NULL;
1090         this_ctx = NULL;
1091         depth = 0;
1092
1093         while (!list_empty(&list)) {
1094                 rq = list_entry_rq(list.next);
1095                 list_del_init(&rq->queuelist);
1096                 BUG_ON(!rq->q);
1097                 if (rq->mq_ctx != this_ctx) {
1098                         if (this_ctx) {
1099                                 blk_mq_insert_requests(this_q, this_ctx,
1100                                                         &ctx_list, depth,
1101                                                         from_schedule);
1102                         }
1103
1104                         this_ctx = rq->mq_ctx;
1105                         this_q = rq->q;
1106                         depth = 0;
1107                 }
1108
1109                 depth++;
1110                 list_add_tail(&rq->queuelist, &ctx_list);
1111         }
1112
1113         /*
1114          * If 'this_ctx' is set, we know we have entries to complete
1115          * on 'ctx_list'. Do those.
1116          */
1117         if (this_ctx) {
1118                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1119                                        from_schedule);
1120         }
1121 }
1122
1123 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1124 {
1125         init_request_from_bio(rq, bio);
1126         blk_account_io_start(rq, 1);
1127 }
1128
1129 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1130                                          struct blk_mq_ctx *ctx,
1131                                          struct request *rq, struct bio *bio)
1132 {
1133         struct request_queue *q = hctx->queue;
1134
1135         if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
1136                 blk_mq_bio_to_request(rq, bio);
1137                 spin_lock(&ctx->lock);
1138 insert_rq:
1139                 __blk_mq_insert_request(hctx, rq, false);
1140                 spin_unlock(&ctx->lock);
1141                 return false;
1142         } else {
1143                 spin_lock(&ctx->lock);
1144                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1145                         blk_mq_bio_to_request(rq, bio);
1146                         goto insert_rq;
1147                 }
1148
1149                 spin_unlock(&ctx->lock);
1150                 __blk_mq_free_request(hctx, ctx, rq);
1151                 return true;
1152         }
1153 }
1154
1155 struct blk_map_ctx {
1156         struct blk_mq_hw_ctx *hctx;
1157         struct blk_mq_ctx *ctx;
1158 };
1159
1160 static struct request *blk_mq_map_request(struct request_queue *q,
1161                                           struct bio *bio,
1162                                           struct blk_map_ctx *data)
1163 {
1164         struct blk_mq_hw_ctx *hctx;
1165         struct blk_mq_ctx *ctx;
1166         struct request *rq;
1167         int rw = bio_data_dir(bio);
1168
1169         if (unlikely(blk_mq_queue_enter(q))) {
1170                 bio_endio(bio, -EIO);
1171                 return NULL;
1172         }
1173
1174         ctx = blk_mq_get_ctx(q);
1175         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1176
1177         if (rw_is_sync(bio->bi_rw))
1178                 rw |= REQ_SYNC;
1179
1180         trace_block_getrq(q, bio, rw);
1181         rq = __blk_mq_alloc_request(q, hctx, ctx, rw, GFP_ATOMIC, false);
1182         if (unlikely(!rq)) {
1183                 blk_mq_put_ctx(ctx);
1184                 trace_block_sleeprq(q, bio, rw);
1185                 rq = blk_mq_alloc_request_pinned(q, rw, __GFP_WAIT|GFP_ATOMIC,
1186                                                         false);
1187                 ctx = rq->mq_ctx;
1188                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1189         }
1190
1191         hctx->queued++;
1192         data->hctx = hctx;
1193         data->ctx = ctx;
1194         return rq;
1195 }
1196
1197 /*
1198  * Multiple hardware queue variant. This will not use per-process plugs,
1199  * but will attempt to bypass the hctx queueing if we can go straight to
1200  * hardware for SYNC IO.
1201  */
1202 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1203 {
1204         const int is_sync = rw_is_sync(bio->bi_rw);
1205         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1206         struct blk_map_ctx data;
1207         struct request *rq;
1208
1209         blk_queue_bounce(q, &bio);
1210
1211         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1212                 bio_endio(bio, -EIO);
1213                 return;
1214         }
1215
1216         rq = blk_mq_map_request(q, bio, &data);
1217         if (unlikely(!rq))
1218                 return;
1219
1220         if (unlikely(is_flush_fua)) {
1221                 blk_mq_bio_to_request(rq, bio);
1222                 blk_insert_flush(rq);
1223                 goto run_queue;
1224         }
1225
1226         if (is_sync) {
1227                 int ret;
1228
1229                 blk_mq_bio_to_request(rq, bio);
1230                 blk_mq_start_request(rq, true);
1231
1232                 /*
1233                  * For OK queue, we are done. For error, kill it. Any other
1234                  * error (busy), just add it to our list as we previously
1235                  * would have done
1236                  */
1237                 ret = q->mq_ops->queue_rq(data.hctx, rq);
1238                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1239                         goto done;
1240                 else {
1241                         __blk_mq_requeue_request(rq);
1242
1243                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1244                                 rq->errors = -EIO;
1245                                 blk_mq_end_io(rq, rq->errors);
1246                                 goto done;
1247                         }
1248                 }
1249         }
1250
1251         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1252                 /*
1253                  * For a SYNC request, send it to the hardware immediately. For
1254                  * an ASYNC request, just ensure that we run it later on. The
1255                  * latter allows for merging opportunities and more efficient
1256                  * dispatching.
1257                  */
1258 run_queue:
1259                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1260         }
1261 done:
1262         blk_mq_put_ctx(data.ctx);
1263 }
1264
1265 /*
1266  * Single hardware queue variant. This will attempt to use any per-process
1267  * plug for merging and IO deferral.
1268  */
1269 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1270 {
1271         const int is_sync = rw_is_sync(bio->bi_rw);
1272         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1273         unsigned int use_plug, request_count = 0;
1274         struct blk_map_ctx data;
1275         struct request *rq;
1276
1277         /*
1278          * If we have multiple hardware queues, just go directly to
1279          * one of those for sync IO.
1280          */
1281         use_plug = !is_flush_fua && !is_sync;
1282
1283         blk_queue_bounce(q, &bio);
1284
1285         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1286                 bio_endio(bio, -EIO);
1287                 return;
1288         }
1289
1290         if (use_plug && !blk_queue_nomerges(q) &&
1291             blk_attempt_plug_merge(q, bio, &request_count))
1292                 return;
1293
1294         rq = blk_mq_map_request(q, bio, &data);
1295
1296         if (unlikely(is_flush_fua)) {
1297                 blk_mq_bio_to_request(rq, bio);
1298                 blk_insert_flush(rq);
1299                 goto run_queue;
1300         }
1301
1302         /*
1303          * A task plug currently exists. Since this is completely lockless,
1304          * utilize that to temporarily store requests until the task is
1305          * either done or scheduled away.
1306          */
1307         if (use_plug) {
1308                 struct blk_plug *plug = current->plug;
1309
1310                 if (plug) {
1311                         blk_mq_bio_to_request(rq, bio);
1312                         if (list_empty(&plug->mq_list))
1313                                 trace_block_plug(q);
1314                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1315                                 blk_flush_plug_list(plug, false);
1316                                 trace_block_plug(q);
1317                         }
1318                         list_add_tail(&rq->queuelist, &plug->mq_list);
1319                         blk_mq_put_ctx(data.ctx);
1320                         return;
1321                 }
1322         }
1323
1324         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1325                 /*
1326                  * For a SYNC request, send it to the hardware immediately. For
1327                  * an ASYNC request, just ensure that we run it later on. The
1328                  * latter allows for merging opportunities and more efficient
1329                  * dispatching.
1330                  */
1331 run_queue:
1332                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1333         }
1334
1335         blk_mq_put_ctx(data.ctx);
1336 }
1337
1338 /*
1339  * Default mapping to a software queue, since we use one per CPU.
1340  */
1341 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1342 {
1343         return q->queue_hw_ctx[q->mq_map[cpu]];
1344 }
1345 EXPORT_SYMBOL(blk_mq_map_queue);
1346
1347 struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *set,
1348                                                    unsigned int hctx_index,
1349                                                    int node)
1350 {
1351         return kzalloc_node(sizeof(struct blk_mq_hw_ctx), GFP_KERNEL, node);
1352 }
1353 EXPORT_SYMBOL(blk_mq_alloc_single_hw_queue);
1354
1355 void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *hctx,
1356                                  unsigned int hctx_index)
1357 {
1358         kfree(hctx);
1359 }
1360 EXPORT_SYMBOL(blk_mq_free_single_hw_queue);
1361
1362 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1363                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1364 {
1365         struct page *page;
1366
1367         if (tags->rqs && set->ops->exit_request) {
1368                 int i;
1369
1370                 for (i = 0; i < tags->nr_tags; i++) {
1371                         if (!tags->rqs[i])
1372                                 continue;
1373                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1374                                                 hctx_idx, i);
1375                 }
1376         }
1377
1378         while (!list_empty(&tags->page_list)) {
1379                 page = list_first_entry(&tags->page_list, struct page, lru);
1380                 list_del_init(&page->lru);
1381                 __free_pages(page, page->private);
1382         }
1383
1384         kfree(tags->rqs);
1385
1386         blk_mq_free_tags(tags);
1387 }
1388
1389 static size_t order_to_size(unsigned int order)
1390 {
1391         return (size_t)PAGE_SIZE << order;
1392 }
1393
1394 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1395                 unsigned int hctx_idx)
1396 {
1397         struct blk_mq_tags *tags;
1398         unsigned int i, j, entries_per_page, max_order = 4;
1399         size_t rq_size, left;
1400
1401         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1402                                 set->numa_node);
1403         if (!tags)
1404                 return NULL;
1405
1406         INIT_LIST_HEAD(&tags->page_list);
1407
1408         tags->rqs = kmalloc_node(set->queue_depth * sizeof(struct request *),
1409                                         GFP_KERNEL, set->numa_node);
1410         if (!tags->rqs) {
1411                 blk_mq_free_tags(tags);
1412                 return NULL;
1413         }
1414
1415         /*
1416          * rq_size is the size of the request plus driver payload, rounded
1417          * to the cacheline size
1418          */
1419         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1420                                 cache_line_size());
1421         left = rq_size * set->queue_depth;
1422
1423         for (i = 0; i < set->queue_depth; ) {
1424                 int this_order = max_order;
1425                 struct page *page;
1426                 int to_do;
1427                 void *p;
1428
1429                 while (left < order_to_size(this_order - 1) && this_order)
1430                         this_order--;
1431
1432                 do {
1433                         page = alloc_pages_node(set->numa_node, GFP_KERNEL,
1434                                                 this_order);
1435                         if (page)
1436                                 break;
1437                         if (!this_order--)
1438                                 break;
1439                         if (order_to_size(this_order) < rq_size)
1440                                 break;
1441                 } while (1);
1442
1443                 if (!page)
1444                         goto fail;
1445
1446                 page->private = this_order;
1447                 list_add_tail(&page->lru, &tags->page_list);
1448
1449                 p = page_address(page);
1450                 entries_per_page = order_to_size(this_order) / rq_size;
1451                 to_do = min(entries_per_page, set->queue_depth - i);
1452                 left -= to_do * rq_size;
1453                 for (j = 0; j < to_do; j++) {
1454                         tags->rqs[i] = p;
1455                         if (set->ops->init_request) {
1456                                 if (set->ops->init_request(set->driver_data,
1457                                                 tags->rqs[i], hctx_idx, i,
1458                                                 set->numa_node))
1459                                         goto fail;
1460                         }
1461
1462                         p += rq_size;
1463                         i++;
1464                 }
1465         }
1466
1467         return tags;
1468
1469 fail:
1470         pr_warn("%s: failed to allocate requests\n", __func__);
1471         blk_mq_free_rq_map(set, tags, hctx_idx);
1472         return NULL;
1473 }
1474
1475 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1476 {
1477         kfree(bitmap->map);
1478 }
1479
1480 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1481 {
1482         unsigned int bpw = 8, total, num_maps, i;
1483
1484         bitmap->bits_per_word = bpw;
1485
1486         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1487         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1488                                         GFP_KERNEL, node);
1489         if (!bitmap->map)
1490                 return -ENOMEM;
1491
1492         bitmap->map_size = num_maps;
1493
1494         total = nr_cpu_ids;
1495         for (i = 0; i < num_maps; i++) {
1496                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1497                 total -= bitmap->map[i].depth;
1498         }
1499
1500         return 0;
1501 }
1502
1503 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1504 {
1505         struct request_queue *q = hctx->queue;
1506         struct blk_mq_ctx *ctx;
1507         LIST_HEAD(tmp);
1508
1509         /*
1510          * Move ctx entries to new CPU, if this one is going away.
1511          */
1512         ctx = __blk_mq_get_ctx(q, cpu);
1513
1514         spin_lock(&ctx->lock);
1515         if (!list_empty(&ctx->rq_list)) {
1516                 list_splice_init(&ctx->rq_list, &tmp);
1517                 blk_mq_hctx_clear_pending(hctx, ctx);
1518         }
1519         spin_unlock(&ctx->lock);
1520
1521         if (list_empty(&tmp))
1522                 return NOTIFY_OK;
1523
1524         ctx = blk_mq_get_ctx(q);
1525         spin_lock(&ctx->lock);
1526
1527         while (!list_empty(&tmp)) {
1528                 struct request *rq;
1529
1530                 rq = list_first_entry(&tmp, struct request, queuelist);
1531                 rq->mq_ctx = ctx;
1532                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1533         }
1534
1535         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1536         blk_mq_hctx_mark_pending(hctx, ctx);
1537
1538         spin_unlock(&ctx->lock);
1539
1540         blk_mq_run_hw_queue(hctx, true);
1541         blk_mq_put_ctx(ctx);
1542         return NOTIFY_OK;
1543 }
1544
1545 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1546 {
1547         struct request_queue *q = hctx->queue;
1548         struct blk_mq_tag_set *set = q->tag_set;
1549
1550         if (set->tags[hctx->queue_num])
1551                 return NOTIFY_OK;
1552
1553         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1554         if (!set->tags[hctx->queue_num])
1555                 return NOTIFY_STOP;
1556
1557         hctx->tags = set->tags[hctx->queue_num];
1558         return NOTIFY_OK;
1559 }
1560
1561 static int blk_mq_hctx_notify(void *data, unsigned long action,
1562                               unsigned int cpu)
1563 {
1564         struct blk_mq_hw_ctx *hctx = data;
1565
1566         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1567                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1568         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1569                 return blk_mq_hctx_cpu_online(hctx, cpu);
1570
1571         return NOTIFY_OK;
1572 }
1573
1574 static void blk_mq_exit_hw_queues(struct request_queue *q,
1575                 struct blk_mq_tag_set *set, int nr_queue)
1576 {
1577         struct blk_mq_hw_ctx *hctx;
1578         unsigned int i;
1579
1580         queue_for_each_hw_ctx(q, hctx, i) {
1581                 if (i == nr_queue)
1582                         break;
1583
1584                 if (set->ops->exit_hctx)
1585                         set->ops->exit_hctx(hctx, i);
1586
1587                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1588                 kfree(hctx->ctxs);
1589                 blk_mq_free_bitmap(&hctx->ctx_map);
1590         }
1591
1592 }
1593
1594 static void blk_mq_free_hw_queues(struct request_queue *q,
1595                 struct blk_mq_tag_set *set)
1596 {
1597         struct blk_mq_hw_ctx *hctx;
1598         unsigned int i;
1599
1600         queue_for_each_hw_ctx(q, hctx, i) {
1601                 free_cpumask_var(hctx->cpumask);
1602                 set->ops->free_hctx(hctx, i);
1603         }
1604 }
1605
1606 static int blk_mq_init_hw_queues(struct request_queue *q,
1607                 struct blk_mq_tag_set *set)
1608 {
1609         struct blk_mq_hw_ctx *hctx;
1610         unsigned int i;
1611
1612         /*
1613          * Initialize hardware queues
1614          */
1615         queue_for_each_hw_ctx(q, hctx, i) {
1616                 int node;
1617
1618                 node = hctx->numa_node;
1619                 if (node == NUMA_NO_NODE)
1620                         node = hctx->numa_node = set->numa_node;
1621
1622                 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1623                 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1624                 spin_lock_init(&hctx->lock);
1625                 INIT_LIST_HEAD(&hctx->dispatch);
1626                 hctx->queue = q;
1627                 hctx->queue_num = i;
1628                 hctx->flags = set->flags;
1629                 hctx->cmd_size = set->cmd_size;
1630
1631                 blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1632                                                 blk_mq_hctx_notify, hctx);
1633                 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1634
1635                 hctx->tags = set->tags[i];
1636
1637                 /*
1638                  * Allocate space for all possible cpus to avoid allocation in
1639                  * runtime
1640                  */
1641                 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1642                                                 GFP_KERNEL, node);
1643                 if (!hctx->ctxs)
1644                         break;
1645
1646                 if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1647                         break;
1648
1649                 hctx->nr_ctx = 0;
1650
1651                 if (set->ops->init_hctx &&
1652                     set->ops->init_hctx(hctx, set->driver_data, i))
1653                         break;
1654         }
1655
1656         if (i == q->nr_hw_queues)
1657                 return 0;
1658
1659         /*
1660          * Init failed
1661          */
1662         blk_mq_exit_hw_queues(q, set, i);
1663
1664         return 1;
1665 }
1666
1667 static void blk_mq_init_cpu_queues(struct request_queue *q,
1668                                    unsigned int nr_hw_queues)
1669 {
1670         unsigned int i;
1671
1672         for_each_possible_cpu(i) {
1673                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1674                 struct blk_mq_hw_ctx *hctx;
1675
1676                 memset(__ctx, 0, sizeof(*__ctx));
1677                 __ctx->cpu = i;
1678                 spin_lock_init(&__ctx->lock);
1679                 INIT_LIST_HEAD(&__ctx->rq_list);
1680                 __ctx->queue = q;
1681
1682                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1683                 if (!cpu_online(i))
1684                         continue;
1685
1686                 hctx = q->mq_ops->map_queue(q, i);
1687                 cpumask_set_cpu(i, hctx->cpumask);
1688                 hctx->nr_ctx++;
1689
1690                 /*
1691                  * Set local node, IFF we have more than one hw queue. If
1692                  * not, we remain on the home node of the device
1693                  */
1694                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1695                         hctx->numa_node = cpu_to_node(i);
1696         }
1697 }
1698
1699 static void blk_mq_map_swqueue(struct request_queue *q)
1700 {
1701         unsigned int i;
1702         struct blk_mq_hw_ctx *hctx;
1703         struct blk_mq_ctx *ctx;
1704
1705         queue_for_each_hw_ctx(q, hctx, i) {
1706                 cpumask_clear(hctx->cpumask);
1707                 hctx->nr_ctx = 0;
1708         }
1709
1710         /*
1711          * Map software to hardware queues
1712          */
1713         queue_for_each_ctx(q, ctx, i) {
1714                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1715                 if (!cpu_online(i))
1716                         continue;
1717
1718                 hctx = q->mq_ops->map_queue(q, i);
1719                 cpumask_set_cpu(i, hctx->cpumask);
1720                 ctx->index_hw = hctx->nr_ctx;
1721                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1722         }
1723
1724         queue_for_each_hw_ctx(q, hctx, i) {
1725                 /*
1726                  * If not software queues are mapped to this hardware queue,
1727                  * disable it and free the request entries
1728                  */
1729                 if (!hctx->nr_ctx) {
1730                         struct blk_mq_tag_set *set = q->tag_set;
1731
1732                         if (set->tags[i]) {
1733                                 blk_mq_free_rq_map(set, set->tags[i], i);
1734                                 set->tags[i] = NULL;
1735                                 hctx->tags = NULL;
1736                         }
1737                         continue;
1738                 }
1739
1740                 /*
1741                  * Initialize batch roundrobin counts
1742                  */
1743                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1744                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1745         }
1746 }
1747
1748 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1749 {
1750         struct blk_mq_hw_ctx *hctx;
1751         struct request_queue *q;
1752         bool shared;
1753         int i;
1754
1755         if (set->tag_list.next == set->tag_list.prev)
1756                 shared = false;
1757         else
1758                 shared = true;
1759
1760         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1761                 blk_mq_freeze_queue(q);
1762
1763                 queue_for_each_hw_ctx(q, hctx, i) {
1764                         if (shared)
1765                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1766                         else
1767                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1768                 }
1769                 blk_mq_unfreeze_queue(q);
1770         }
1771 }
1772
1773 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1774 {
1775         struct blk_mq_tag_set *set = q->tag_set;
1776
1777         blk_mq_freeze_queue(q);
1778
1779         mutex_lock(&set->tag_list_lock);
1780         list_del_init(&q->tag_set_list);
1781         blk_mq_update_tag_set_depth(set);
1782         mutex_unlock(&set->tag_list_lock);
1783
1784         blk_mq_unfreeze_queue(q);
1785 }
1786
1787 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1788                                      struct request_queue *q)
1789 {
1790         q->tag_set = set;
1791
1792         mutex_lock(&set->tag_list_lock);
1793         list_add_tail(&q->tag_set_list, &set->tag_list);
1794         blk_mq_update_tag_set_depth(set);
1795         mutex_unlock(&set->tag_list_lock);
1796 }
1797
1798 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1799 {
1800         struct blk_mq_hw_ctx **hctxs;
1801         struct blk_mq_ctx *ctx;
1802         struct request_queue *q;
1803         unsigned int *map;
1804         int i;
1805
1806         ctx = alloc_percpu(struct blk_mq_ctx);
1807         if (!ctx)
1808                 return ERR_PTR(-ENOMEM);
1809
1810         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1811                         set->numa_node);
1812
1813         if (!hctxs)
1814                 goto err_percpu;
1815
1816         map = blk_mq_make_queue_map(set);
1817         if (!map)
1818                 goto err_map;
1819
1820         for (i = 0; i < set->nr_hw_queues; i++) {
1821                 int node = blk_mq_hw_queue_to_node(map, i);
1822
1823                 hctxs[i] = set->ops->alloc_hctx(set, i, node);
1824                 if (!hctxs[i])
1825                         goto err_hctxs;
1826
1827                 if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL))
1828                         goto err_hctxs;
1829
1830                 atomic_set(&hctxs[i]->nr_active, 0);
1831                 hctxs[i]->numa_node = node;
1832                 hctxs[i]->queue_num = i;
1833         }
1834
1835         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1836         if (!q)
1837                 goto err_hctxs;
1838
1839         if (percpu_counter_init(&q->mq_usage_counter, 0))
1840                 goto err_map;
1841
1842         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1843         blk_queue_rq_timeout(q, 30000);
1844
1845         q->nr_queues = nr_cpu_ids;
1846         q->nr_hw_queues = set->nr_hw_queues;
1847         q->mq_map = map;
1848
1849         q->queue_ctx = ctx;
1850         q->queue_hw_ctx = hctxs;
1851
1852         q->mq_ops = set->ops;
1853         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1854
1855         q->sg_reserved_size = INT_MAX;
1856
1857         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1858         INIT_LIST_HEAD(&q->requeue_list);
1859         spin_lock_init(&q->requeue_lock);
1860
1861         if (q->nr_hw_queues > 1)
1862                 blk_queue_make_request(q, blk_mq_make_request);
1863         else
1864                 blk_queue_make_request(q, blk_sq_make_request);
1865
1866         blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
1867         if (set->timeout)
1868                 blk_queue_rq_timeout(q, set->timeout);
1869
1870         /*
1871          * Do this after blk_queue_make_request() overrides it...
1872          */
1873         q->nr_requests = set->queue_depth;
1874
1875         if (set->ops->complete)
1876                 blk_queue_softirq_done(q, set->ops->complete);
1877
1878         blk_mq_init_flush(q);
1879         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1880
1881         q->flush_rq = kzalloc(round_up(sizeof(struct request) +
1882                                 set->cmd_size, cache_line_size()),
1883                                 GFP_KERNEL);
1884         if (!q->flush_rq)
1885                 goto err_hw;
1886
1887         if (blk_mq_init_hw_queues(q, set))
1888                 goto err_flush_rq;
1889
1890         mutex_lock(&all_q_mutex);
1891         list_add_tail(&q->all_q_node, &all_q_list);
1892         mutex_unlock(&all_q_mutex);
1893
1894         blk_mq_add_queue_tag_set(set, q);
1895
1896         blk_mq_map_swqueue(q);
1897
1898         return q;
1899
1900 err_flush_rq:
1901         kfree(q->flush_rq);
1902 err_hw:
1903         blk_cleanup_queue(q);
1904 err_hctxs:
1905         kfree(map);
1906         for (i = 0; i < set->nr_hw_queues; i++) {
1907                 if (!hctxs[i])
1908                         break;
1909                 free_cpumask_var(hctxs[i]->cpumask);
1910                 set->ops->free_hctx(hctxs[i], i);
1911         }
1912 err_map:
1913         kfree(hctxs);
1914 err_percpu:
1915         free_percpu(ctx);
1916         return ERR_PTR(-ENOMEM);
1917 }
1918 EXPORT_SYMBOL(blk_mq_init_queue);
1919
1920 void blk_mq_free_queue(struct request_queue *q)
1921 {
1922         struct blk_mq_tag_set   *set = q->tag_set;
1923
1924         blk_mq_del_queue_tag_set(q);
1925
1926         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1927         blk_mq_free_hw_queues(q, set);
1928
1929         percpu_counter_destroy(&q->mq_usage_counter);
1930
1931         free_percpu(q->queue_ctx);
1932         kfree(q->queue_hw_ctx);
1933         kfree(q->mq_map);
1934
1935         q->queue_ctx = NULL;
1936         q->queue_hw_ctx = NULL;
1937         q->mq_map = NULL;
1938
1939         mutex_lock(&all_q_mutex);
1940         list_del_init(&q->all_q_node);
1941         mutex_unlock(&all_q_mutex);
1942 }
1943
1944 /* Basically redo blk_mq_init_queue with queue frozen */
1945 static void blk_mq_queue_reinit(struct request_queue *q)
1946 {
1947         blk_mq_freeze_queue(q);
1948
1949         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1950
1951         /*
1952          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1953          * we should change hctx numa_node according to new topology (this
1954          * involves free and re-allocate memory, worthy doing?)
1955          */
1956
1957         blk_mq_map_swqueue(q);
1958
1959         blk_mq_unfreeze_queue(q);
1960 }
1961
1962 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1963                                       unsigned long action, void *hcpu)
1964 {
1965         struct request_queue *q;
1966
1967         /*
1968          * Before new mappings are established, hotadded cpu might already
1969          * start handling requests. This doesn't break anything as we map
1970          * offline CPUs to first hardware queue. We will re-init the queue
1971          * below to get optimal settings.
1972          */
1973         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1974             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1975                 return NOTIFY_OK;
1976
1977         mutex_lock(&all_q_mutex);
1978         list_for_each_entry(q, &all_q_list, all_q_node)
1979                 blk_mq_queue_reinit(q);
1980         mutex_unlock(&all_q_mutex);
1981         return NOTIFY_OK;
1982 }
1983
1984 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
1985 {
1986         int i;
1987
1988         if (!set->nr_hw_queues)
1989                 return -EINVAL;
1990         if (!set->queue_depth || set->queue_depth > BLK_MQ_MAX_DEPTH)
1991                 return -EINVAL;
1992         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
1993                 return -EINVAL;
1994
1995         if (!set->nr_hw_queues ||
1996             !set->ops->queue_rq || !set->ops->map_queue ||
1997             !set->ops->alloc_hctx || !set->ops->free_hctx)
1998                 return -EINVAL;
1999
2000
2001         set->tags = kmalloc_node(set->nr_hw_queues *
2002                                  sizeof(struct blk_mq_tags *),
2003                                  GFP_KERNEL, set->numa_node);
2004         if (!set->tags)
2005                 goto out;
2006
2007         for (i = 0; i < set->nr_hw_queues; i++) {
2008                 set->tags[i] = blk_mq_init_rq_map(set, i);
2009                 if (!set->tags[i])
2010                         goto out_unwind;
2011         }
2012
2013         mutex_init(&set->tag_list_lock);
2014         INIT_LIST_HEAD(&set->tag_list);
2015
2016         return 0;
2017
2018 out_unwind:
2019         while (--i >= 0)
2020                 blk_mq_free_rq_map(set, set->tags[i], i);
2021 out:
2022         return -ENOMEM;
2023 }
2024 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2025
2026 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2027 {
2028         int i;
2029
2030         for (i = 0; i < set->nr_hw_queues; i++) {
2031                 if (set->tags[i])
2032                         blk_mq_free_rq_map(set, set->tags[i], i);
2033         }
2034
2035         kfree(set->tags);
2036 }
2037 EXPORT_SYMBOL(blk_mq_free_tag_set);
2038
2039 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2040 {
2041         struct blk_mq_tag_set *set = q->tag_set;
2042         struct blk_mq_hw_ctx *hctx;
2043         int i, ret;
2044
2045         if (!set || nr > set->queue_depth)
2046                 return -EINVAL;
2047
2048         ret = 0;
2049         queue_for_each_hw_ctx(q, hctx, i) {
2050                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2051                 if (ret)
2052                         break;
2053         }
2054
2055         if (!ret)
2056                 q->nr_requests = nr;
2057
2058         return ret;
2059 }
2060
2061 void blk_mq_disable_hotplug(void)
2062 {
2063         mutex_lock(&all_q_mutex);
2064 }
2065
2066 void blk_mq_enable_hotplug(void)
2067 {
2068         mutex_unlock(&all_q_mutex);
2069 }
2070
2071 static int __init blk_mq_init(void)
2072 {
2073         blk_mq_cpu_init();
2074
2075         /* Must be called after percpu_counter_hotcpu_callback() */
2076         hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
2077
2078         return 0;
2079 }
2080 subsys_initcall(blk_mq_init);