drivers/s390/block/scm_blk.c

   1 /*
   2  * Block driver for s390 storage class memory.
   3  *
   4  * Copyright IBM Corp. 2012
   5  * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
   6  */
   7
   8 #define KMSG_COMPONENT "scm_block"
   9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/interrupt.h>
  12 #include <linux/spinlock.h>
  13 #include <linux/mempool.h>
  14 #include <linux/module.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/blk-mq.h>
  17 #include <linux/genhd.h>
  18 #include <linux/slab.h>
  19 #include <linux/list.h>
  20 #include <asm/eadm.h>
  21 #include "scm_blk.h"
  22
  23 debug_info_t *scm_debug;
  24 static int scm_major;
  25 static mempool_t *aidaw_pool;
  26 static DEFINE_SPINLOCK(list_lock);
  27 static LIST_HEAD(inactive_requests);
  28 static unsigned int nr_requests = 64;
  29 static unsigned int nr_requests_per_io = 8;
  30 static atomic_t nr_devices = ATOMIC_INIT(0);
  31 module_param(nr_requests, uint, S_IRUGO);
  32 MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
  33
  34 module_param(nr_requests_per_io, uint, S_IRUGO);
  35 MODULE_PARM_DESC(nr_requests_per_io, "Number of requests per IO.");
  36
  37 MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
  38 MODULE_LICENSE("GPL");
  39 MODULE_ALIAS("scm:scmdev*");
  40
  41 static void __scm_free_rq(struct scm_request *scmrq)
  42 {
  43         struct aob_rq_header *aobrq = to_aobrq(scmrq);
  44
  45         free_page((unsigned long) scmrq->aob);
  46         kfree(scmrq->request);
  47         kfree(aobrq);
  48 }
  49
  50 static void scm_free_rqs(void)
  51 {
  52         struct list_head *iter, *safe;
  53         struct scm_request *scmrq;
  54
  55         spin_lock_irq(&list_lock);
  56         list_for_each_safe(iter, safe, &inactive_requests) {
  57                 scmrq = list_entry(iter, struct scm_request, list);
  58                 list_del(&scmrq->list);
  59                 __scm_free_rq(scmrq);
  60         }
  61         spin_unlock_irq(&list_lock);
  62
  63         mempool_destroy(aidaw_pool);
  64 }
  65
  66 static int __scm_alloc_rq(void)
  67 {
  68         struct aob_rq_header *aobrq;
  69         struct scm_request *scmrq;
  70
  71         aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
  72         if (!aobrq)
  73                 return -ENOMEM;
  74
  75         scmrq = (void *) aobrq->data;
  76         scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
  77         if (!scmrq->aob)
  78                 goto free;
  79
  80         scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),
  81                                  GFP_KERNEL);
  82         if (!scmrq->request)
  83                 goto free;
  84
  85         INIT_LIST_HEAD(&scmrq->list);
  86         spin_lock_irq(&list_lock);
  87         list_add(&scmrq->list, &inactive_requests);
  88         spin_unlock_irq(&list_lock);
  89
  90         return 0;
  91 free:
  92         __scm_free_rq(scmrq);
  93         return -ENOMEM;
  94 }
  95
  96 static int scm_alloc_rqs(unsigned int nrqs)
  97 {
  98         int ret = 0;
  99
 100         aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);
 101         if (!aidaw_pool)
 102                 return -ENOMEM;
 103
 104         while (nrqs-- && !ret)
 105                 ret = __scm_alloc_rq();
 106
 107         return ret;
 108 }
 109
 110 static struct scm_request *scm_request_fetch(void)
 111 {
 112         struct scm_request *scmrq = NULL;
 113
 114         spin_lock_irq(&list_lock);
 115         if (list_empty(&inactive_requests))
 116                 goto out;
 117         scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
 118         list_del(&scmrq->list);
 119 out:
 120         spin_unlock_irq(&list_lock);
 121         return scmrq;
 122 }
 123
 124 static void scm_request_done(struct scm_request *scmrq)
 125 {
 126         unsigned long flags;
 127         struct msb *msb;
 128         u64 aidaw;
 129         int i;
 130
 131         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 132                 msb = &scmrq->aob->msb[i];
 133                 aidaw = msb->data_addr;
 134
 135                 if ((msb->flags & MSB_FLAG_IDA) && aidaw &&
 136                     IS_ALIGNED(aidaw, PAGE_SIZE))
 137                         mempool_free(virt_to_page(aidaw), aidaw_pool);
 138         }
 139
 140         spin_lock_irqsave(&list_lock, flags);
 141         list_add(&scmrq->list, &inactive_requests);
 142         spin_unlock_irqrestore(&list_lock, flags);
 143 }
 144
 145 static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
 146 {
 147         return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
 148 }
 149
 150 static inline struct aidaw *scm_aidaw_alloc(void)
 151 {
 152         struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);
 153
 154         return page ? page_address(page) : NULL;
 155 }
 156
 157 static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)
 158 {
 159         unsigned long _aidaw = (unsigned long) aidaw;
 160         unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;
 161
 162         return (bytes / sizeof(*aidaw)) * PAGE_SIZE;
 163 }
 164
 165 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)
 166 {
 167         struct aidaw *aidaw;
 168
 169         if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)
 170                 return scmrq->next_aidaw;
 171
 172         aidaw = scm_aidaw_alloc();
 173         if (aidaw)
 174                 memset(aidaw, 0, PAGE_SIZE);
 175         return aidaw;
 176 }
 177
 178 static int scm_request_prepare(struct scm_request *scmrq)
 179 {
 180         struct scm_blk_dev *bdev = scmrq->bdev;
 181         struct scm_device *scmdev = bdev->gendisk->private_data;
 182         int pos = scmrq->aob->request.msb_count;
 183         struct msb *msb = &scmrq->aob->msb[pos];
 184         struct request *req = scmrq->request[pos];
 185         struct req_iterator iter;
 186         struct aidaw *aidaw;
 187         struct bio_vec bv;
 188
 189         aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
 190         if (!aidaw)
 191                 return -ENOMEM;
 192
 193         msb->bs = MSB_BS_4K;
 194         scmrq->aob->request.msb_count++;
 195         msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);
 196         msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;
 197         msb->flags |= MSB_FLAG_IDA;
 198         msb->data_addr = (u64) aidaw;
 199
 200         rq_for_each_segment(bv, req, iter) {
 201                 WARN_ON(bv.bv_offset);
 202                 msb->blk_count += bv.bv_len >> 12;
 203                 aidaw->data_addr = (u64) page_address(bv.bv_page);
 204                 aidaw++;
 205         }
 206
 207         scmrq->next_aidaw = aidaw;
 208         return 0;
 209 }
 210
 211 static inline void scm_request_set(struct scm_request *scmrq,
 212                                    struct request *req)
 213 {
 214         scmrq->request[scmrq->aob->request.msb_count] = req;
 215 }
 216
 217 static inline void scm_request_init(struct scm_blk_dev *bdev,
 218                                     struct scm_request *scmrq)
 219 {
 220         struct aob_rq_header *aobrq = to_aobrq(scmrq);
 221         struct aob *aob = scmrq->aob;
 222
 223         memset(scmrq->request, 0,
 224                nr_requests_per_io * sizeof(scmrq->request[0]));
 225         memset(aob, 0, sizeof(*aob));
 226         aobrq->scmdev = bdev->scmdev;
 227         aob->request.cmd_code = ARQB_CMD_MOVE;
 228         aob->request.data = (u64) aobrq;
 229         scmrq->bdev = bdev;
 230         scmrq->retries = 4;
 231         scmrq->error = BLK_STS_OK;
 232         /* We don't use all msbs - place aidaws at the end of the aob page. */
 233         scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];
 234 }
 235
 236 static void scm_request_requeue(struct scm_request *scmrq)
 237 {
 238         struct scm_blk_dev *bdev = scmrq->bdev;
 239         int i;
 240
 241         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
 242                 blk_mq_requeue_request(scmrq->request[i], false);
 243
 244         atomic_dec(&bdev->queued_reqs);
 245         scm_request_done(scmrq);
 246         blk_mq_kick_requeue_list(bdev->rq);
 247 }
 248
 249 static void scm_request_finish(struct scm_request *scmrq)
 250 {
 251         struct scm_blk_dev *bdev = scmrq->bdev;
 252         int i;
 253
 254         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 255                 if (scmrq->error)
 256                         blk_mq_end_request(scmrq->request[i], scmrq->error);
 257                 else
 258                         blk_mq_complete_request(scmrq->request[i]);
 259         }
 260
 261         atomic_dec(&bdev->queued_reqs);
 262         scm_request_done(scmrq);
 263 }
 264
 265 static void scm_request_start(struct scm_request *scmrq)
 266 {
 267         struct scm_blk_dev *bdev = scmrq->bdev;
 268
 269         atomic_inc(&bdev->queued_reqs);
 270         if (eadm_start_aob(scmrq->aob)) {
 271                 SCM_LOG(5, "no subchannel");
 272                 scm_request_requeue(scmrq);
 273         }
 274 }
 275
 276 struct scm_queue {
 277         struct scm_request *scmrq;
 278         spinlock_t lock;
 279 };
 280
 281 static blk_status_t scm_blk_request(struct blk_mq_hw_ctx *hctx,
 282                            const struct blk_mq_queue_data *qd)
 283 {
 284         struct scm_device *scmdev = hctx->queue->queuedata;
 285         struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
 286         struct scm_queue *sq = hctx->driver_data;
 287         struct request *req = qd->rq;
 288         struct scm_request *scmrq;
 289
 290         spin_lock(&sq->lock);
 291         if (!scm_permit_request(bdev, req)) {
 292                 spin_unlock(&sq->lock);
 293                 return BLK_STS_RESOURCE;
 294         }
 295
 296         scmrq = sq->scmrq;
 297         if (!scmrq) {
 298                 scmrq = scm_request_fetch();
 299                 if (!scmrq) {
 300                         SCM_LOG(5, "no request");
 301                         spin_unlock(&sq->lock);
 302                         return BLK_STS_RESOURCE;
 303                 }
 304                 scm_request_init(bdev, scmrq);
 305                 sq->scmrq = scmrq;
 306         }
 307         scm_request_set(scmrq, req);
 308
 309         if (scm_request_prepare(scmrq)) {
 310                 SCM_LOG(5, "aidaw alloc failed");
 311                 scm_request_set(scmrq, NULL);
 312
 313                 if (scmrq->aob->request.msb_count)
 314                         scm_request_start(scmrq);
 315
 316                 sq->scmrq = NULL;
 317                 spin_unlock(&sq->lock);
 318                 return BLK_STS_RESOURCE;
 319         }
 320         blk_mq_start_request(req);
 321
 322         if (qd->last || scmrq->aob->request.msb_count == nr_requests_per_io) {
 323                 scm_request_start(scmrq);
 324                 sq->scmrq = NULL;
 325         }
 326         spin_unlock(&sq->lock);
 327         return BLK_STS_OK;
 328 }
 329
 330 static int scm_blk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 331                              unsigned int idx)
 332 {
 333         struct scm_queue *qd = kzalloc(sizeof(*qd), GFP_KERNEL);
 334
 335         if (!qd)
 336                 return -ENOMEM;
 337
 338         spin_lock_init(&qd->lock);
 339         hctx->driver_data = qd;
 340
 341         return 0;
 342 }
 343
 344 static void scm_blk_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)
 345 {
 346         struct scm_queue *qd = hctx->driver_data;
 347
 348         WARN_ON(qd->scmrq);
 349         kfree(hctx->driver_data);
 350         hctx->driver_data = NULL;
 351 }
 352
 353 static void __scmrq_log_error(struct scm_request *scmrq)
 354 {
 355         struct aob *aob = scmrq->aob;
 356
 357         if (scmrq->error == BLK_STS_TIMEOUT)
 358                 SCM_LOG(1, "Request timeout");
 359         else {
 360                 SCM_LOG(1, "Request error");
 361                 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
 362         }
 363         if (scmrq->retries)
 364                 SCM_LOG(1, "Retry request");
 365         else
 366                 pr_err("An I/O operation to SCM failed with rc=%d\n",
 367                        scmrq->error);
 368 }
 369
 370 static void scm_blk_handle_error(struct scm_request *scmrq)
 371 {
 372         struct scm_blk_dev *bdev = scmrq->bdev;
 373         unsigned long flags;
 374
 375         if (scmrq->error != BLK_STS_IOERR)
 376                 goto restart;
 377
 378         /* For -EIO the response block is valid. */
 379         switch (scmrq->aob->response.eqc) {
 380         case EQC_WR_PROHIBIT:
 381                 spin_lock_irqsave(&bdev->lock, flags);
 382                 if (bdev->state != SCM_WR_PROHIBIT)
 383                         pr_info("%lx: Write access to the SCM increment is suspended\n",
 384                                 (unsigned long) bdev->scmdev->address);
 385                 bdev->state = SCM_WR_PROHIBIT;
 386                 spin_unlock_irqrestore(&bdev->lock, flags);
 387                 goto requeue;
 388         default:
 389                 break;
 390         }
 391
 392 restart:
 393         if (!eadm_start_aob(scmrq->aob))
 394                 return;
 395
 396 requeue:
 397         scm_request_requeue(scmrq);
 398 }
 399
 400 void scm_blk_irq(struct scm_device *scmdev, void *data, blk_status_t error)
 401 {
 402         struct scm_request *scmrq = data;
 403
 404         scmrq->error = error;
 405         if (error) {
 406                 __scmrq_log_error(scmrq);
 407                 if (scmrq->retries-- > 0) {
 408                         scm_blk_handle_error(scmrq);
 409                         return;
 410                 }
 411         }
 412
 413         scm_request_finish(scmrq);
 414 }
 415
 416 static void scm_blk_request_done(struct request *req)
 417 {
 418         blk_mq_end_request(req, 0);
 419 }
 420
 421 static const struct block_device_operations scm_blk_devops = {
 422         .owner = THIS_MODULE,
 423 };
 424
 425 static const struct blk_mq_ops scm_mq_ops = {
 426         .queue_rq = scm_blk_request,
 427         .complete = scm_blk_request_done,
 428         .init_hctx = scm_blk_init_hctx,
 429         .exit_hctx = scm_blk_exit_hctx,
 430 };
 431
 432 int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
 433 {
 434         unsigned int devindex, nr_max_blk;
 435         struct request_queue *rq;
 436         int len, ret;
 437
 438         devindex = atomic_inc_return(&nr_devices) - 1;
 439         /* scma..scmz + scmaa..scmzz */
 440         if (devindex > 701) {
 441                 ret = -ENODEV;
 442                 goto out;
 443         }
 444
 445         bdev->scmdev = scmdev;
 446         bdev->state = SCM_OPER;
 447         spin_lock_init(&bdev->lock);
 448         atomic_set(&bdev->queued_reqs, 0);
 449
 450         bdev->tag_set.ops = &scm_mq_ops;
 451         bdev->tag_set.nr_hw_queues = nr_requests;
 452         bdev->tag_set.queue_depth = nr_requests_per_io * nr_requests;
 453         bdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
 454
 455         ret = blk_mq_alloc_tag_set(&bdev->tag_set);
 456         if (ret)
 457                 goto out;
 458
 459         rq = blk_mq_init_queue(&bdev->tag_set);
 460         if (IS_ERR(rq)) {
 461                 ret = PTR_ERR(rq);
 462                 goto out_tag;
 463         }
 464         bdev->rq = rq;
 465         nr_max_blk = min(scmdev->nr_max_block,
 466                          (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
 467
 468         blk_queue_logical_block_size(rq, 1 << 12);
 469         blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
 470         blk_queue_max_segments(rq, nr_max_blk);
 471         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
 472         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
 473
 474         bdev->gendisk = alloc_disk(SCM_NR_PARTS);
 475         if (!bdev->gendisk) {
 476                 ret = -ENOMEM;
 477                 goto out_queue;
 478         }
 479         rq->queuedata = scmdev;
 480         bdev->gendisk->private_data = scmdev;
 481         bdev->gendisk->fops = &scm_blk_devops;
 482         bdev->gendisk->queue = rq;
 483         bdev->gendisk->major = scm_major;
 484         bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
 485
 486         len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
 487         if (devindex > 25) {
 488                 len += snprintf(bdev->gendisk->disk_name + len,
 489                                 DISK_NAME_LEN - len, "%c",
 490                                 'a' + (devindex / 26) - 1);
 491                 devindex = devindex % 26;
 492         }
 493         snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
 494                  'a' + devindex);
 495
 496         /* 512 byte sectors */
 497         set_capacity(bdev->gendisk, scmdev->size >> 9);
 498         device_add_disk(&scmdev->dev, bdev->gendisk);
 499         return 0;
 500
 501 out_queue:
 502         blk_cleanup_queue(rq);
 503 out_tag:
 504         blk_mq_free_tag_set(&bdev->tag_set);
 505 out:
 506         atomic_dec(&nr_devices);
 507         return ret;
 508 }
 509
 510 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
 511 {
 512         del_gendisk(bdev->gendisk);
 513         blk_cleanup_queue(bdev->gendisk->queue);
 514         blk_mq_free_tag_set(&bdev->tag_set);
 515         put_disk(bdev->gendisk);
 516 }
 517
 518 void scm_blk_set_available(struct scm_blk_dev *bdev)
 519 {
 520         unsigned long flags;
 521
 522         spin_lock_irqsave(&bdev->lock, flags);
 523         if (bdev->state == SCM_WR_PROHIBIT)
 524                 pr_info("%lx: Write access to the SCM increment is restored\n",
 525                         (unsigned long) bdev->scmdev->address);
 526         bdev->state = SCM_OPER;
 527         spin_unlock_irqrestore(&bdev->lock, flags);
 528 }
 529
 530 static bool __init scm_blk_params_valid(void)
 531 {
 532         if (!nr_requests_per_io || nr_requests_per_io > 64)
 533                 return false;
 534
 535         return true;
 536 }
 537
 538 static int __init scm_blk_init(void)
 539 {
 540         int ret = -EINVAL;
 541
 542         if (!scm_blk_params_valid())
 543                 goto out;
 544
 545         ret = register_blkdev(0, "scm");
 546         if (ret < 0)
 547                 goto out;
 548
 549         scm_major = ret;
 550         ret = scm_alloc_rqs(nr_requests);
 551         if (ret)
 552                 goto out_free;
 553
 554         scm_debug = debug_register("scm_log", 16, 1, 16);
 555         if (!scm_debug) {
 556                 ret = -ENOMEM;
 557                 goto out_free;
 558         }
 559
 560         debug_register_view(scm_debug, &debug_hex_ascii_view);
 561         debug_set_level(scm_debug, 2);
 562
 563         ret = scm_drv_init();
 564         if (ret)
 565                 goto out_dbf;
 566
 567         return ret;
 568
 569 out_dbf:
 570         debug_unregister(scm_debug);
 571 out_free:
 572         scm_free_rqs();
 573         unregister_blkdev(scm_major, "scm");
 574 out:
 575         return ret;
 576 }
 577 module_init(scm_blk_init);
 578
 579 static void __exit scm_blk_cleanup(void)
 580 {
 581         scm_drv_cleanup();
 582         debug_unregister(scm_debug);
 583         scm_free_rqs();
 584         unregister_blkdev(scm_major, "scm");
 585 }
 586 module_exit(scm_blk_cleanup);