2 * Block driver for s390 storage class memory.
4 * Copyright IBM Corp. 2012
5 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
8 #define KMSG_COMPONENT "scm_block"
9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11 #include <linux/interrupt.h>
12 #include <linux/spinlock.h>
13 #include <linux/module.h>
14 #include <linux/blkdev.h>
15 #include <linux/genhd.h>
16 #include <linux/slab.h>
17 #include <linux/list.h>
21 debug_info_t *scm_debug;
23 static DEFINE_SPINLOCK(list_lock);
24 static LIST_HEAD(inactive_requests);
25 static unsigned int nr_requests = 64;
26 static atomic_t nr_devices = ATOMIC_INIT(0);
27 module_param(nr_requests, uint, S_IRUGO);
28 MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
30 MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
31 MODULE_LICENSE("GPL");
32 MODULE_ALIAS("scm:scmdev*");
34 static void __scm_free_rq(struct scm_request *scmrq)
36 struct aob_rq_header *aobrq = to_aobrq(scmrq);
38 free_page((unsigned long) scmrq->aob);
39 free_page((unsigned long) scmrq->aidaw);
40 __scm_free_rq_cluster(scmrq);
44 static void scm_free_rqs(void)
46 struct list_head *iter, *safe;
47 struct scm_request *scmrq;
49 spin_lock_irq(&list_lock);
50 list_for_each_safe(iter, safe, &inactive_requests) {
51 scmrq = list_entry(iter, struct scm_request, list);
52 list_del(&scmrq->list);
55 spin_unlock_irq(&list_lock);
58 static int __scm_alloc_rq(void)
60 struct aob_rq_header *aobrq;
61 struct scm_request *scmrq;
63 aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
67 scmrq = (void *) aobrq->data;
68 scmrq->aidaw = (void *) get_zeroed_page(GFP_DMA);
69 scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
70 if (!scmrq->aob || !scmrq->aidaw) {
75 if (__scm_alloc_rq_cluster(scmrq)) {
80 INIT_LIST_HEAD(&scmrq->list);
81 spin_lock_irq(&list_lock);
82 list_add(&scmrq->list, &inactive_requests);
83 spin_unlock_irq(&list_lock);
88 static int scm_alloc_rqs(unsigned int nrqs)
92 while (nrqs-- && !ret)
93 ret = __scm_alloc_rq();
98 static struct scm_request *scm_request_fetch(void)
100 struct scm_request *scmrq = NULL;
102 spin_lock(&list_lock);
103 if (list_empty(&inactive_requests))
105 scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
106 list_del(&scmrq->list);
108 spin_unlock(&list_lock);
112 static void scm_request_done(struct scm_request *scmrq)
116 spin_lock_irqsave(&list_lock, flags);
117 list_add(&scmrq->list, &inactive_requests);
118 spin_unlock_irqrestore(&list_lock, flags);
121 static int scm_open(struct block_device *blkdev, fmode_t mode)
123 return scm_get_ref();
126 static void scm_release(struct gendisk *gendisk, fmode_t mode)
131 static const struct block_device_operations scm_blk_devops = {
132 .owner = THIS_MODULE,
134 .release = scm_release,
137 static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
139 return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
142 static void scm_request_prepare(struct scm_request *scmrq)
144 struct scm_blk_dev *bdev = scmrq->bdev;
145 struct scm_device *scmdev = bdev->gendisk->private_data;
146 struct aidaw *aidaw = scmrq->aidaw;
147 struct msb *msb = &scmrq->aob->msb[0];
148 struct req_iterator iter;
152 scmrq->aob->request.msb_count = 1;
153 msb->scm_addr = scmdev->address +
154 ((u64) blk_rq_pos(scmrq->request) << 9);
155 msb->oc = (rq_data_dir(scmrq->request) == READ) ?
156 MSB_OC_READ : MSB_OC_WRITE;
157 msb->flags |= MSB_FLAG_IDA;
158 msb->data_addr = (u64) aidaw;
160 rq_for_each_segment(bv, scmrq->request, iter) {
161 WARN_ON(bv->bv_offset);
162 msb->blk_count += bv->bv_len >> 12;
163 aidaw->data_addr = (u64) page_address(bv->bv_page);
168 static inline void scm_request_init(struct scm_blk_dev *bdev,
169 struct scm_request *scmrq,
172 struct aob_rq_header *aobrq = to_aobrq(scmrq);
173 struct aob *aob = scmrq->aob;
175 memset(aob, 0, sizeof(*aob));
176 memset(scmrq->aidaw, 0, PAGE_SIZE);
177 aobrq->scmdev = bdev->scmdev;
178 aob->request.cmd_code = ARQB_CMD_MOVE;
179 aob->request.data = (u64) aobrq;
180 scmrq->request = req;
184 scm_request_cluster_init(scmrq);
187 static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
189 if (atomic_read(&bdev->queued_reqs)) {
190 /* Queue restart is triggered by the next interrupt. */
193 blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
196 void scm_request_requeue(struct scm_request *scmrq)
198 struct scm_blk_dev *bdev = scmrq->bdev;
200 scm_release_cluster(scmrq);
201 blk_requeue_request(bdev->rq, scmrq->request);
202 atomic_dec(&bdev->queued_reqs);
203 scm_request_done(scmrq);
204 scm_ensure_queue_restart(bdev);
207 void scm_request_finish(struct scm_request *scmrq)
209 struct scm_blk_dev *bdev = scmrq->bdev;
211 scm_release_cluster(scmrq);
212 blk_end_request_all(scmrq->request, scmrq->error);
213 atomic_dec(&bdev->queued_reqs);
214 scm_request_done(scmrq);
217 static void scm_blk_request(struct request_queue *rq)
219 struct scm_device *scmdev = rq->queuedata;
220 struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
221 struct scm_request *scmrq;
225 while ((req = blk_peek_request(rq))) {
226 if (req->cmd_type != REQ_TYPE_FS) {
227 blk_start_request(req);
228 blk_dump_rq_flags(req, KMSG_COMPONENT " bad request");
229 blk_end_request_all(req, -EIO);
233 if (!scm_permit_request(bdev, req)) {
234 scm_ensure_queue_restart(bdev);
237 scmrq = scm_request_fetch();
239 SCM_LOG(5, "no request");
240 scm_ensure_queue_restart(bdev);
243 scm_request_init(bdev, scmrq, req);
244 if (!scm_reserve_cluster(scmrq)) {
245 SCM_LOG(5, "cluster busy");
246 scm_request_done(scmrq);
249 if (scm_need_cluster_request(scmrq)) {
250 atomic_inc(&bdev->queued_reqs);
251 blk_start_request(req);
252 scm_initiate_cluster_request(scmrq);
255 scm_request_prepare(scmrq);
256 atomic_inc(&bdev->queued_reqs);
257 blk_start_request(req);
259 ret = scm_start_aob(scmrq->aob);
261 SCM_LOG(5, "no subchannel");
262 scm_request_requeue(scmrq);
268 static void __scmrq_log_error(struct scm_request *scmrq)
270 struct aob *aob = scmrq->aob;
272 if (scmrq->error == -ETIMEDOUT)
273 SCM_LOG(1, "Request timeout");
275 SCM_LOG(1, "Request error");
276 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
279 SCM_LOG(1, "Retry request");
281 pr_err("An I/O operation to SCM failed with rc=%d\n",
285 void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
287 struct scm_request *scmrq = data;
288 struct scm_blk_dev *bdev = scmrq->bdev;
290 scmrq->error = error;
292 __scmrq_log_error(scmrq);
294 spin_lock(&bdev->lock);
295 list_add_tail(&scmrq->list, &bdev->finished_requests);
296 spin_unlock(&bdev->lock);
297 tasklet_hi_schedule(&bdev->tasklet);
300 static void scm_blk_handle_error(struct scm_request *scmrq)
302 struct scm_blk_dev *bdev = scmrq->bdev;
305 if (scmrq->error != -EIO)
308 /* For -EIO the response block is valid. */
309 switch (scmrq->aob->response.eqc) {
310 case EQC_WR_PROHIBIT:
311 spin_lock_irqsave(&bdev->lock, flags);
312 if (bdev->state != SCM_WR_PROHIBIT)
313 pr_info("%lx: Write access to the SCM increment is suspended\n",
314 (unsigned long) bdev->scmdev->address);
315 bdev->state = SCM_WR_PROHIBIT;
316 spin_unlock_irqrestore(&bdev->lock, flags);
323 if (!scm_start_aob(scmrq->aob))
327 spin_lock_irqsave(&bdev->rq_lock, flags);
328 scm_request_requeue(scmrq);
329 spin_unlock_irqrestore(&bdev->rq_lock, flags);
332 static void scm_blk_tasklet(struct scm_blk_dev *bdev)
334 struct scm_request *scmrq;
337 spin_lock_irqsave(&bdev->lock, flags);
338 while (!list_empty(&bdev->finished_requests)) {
339 scmrq = list_first_entry(&bdev->finished_requests,
340 struct scm_request, list);
341 list_del(&scmrq->list);
342 spin_unlock_irqrestore(&bdev->lock, flags);
344 if (scmrq->error && scmrq->retries-- > 0) {
345 scm_blk_handle_error(scmrq);
347 /* Request restarted or requeued, handle next. */
348 spin_lock_irqsave(&bdev->lock, flags);
352 if (scm_test_cluster_request(scmrq)) {
353 scm_cluster_request_irq(scmrq);
354 spin_lock_irqsave(&bdev->lock, flags);
358 scm_request_finish(scmrq);
359 spin_lock_irqsave(&bdev->lock, flags);
361 spin_unlock_irqrestore(&bdev->lock, flags);
362 /* Look out for more requests. */
363 blk_run_queue(bdev->rq);
366 int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
368 struct request_queue *rq;
369 int len, ret = -ENOMEM;
370 unsigned int devindex, nr_max_blk;
372 devindex = atomic_inc_return(&nr_devices) - 1;
373 /* scma..scmz + scmaa..scmzz */
374 if (devindex > 701) {
379 bdev->scmdev = scmdev;
380 bdev->state = SCM_OPER;
381 spin_lock_init(&bdev->rq_lock);
382 spin_lock_init(&bdev->lock);
383 INIT_LIST_HEAD(&bdev->finished_requests);
384 atomic_set(&bdev->queued_reqs, 0);
385 tasklet_init(&bdev->tasklet,
386 (void (*)(unsigned long)) scm_blk_tasklet,
387 (unsigned long) bdev);
389 rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
394 nr_max_blk = min(scmdev->nr_max_block,
395 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
397 blk_queue_logical_block_size(rq, 1 << 12);
398 blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
399 blk_queue_max_segments(rq, nr_max_blk);
400 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
401 scm_blk_dev_cluster_setup(bdev);
403 bdev->gendisk = alloc_disk(SCM_NR_PARTS);
407 rq->queuedata = scmdev;
408 bdev->gendisk->driverfs_dev = &scmdev->dev;
409 bdev->gendisk->private_data = scmdev;
410 bdev->gendisk->fops = &scm_blk_devops;
411 bdev->gendisk->queue = rq;
412 bdev->gendisk->major = scm_major;
413 bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
415 len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
417 len += snprintf(bdev->gendisk->disk_name + len,
418 DISK_NAME_LEN - len, "%c",
419 'a' + (devindex / 26) - 1);
420 devindex = devindex % 26;
422 snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
425 /* 512 byte sectors */
426 set_capacity(bdev->gendisk, scmdev->size >> 9);
427 add_disk(bdev->gendisk);
431 blk_cleanup_queue(rq);
433 atomic_dec(&nr_devices);
437 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
439 tasklet_kill(&bdev->tasklet);
440 del_gendisk(bdev->gendisk);
441 blk_cleanup_queue(bdev->gendisk->queue);
442 put_disk(bdev->gendisk);
445 void scm_blk_set_available(struct scm_blk_dev *bdev)
449 spin_lock_irqsave(&bdev->lock, flags);
450 if (bdev->state == SCM_WR_PROHIBIT)
451 pr_info("%lx: Write access to the SCM increment is restored\n",
452 (unsigned long) bdev->scmdev->address);
453 bdev->state = SCM_OPER;
454 spin_unlock_irqrestore(&bdev->lock, flags);
457 static int __init scm_blk_init(void)
461 if (!scm_cluster_size_valid())
464 ret = register_blkdev(0, "scm");
469 ret = scm_alloc_rqs(nr_requests);
473 scm_debug = debug_register("scm_log", 16, 1, 16);
479 debug_register_view(scm_debug, &debug_hex_ascii_view);
480 debug_set_level(scm_debug, 2);
482 ret = scm_drv_init();
489 debug_unregister(scm_debug);
492 unregister_blkdev(scm_major, "scm");
496 module_init(scm_blk_init);
498 static void __exit scm_blk_cleanup(void)
501 debug_unregister(scm_debug);
503 unregister_blkdev(scm_major, "scm");
505 module_exit(scm_blk_cleanup);