]> git.karo-electronics.de Git - karo-tx-linux.git/blob - drivers/nvme/host/core.c
26ae4afd373733fc28e72049513b0e896b0b61ab
[karo-tx-linux.git] / drivers / nvme / host / core.c
1 /*
2  * NVM Express device driver
3  * Copyright (c) 2011-2014, Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  */
14
15 #include <linux/blkdev.h>
16 #include <linux/blk-mq.h>
17 #include <linux/delay.h>
18 #include <linux/errno.h>
19 #include <linux/hdreg.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/list_sort.h>
23 #include <linux/slab.h>
24 #include <linux/types.h>
25 #include <linux/pr.h>
26 #include <linux/ptrace.h>
27 #include <linux/nvme_ioctl.h>
28 #include <linux/t10-pi.h>
29 #include <scsi/sg.h>
30 #include <asm/unaligned.h>
31
32 #include "nvme.h"
33 #include "fabrics.h"
34
35 #define NVME_MINORS             (1U << MINORBITS)
36
37 unsigned char admin_timeout = 60;
38 module_param(admin_timeout, byte, 0644);
39 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
40 EXPORT_SYMBOL_GPL(admin_timeout);
41
42 unsigned char nvme_io_timeout = 30;
43 module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
44 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
45 EXPORT_SYMBOL_GPL(nvme_io_timeout);
46
47 unsigned char shutdown_timeout = 5;
48 module_param(shutdown_timeout, byte, 0644);
49 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
50
51 unsigned int nvme_max_retries = 5;
52 module_param_named(max_retries, nvme_max_retries, uint, 0644);
53 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
54 EXPORT_SYMBOL_GPL(nvme_max_retries);
55
56 static int nvme_char_major;
57 module_param(nvme_char_major, int, 0);
58
59 static LIST_HEAD(nvme_ctrl_list);
60 static DEFINE_SPINLOCK(dev_list_lock);
61
62 static struct class *nvme_class;
63
64 void nvme_cancel_request(struct request *req, void *data, bool reserved)
65 {
66         int status;
67
68         if (!blk_mq_request_started(req))
69                 return;
70
71         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
72                                 "Cancelling I/O %d", req->tag);
73
74         status = NVME_SC_ABORT_REQ;
75         if (blk_queue_dying(req->q))
76                 status |= NVME_SC_DNR;
77         blk_mq_complete_request(req, status);
78 }
79 EXPORT_SYMBOL_GPL(nvme_cancel_request);
80
81 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
82                 enum nvme_ctrl_state new_state)
83 {
84         enum nvme_ctrl_state old_state;
85         bool changed = false;
86
87         spin_lock_irq(&ctrl->lock);
88
89         old_state = ctrl->state;
90         switch (new_state) {
91         case NVME_CTRL_LIVE:
92                 switch (old_state) {
93                 case NVME_CTRL_NEW:
94                 case NVME_CTRL_RESETTING:
95                 case NVME_CTRL_RECONNECTING:
96                         changed = true;
97                         /* FALLTHRU */
98                 default:
99                         break;
100                 }
101                 break;
102         case NVME_CTRL_RESETTING:
103                 switch (old_state) {
104                 case NVME_CTRL_NEW:
105                 case NVME_CTRL_LIVE:
106                 case NVME_CTRL_RECONNECTING:
107                         changed = true;
108                         /* FALLTHRU */
109                 default:
110                         break;
111                 }
112                 break;
113         case NVME_CTRL_RECONNECTING:
114                 switch (old_state) {
115                 case NVME_CTRL_LIVE:
116                         changed = true;
117                         /* FALLTHRU */
118                 default:
119                         break;
120                 }
121                 break;
122         case NVME_CTRL_DELETING:
123                 switch (old_state) {
124                 case NVME_CTRL_LIVE:
125                 case NVME_CTRL_RESETTING:
126                 case NVME_CTRL_RECONNECTING:
127                         changed = true;
128                         /* FALLTHRU */
129                 default:
130                         break;
131                 }
132                 break;
133         case NVME_CTRL_DEAD:
134                 switch (old_state) {
135                 case NVME_CTRL_DELETING:
136                         changed = true;
137                         /* FALLTHRU */
138                 default:
139                         break;
140                 }
141                 break;
142         default:
143                 break;
144         }
145
146         if (changed)
147                 ctrl->state = new_state;
148
149         spin_unlock_irq(&ctrl->lock);
150
151         return changed;
152 }
153 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
154
155 static void nvme_free_ns(struct kref *kref)
156 {
157         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
158
159         if (ns->ndev)
160                 nvme_nvm_unregister(ns);
161
162         if (ns->disk) {
163                 spin_lock(&dev_list_lock);
164                 ns->disk->private_data = NULL;
165                 spin_unlock(&dev_list_lock);
166         }
167
168         put_disk(ns->disk);
169         ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
170         nvme_put_ctrl(ns->ctrl);
171         kfree(ns);
172 }
173
174 static void nvme_put_ns(struct nvme_ns *ns)
175 {
176         kref_put(&ns->kref, nvme_free_ns);
177 }
178
179 static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
180 {
181         struct nvme_ns *ns;
182
183         spin_lock(&dev_list_lock);
184         ns = disk->private_data;
185         if (ns) {
186                 if (!kref_get_unless_zero(&ns->kref))
187                         goto fail;
188                 if (!try_module_get(ns->ctrl->ops->module))
189                         goto fail_put_ns;
190         }
191         spin_unlock(&dev_list_lock);
192
193         return ns;
194
195 fail_put_ns:
196         kref_put(&ns->kref, nvme_free_ns);
197 fail:
198         spin_unlock(&dev_list_lock);
199         return NULL;
200 }
201
202 void nvme_requeue_req(struct request *req)
203 {
204         blk_mq_requeue_request(req, !blk_mq_queue_stopped(req->q));
205 }
206 EXPORT_SYMBOL_GPL(nvme_requeue_req);
207
208 struct request *nvme_alloc_request(struct request_queue *q,
209                 struct nvme_command *cmd, unsigned int flags, int qid)
210 {
211         struct request *req;
212
213         if (qid == NVME_QID_ANY) {
214                 req = blk_mq_alloc_request(q, nvme_is_write(cmd), flags);
215         } else {
216                 req = blk_mq_alloc_request_hctx(q, nvme_is_write(cmd), flags,
217                                 qid ? qid - 1 : 0);
218         }
219         if (IS_ERR(req))
220                 return req;
221
222         req->cmd_type = REQ_TYPE_DRV_PRIV;
223         req->cmd_flags |= REQ_FAILFAST_DRIVER;
224         nvme_req(req)->cmd = cmd;
225
226         return req;
227 }
228 EXPORT_SYMBOL_GPL(nvme_alloc_request);
229
230 static inline void nvme_setup_flush(struct nvme_ns *ns,
231                 struct nvme_command *cmnd)
232 {
233         memset(cmnd, 0, sizeof(*cmnd));
234         cmnd->common.opcode = nvme_cmd_flush;
235         cmnd->common.nsid = cpu_to_le32(ns->ns_id);
236 }
237
238 static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req,
239                 struct nvme_command *cmnd)
240 {
241         struct nvme_dsm_range *range;
242         unsigned int nr_bytes = blk_rq_bytes(req);
243
244         range = kmalloc(sizeof(*range), GFP_ATOMIC);
245         if (!range)
246                 return BLK_MQ_RQ_QUEUE_BUSY;
247
248         range->cattr = cpu_to_le32(0);
249         range->nlb = cpu_to_le32(nr_bytes >> ns->lba_shift);
250         range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
251
252         memset(cmnd, 0, sizeof(*cmnd));
253         cmnd->dsm.opcode = nvme_cmd_dsm;
254         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
255         cmnd->dsm.nr = 0;
256         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
257
258         req->special_vec.bv_page = virt_to_page(range);
259         req->special_vec.bv_offset = offset_in_page(range);
260         req->special_vec.bv_len = sizeof(*range);
261         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
262
263         return BLK_MQ_RQ_QUEUE_OK;
264 }
265
266 static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
267                 struct nvme_command *cmnd)
268 {
269         u16 control = 0;
270         u32 dsmgmt = 0;
271
272         if (req->cmd_flags & REQ_FUA)
273                 control |= NVME_RW_FUA;
274         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
275                 control |= NVME_RW_LR;
276
277         if (req->cmd_flags & REQ_RAHEAD)
278                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
279
280         memset(cmnd, 0, sizeof(*cmnd));
281         cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
282         cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
283         cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
284         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
285
286         if (ns->ms) {
287                 switch (ns->pi_type) {
288                 case NVME_NS_DPS_PI_TYPE3:
289                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
290                         break;
291                 case NVME_NS_DPS_PI_TYPE1:
292                 case NVME_NS_DPS_PI_TYPE2:
293                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
294                                         NVME_RW_PRINFO_PRCHK_REF;
295                         cmnd->rw.reftag = cpu_to_le32(
296                                         nvme_block_nr(ns, blk_rq_pos(req)));
297                         break;
298                 }
299                 if (!blk_integrity_rq(req))
300                         control |= NVME_RW_PRINFO_PRACT;
301         }
302
303         cmnd->rw.control = cpu_to_le16(control);
304         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
305 }
306
307 int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
308                 struct nvme_command *cmd)
309 {
310         int ret = BLK_MQ_RQ_QUEUE_OK;
311
312         if (req->cmd_type == REQ_TYPE_DRV_PRIV)
313                 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
314         else if (req_op(req) == REQ_OP_FLUSH)
315                 nvme_setup_flush(ns, cmd);
316         else if (req_op(req) == REQ_OP_DISCARD)
317                 ret = nvme_setup_discard(ns, req, cmd);
318         else
319                 nvme_setup_rw(ns, req, cmd);
320
321         cmd->common.command_id = req->tag;
322
323         return ret;
324 }
325 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
326
327 /*
328  * Returns 0 on success.  If the result is negative, it's a Linux error code;
329  * if the result is positive, it's an NVM Express status code
330  */
331 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
332                 union nvme_result *result, void *buffer, unsigned bufflen,
333                 unsigned timeout, int qid, int at_head, int flags)
334 {
335         struct request *req;
336         int ret;
337
338         req = nvme_alloc_request(q, cmd, flags, qid);
339         if (IS_ERR(req))
340                 return PTR_ERR(req);
341
342         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
343
344         if (buffer && bufflen) {
345                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
346                 if (ret)
347                         goto out;
348         }
349
350         blk_execute_rq(req->q, NULL, req, at_head);
351         if (result)
352                 *result = nvme_req(req)->result;
353         ret = req->errors;
354  out:
355         blk_mq_free_request(req);
356         return ret;
357 }
358 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
359
360 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
361                 void *buffer, unsigned bufflen)
362 {
363         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
364                         NVME_QID_ANY, 0, 0);
365 }
366 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
367
368 int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
369                 void __user *ubuffer, unsigned bufflen,
370                 void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
371                 u32 *result, unsigned timeout)
372 {
373         bool write = nvme_is_write(cmd);
374         struct nvme_ns *ns = q->queuedata;
375         struct gendisk *disk = ns ? ns->disk : NULL;
376         struct request *req;
377         struct bio *bio = NULL;
378         void *meta = NULL;
379         int ret;
380
381         req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
382         if (IS_ERR(req))
383                 return PTR_ERR(req);
384
385         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
386
387         if (ubuffer && bufflen) {
388                 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
389                                 GFP_KERNEL);
390                 if (ret)
391                         goto out;
392                 bio = req->bio;
393
394                 if (!disk)
395                         goto submit;
396                 bio->bi_bdev = bdget_disk(disk, 0);
397                 if (!bio->bi_bdev) {
398                         ret = -ENODEV;
399                         goto out_unmap;
400                 }
401
402                 if (meta_buffer && meta_len) {
403                         struct bio_integrity_payload *bip;
404
405                         meta = kmalloc(meta_len, GFP_KERNEL);
406                         if (!meta) {
407                                 ret = -ENOMEM;
408                                 goto out_unmap;
409                         }
410
411                         if (write) {
412                                 if (copy_from_user(meta, meta_buffer,
413                                                 meta_len)) {
414                                         ret = -EFAULT;
415                                         goto out_free_meta;
416                                 }
417                         }
418
419                         bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
420                         if (IS_ERR(bip)) {
421                                 ret = PTR_ERR(bip);
422                                 goto out_free_meta;
423                         }
424
425                         bip->bip_iter.bi_size = meta_len;
426                         bip->bip_iter.bi_sector = meta_seed;
427
428                         ret = bio_integrity_add_page(bio, virt_to_page(meta),
429                                         meta_len, offset_in_page(meta));
430                         if (ret != meta_len) {
431                                 ret = -ENOMEM;
432                                 goto out_free_meta;
433                         }
434                 }
435         }
436  submit:
437         blk_execute_rq(req->q, disk, req, 0);
438         ret = req->errors;
439         if (result)
440                 *result = le32_to_cpu(nvme_req(req)->result.u32);
441         if (meta && !ret && !write) {
442                 if (copy_to_user(meta_buffer, meta, meta_len))
443                         ret = -EFAULT;
444         }
445  out_free_meta:
446         kfree(meta);
447  out_unmap:
448         if (bio) {
449                 if (disk && bio->bi_bdev)
450                         bdput(bio->bi_bdev);
451                 blk_rq_unmap_user(bio);
452         }
453  out:
454         blk_mq_free_request(req);
455         return ret;
456 }
457
458 int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
459                 void __user *ubuffer, unsigned bufflen, u32 *result,
460                 unsigned timeout)
461 {
462         return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
463                         result, timeout);
464 }
465
466 static void nvme_keep_alive_end_io(struct request *rq, int error)
467 {
468         struct nvme_ctrl *ctrl = rq->end_io_data;
469
470         blk_mq_free_request(rq);
471
472         if (error) {
473                 dev_err(ctrl->device,
474                         "failed nvme_keep_alive_end_io error=%d\n", error);
475                 return;
476         }
477
478         schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
479 }
480
481 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
482 {
483         struct nvme_command c;
484         struct request *rq;
485
486         memset(&c, 0, sizeof(c));
487         c.common.opcode = nvme_admin_keep_alive;
488
489         rq = nvme_alloc_request(ctrl->admin_q, &c, BLK_MQ_REQ_RESERVED,
490                         NVME_QID_ANY);
491         if (IS_ERR(rq))
492                 return PTR_ERR(rq);
493
494         rq->timeout = ctrl->kato * HZ;
495         rq->end_io_data = ctrl;
496
497         blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
498
499         return 0;
500 }
501
502 static void nvme_keep_alive_work(struct work_struct *work)
503 {
504         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
505                         struct nvme_ctrl, ka_work);
506
507         if (nvme_keep_alive(ctrl)) {
508                 /* allocation failure, reset the controller */
509                 dev_err(ctrl->device, "keep-alive failed\n");
510                 ctrl->ops->reset_ctrl(ctrl);
511                 return;
512         }
513 }
514
515 void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
516 {
517         if (unlikely(ctrl->kato == 0))
518                 return;
519
520         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
521         schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
522 }
523 EXPORT_SYMBOL_GPL(nvme_start_keep_alive);
524
525 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
526 {
527         if (unlikely(ctrl->kato == 0))
528                 return;
529
530         cancel_delayed_work_sync(&ctrl->ka_work);
531 }
532 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
533
534 int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
535 {
536         struct nvme_command c = { };
537         int error;
538
539         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
540         c.identify.opcode = nvme_admin_identify;
541         c.identify.cns = cpu_to_le32(NVME_ID_CNS_CTRL);
542
543         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
544         if (!*id)
545                 return -ENOMEM;
546
547         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
548                         sizeof(struct nvme_id_ctrl));
549         if (error)
550                 kfree(*id);
551         return error;
552 }
553
554 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
555 {
556         struct nvme_command c = { };
557
558         c.identify.opcode = nvme_admin_identify;
559         c.identify.cns = cpu_to_le32(NVME_ID_CNS_NS_ACTIVE_LIST);
560         c.identify.nsid = cpu_to_le32(nsid);
561         return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
562 }
563
564 int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
565                 struct nvme_id_ns **id)
566 {
567         struct nvme_command c = { };
568         int error;
569
570         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
571         c.identify.opcode = nvme_admin_identify,
572         c.identify.nsid = cpu_to_le32(nsid),
573
574         *id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
575         if (!*id)
576                 return -ENOMEM;
577
578         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
579                         sizeof(struct nvme_id_ns));
580         if (error)
581                 kfree(*id);
582         return error;
583 }
584
585 int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
586                       void *buffer, size_t buflen, u32 *result)
587 {
588         struct nvme_command c;
589         union nvme_result res;
590         int ret;
591
592         memset(&c, 0, sizeof(c));
593         c.features.opcode = nvme_admin_get_features;
594         c.features.nsid = cpu_to_le32(nsid);
595         c.features.fid = cpu_to_le32(fid);
596
597         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res, buffer, buflen, 0,
598                         NVME_QID_ANY, 0, 0);
599         if (ret >= 0 && result)
600                 *result = le32_to_cpu(res.u32);
601         return ret;
602 }
603
604 int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
605                       void *buffer, size_t buflen, u32 *result)
606 {
607         struct nvme_command c;
608         union nvme_result res;
609         int ret;
610
611         memset(&c, 0, sizeof(c));
612         c.features.opcode = nvme_admin_set_features;
613         c.features.fid = cpu_to_le32(fid);
614         c.features.dword11 = cpu_to_le32(dword11);
615
616         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
617                         buffer, buflen, 0, NVME_QID_ANY, 0, 0);
618         if (ret >= 0 && result)
619                 *result = le32_to_cpu(res.u32);
620         return ret;
621 }
622
623 int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
624 {
625         struct nvme_command c = { };
626         int error;
627
628         c.common.opcode = nvme_admin_get_log_page,
629         c.common.nsid = cpu_to_le32(0xFFFFFFFF),
630         c.common.cdw10[0] = cpu_to_le32(
631                         (((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
632                          NVME_LOG_SMART),
633
634         *log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
635         if (!*log)
636                 return -ENOMEM;
637
638         error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
639                         sizeof(struct nvme_smart_log));
640         if (error)
641                 kfree(*log);
642         return error;
643 }
644
645 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
646 {
647         u32 q_count = (*count - 1) | ((*count - 1) << 16);
648         u32 result;
649         int status, nr_io_queues;
650
651         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
652                         &result);
653         if (status < 0)
654                 return status;
655
656         /*
657          * Degraded controllers might return an error when setting the queue
658          * count.  We still want to be able to bring them online and offer
659          * access to the admin queue, as that might be only way to fix them up.
660          */
661         if (status > 0) {
662                 dev_err(ctrl->dev, "Could not set queue count (%d)\n", status);
663                 *count = 0;
664         } else {
665                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
666                 *count = min(*count, nr_io_queues);
667         }
668
669         return 0;
670 }
671 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
672
673 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
674 {
675         struct nvme_user_io io;
676         struct nvme_command c;
677         unsigned length, meta_len;
678         void __user *metadata;
679
680         if (copy_from_user(&io, uio, sizeof(io)))
681                 return -EFAULT;
682         if (io.flags)
683                 return -EINVAL;
684
685         switch (io.opcode) {
686         case nvme_cmd_write:
687         case nvme_cmd_read:
688         case nvme_cmd_compare:
689                 break;
690         default:
691                 return -EINVAL;
692         }
693
694         length = (io.nblocks + 1) << ns->lba_shift;
695         meta_len = (io.nblocks + 1) * ns->ms;
696         metadata = (void __user *)(uintptr_t)io.metadata;
697
698         if (ns->ext) {
699                 length += meta_len;
700                 meta_len = 0;
701         } else if (meta_len) {
702                 if ((io.metadata & 3) || !io.metadata)
703                         return -EINVAL;
704         }
705
706         memset(&c, 0, sizeof(c));
707         c.rw.opcode = io.opcode;
708         c.rw.flags = io.flags;
709         c.rw.nsid = cpu_to_le32(ns->ns_id);
710         c.rw.slba = cpu_to_le64(io.slba);
711         c.rw.length = cpu_to_le16(io.nblocks);
712         c.rw.control = cpu_to_le16(io.control);
713         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
714         c.rw.reftag = cpu_to_le32(io.reftag);
715         c.rw.apptag = cpu_to_le16(io.apptag);
716         c.rw.appmask = cpu_to_le16(io.appmask);
717
718         return __nvme_submit_user_cmd(ns->queue, &c,
719                         (void __user *)(uintptr_t)io.addr, length,
720                         metadata, meta_len, io.slba, NULL, 0);
721 }
722
723 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
724                         struct nvme_passthru_cmd __user *ucmd)
725 {
726         struct nvme_passthru_cmd cmd;
727         struct nvme_command c;
728         unsigned timeout = 0;
729         int status;
730
731         if (!capable(CAP_SYS_ADMIN))
732                 return -EACCES;
733         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
734                 return -EFAULT;
735         if (cmd.flags)
736                 return -EINVAL;
737
738         memset(&c, 0, sizeof(c));
739         c.common.opcode = cmd.opcode;
740         c.common.flags = cmd.flags;
741         c.common.nsid = cpu_to_le32(cmd.nsid);
742         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
743         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
744         c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
745         c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
746         c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
747         c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
748         c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
749         c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
750
751         if (cmd.timeout_ms)
752                 timeout = msecs_to_jiffies(cmd.timeout_ms);
753
754         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
755                         (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
756                         &cmd.result, timeout);
757         if (status >= 0) {
758                 if (put_user(cmd.result, &ucmd->result))
759                         return -EFAULT;
760         }
761
762         return status;
763 }
764
765 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
766                 unsigned int cmd, unsigned long arg)
767 {
768         struct nvme_ns *ns = bdev->bd_disk->private_data;
769
770         switch (cmd) {
771         case NVME_IOCTL_ID:
772                 force_successful_syscall_return();
773                 return ns->ns_id;
774         case NVME_IOCTL_ADMIN_CMD:
775                 return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
776         case NVME_IOCTL_IO_CMD:
777                 return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
778         case NVME_IOCTL_SUBMIT_IO:
779                 return nvme_submit_io(ns, (void __user *)arg);
780 #ifdef CONFIG_BLK_DEV_NVME_SCSI
781         case SG_GET_VERSION_NUM:
782                 return nvme_sg_get_version_num((void __user *)arg);
783         case SG_IO:
784                 return nvme_sg_io(ns, (void __user *)arg);
785 #endif
786         default:
787 #ifdef CONFIG_NVM
788                 if (ns->ndev)
789                         return nvme_nvm_ioctl(ns, cmd, arg);
790 #endif
791                 if (is_sed_ioctl(cmd))
792                         return sed_ioctl(&ns->ctrl->opal_dev, cmd, arg);
793                 return -ENOTTY;
794         }
795 }
796
797 #ifdef CONFIG_COMPAT
798 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
799                         unsigned int cmd, unsigned long arg)
800 {
801         switch (cmd) {
802         case SG_IO:
803                 return -ENOIOCTLCMD;
804         }
805         return nvme_ioctl(bdev, mode, cmd, arg);
806 }
807 #else
808 #define nvme_compat_ioctl       NULL
809 #endif
810
811 static int nvme_open(struct block_device *bdev, fmode_t mode)
812 {
813         return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
814 }
815
816 static void nvme_release(struct gendisk *disk, fmode_t mode)
817 {
818         struct nvme_ns *ns = disk->private_data;
819
820         module_put(ns->ctrl->ops->module);
821         nvme_put_ns(ns);
822 }
823
824 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
825 {
826         /* some standard values */
827         geo->heads = 1 << 6;
828         geo->sectors = 1 << 5;
829         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
830         return 0;
831 }
832
833 #ifdef CONFIG_BLK_DEV_INTEGRITY
834 static void nvme_init_integrity(struct nvme_ns *ns)
835 {
836         struct blk_integrity integrity;
837
838         memset(&integrity, 0, sizeof(integrity));
839         switch (ns->pi_type) {
840         case NVME_NS_DPS_PI_TYPE3:
841                 integrity.profile = &t10_pi_type3_crc;
842                 integrity.tag_size = sizeof(u16) + sizeof(u32);
843                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
844                 break;
845         case NVME_NS_DPS_PI_TYPE1:
846         case NVME_NS_DPS_PI_TYPE2:
847                 integrity.profile = &t10_pi_type1_crc;
848                 integrity.tag_size = sizeof(u16);
849                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
850                 break;
851         default:
852                 integrity.profile = NULL;
853                 break;
854         }
855         integrity.tuple_size = ns->ms;
856         blk_integrity_register(ns->disk, &integrity);
857         blk_queue_max_integrity_segments(ns->queue, 1);
858 }
859 #else
860 static void nvme_init_integrity(struct nvme_ns *ns)
861 {
862 }
863 #endif /* CONFIG_BLK_DEV_INTEGRITY */
864
865 static void nvme_config_discard(struct nvme_ns *ns)
866 {
867         struct nvme_ctrl *ctrl = ns->ctrl;
868         u32 logical_block_size = queue_logical_block_size(ns->queue);
869
870         if (ctrl->quirks & NVME_QUIRK_DISCARD_ZEROES)
871                 ns->queue->limits.discard_zeroes_data = 1;
872         else
873                 ns->queue->limits.discard_zeroes_data = 0;
874
875         ns->queue->limits.discard_alignment = logical_block_size;
876         ns->queue->limits.discard_granularity = logical_block_size;
877         blk_queue_max_discard_sectors(ns->queue, UINT_MAX);
878         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
879 }
880
881 static int nvme_revalidate_ns(struct nvme_ns *ns, struct nvme_id_ns **id)
882 {
883         if (nvme_identify_ns(ns->ctrl, ns->ns_id, id)) {
884                 dev_warn(ns->ctrl->dev, "%s: Identify failure\n", __func__);
885                 return -ENODEV;
886         }
887
888         if ((*id)->ncap == 0) {
889                 kfree(*id);
890                 return -ENODEV;
891         }
892
893         if (ns->ctrl->vs >= NVME_VS(1, 1, 0))
894                 memcpy(ns->eui, (*id)->eui64, sizeof(ns->eui));
895         if (ns->ctrl->vs >= NVME_VS(1, 2, 0))
896                 memcpy(ns->uuid, (*id)->nguid, sizeof(ns->uuid));
897
898         return 0;
899 }
900
901 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
902 {
903         struct nvme_ns *ns = disk->private_data;
904         u8 lbaf, pi_type;
905         u16 old_ms;
906         unsigned short bs;
907
908         old_ms = ns->ms;
909         lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
910         ns->lba_shift = id->lbaf[lbaf].ds;
911         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
912         ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
913
914         /*
915          * If identify namespace failed, use default 512 byte block size so
916          * block layer can use before failing read/write for 0 capacity.
917          */
918         if (ns->lba_shift == 0)
919                 ns->lba_shift = 9;
920         bs = 1 << ns->lba_shift;
921         /* XXX: PI implementation requires metadata equal t10 pi tuple size */
922         pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
923                                         id->dps & NVME_NS_DPS_PI_MASK : 0;
924
925         blk_mq_freeze_queue(disk->queue);
926         if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
927                                 ns->ms != old_ms ||
928                                 bs != queue_logical_block_size(disk->queue) ||
929                                 (ns->ms && ns->ext)))
930                 blk_integrity_unregister(disk);
931
932         ns->pi_type = pi_type;
933         blk_queue_logical_block_size(ns->queue, bs);
934
935         if (ns->ms && !blk_get_integrity(disk) && !ns->ext)
936                 nvme_init_integrity(ns);
937         if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
938                 set_capacity(disk, 0);
939         else
940                 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
941
942         if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
943                 nvme_config_discard(ns);
944         blk_mq_unfreeze_queue(disk->queue);
945 }
946
947 static int nvme_revalidate_disk(struct gendisk *disk)
948 {
949         struct nvme_ns *ns = disk->private_data;
950         struct nvme_id_ns *id = NULL;
951         int ret;
952
953         if (test_bit(NVME_NS_DEAD, &ns->flags)) {
954                 set_capacity(disk, 0);
955                 return -ENODEV;
956         }
957
958         ret = nvme_revalidate_ns(ns, &id);
959         if (ret)
960                 return ret;
961
962         __nvme_revalidate_disk(disk, id);
963         kfree(id);
964
965         return 0;
966 }
967
968 static char nvme_pr_type(enum pr_type type)
969 {
970         switch (type) {
971         case PR_WRITE_EXCLUSIVE:
972                 return 1;
973         case PR_EXCLUSIVE_ACCESS:
974                 return 2;
975         case PR_WRITE_EXCLUSIVE_REG_ONLY:
976                 return 3;
977         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
978                 return 4;
979         case PR_WRITE_EXCLUSIVE_ALL_REGS:
980                 return 5;
981         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
982                 return 6;
983         default:
984                 return 0;
985         }
986 };
987
988 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
989                                 u64 key, u64 sa_key, u8 op)
990 {
991         struct nvme_ns *ns = bdev->bd_disk->private_data;
992         struct nvme_command c;
993         u8 data[16] = { 0, };
994
995         put_unaligned_le64(key, &data[0]);
996         put_unaligned_le64(sa_key, &data[8]);
997
998         memset(&c, 0, sizeof(c));
999         c.common.opcode = op;
1000         c.common.nsid = cpu_to_le32(ns->ns_id);
1001         c.common.cdw10[0] = cpu_to_le32(cdw10);
1002
1003         return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1004 }
1005
1006 static int nvme_pr_register(struct block_device *bdev, u64 old,
1007                 u64 new, unsigned flags)
1008 {
1009         u32 cdw10;
1010
1011         if (flags & ~PR_FL_IGNORE_KEY)
1012                 return -EOPNOTSUPP;
1013
1014         cdw10 = old ? 2 : 0;
1015         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
1016         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
1017         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
1018 }
1019
1020 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
1021                 enum pr_type type, unsigned flags)
1022 {
1023         u32 cdw10;
1024
1025         if (flags & ~PR_FL_IGNORE_KEY)
1026                 return -EOPNOTSUPP;
1027
1028         cdw10 = nvme_pr_type(type) << 8;
1029         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
1030         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
1031 }
1032
1033 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
1034                 enum pr_type type, bool abort)
1035 {
1036         u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
1037         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
1038 }
1039
1040 static int nvme_pr_clear(struct block_device *bdev, u64 key)
1041 {
1042         u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1043         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
1044 }
1045
1046 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
1047 {
1048         u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
1049         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1050 }
1051
1052 static const struct pr_ops nvme_pr_ops = {
1053         .pr_register    = nvme_pr_register,
1054         .pr_reserve     = nvme_pr_reserve,
1055         .pr_release     = nvme_pr_release,
1056         .pr_preempt     = nvme_pr_preempt,
1057         .pr_clear       = nvme_pr_clear,
1058 };
1059
1060 #ifdef CONFIG_BLK_SED_OPAL
1061 int nvme_sec_submit(struct opal_dev *dev, u16 spsp, u8 secp,
1062                     void *buffer, size_t len, bool send)
1063 {
1064         struct nvme_command cmd;
1065         struct nvme_ctrl *ctrl = NULL;
1066
1067         memset(&cmd, 0, sizeof(cmd));
1068         if (send)
1069                 cmd.common.opcode = nvme_admin_security_send;
1070         else
1071                 cmd.common.opcode = nvme_admin_security_recv;
1072         ctrl = container_of(dev, struct nvme_ctrl, opal_dev);
1073         cmd.common.nsid = 0;
1074         cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
1075         cmd.common.cdw10[1] = cpu_to_le32(len);
1076
1077         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
1078                                       ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
1079 }
1080 EXPORT_SYMBOL_GPL(nvme_sec_submit);
1081 #endif /* CONFIG_BLK_SED_OPAL */
1082
1083 static const struct block_device_operations nvme_fops = {
1084         .owner          = THIS_MODULE,
1085         .ioctl          = nvme_ioctl,
1086         .compat_ioctl   = nvme_compat_ioctl,
1087         .open           = nvme_open,
1088         .release        = nvme_release,
1089         .getgeo         = nvme_getgeo,
1090         .revalidate_disk= nvme_revalidate_disk,
1091         .pr_ops         = &nvme_pr_ops,
1092 };
1093
1094 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
1095 {
1096         unsigned long timeout =
1097                 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1098         u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
1099         int ret;
1100
1101         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
1102                 if (csts == ~0)
1103                         return -ENODEV;
1104                 if ((csts & NVME_CSTS_RDY) == bit)
1105                         break;
1106
1107                 msleep(100);
1108                 if (fatal_signal_pending(current))
1109                         return -EINTR;
1110                 if (time_after(jiffies, timeout)) {
1111                         dev_err(ctrl->device,
1112                                 "Device not ready; aborting %s\n", enabled ?
1113                                                 "initialisation" : "reset");
1114                         return -ENODEV;
1115                 }
1116         }
1117
1118         return ret;
1119 }
1120
1121 /*
1122  * If the device has been passed off to us in an enabled state, just clear
1123  * the enabled bit.  The spec says we should set the 'shutdown notification
1124  * bits', but doing so may cause the device to complete commands to the
1125  * admin queue ... and we don't know what memory that might be pointing at!
1126  */
1127 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
1128 {
1129         int ret;
1130
1131         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
1132         ctrl->ctrl_config &= ~NVME_CC_ENABLE;
1133
1134         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1135         if (ret)
1136                 return ret;
1137
1138         /* Checking for ctrl->tagset is a trick to avoid sleeping on module
1139          * load, since we only need the quirk on reset_controller. Notice
1140          * that the HGST device needs this delay only in firmware activation
1141          * procedure; unfortunately we have no (easy) way to verify this.
1142          */
1143         if ((ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) && ctrl->tagset)
1144                 msleep(NVME_QUIRK_DELAY_AMOUNT);
1145
1146         return nvme_wait_ready(ctrl, cap, false);
1147 }
1148 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1149
1150 int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
1151 {
1152         /*
1153          * Default to a 4K page size, with the intention to update this
1154          * path in the future to accomodate architectures with differing
1155          * kernel and IO page sizes.
1156          */
1157         unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
1158         int ret;
1159
1160         if (page_shift < dev_page_min) {
1161                 dev_err(ctrl->device,
1162                         "Minimum device page size %u too large for host (%u)\n",
1163                         1 << dev_page_min, 1 << page_shift);
1164                 return -ENODEV;
1165         }
1166
1167         ctrl->page_size = 1 << page_shift;
1168
1169         ctrl->ctrl_config = NVME_CC_CSS_NVM;
1170         ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
1171         ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1172         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1173         ctrl->ctrl_config |= NVME_CC_ENABLE;
1174
1175         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1176         if (ret)
1177                 return ret;
1178         return nvme_wait_ready(ctrl, cap, true);
1179 }
1180 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1181
1182 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
1183 {
1184         unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
1185         u32 csts;
1186         int ret;
1187
1188         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
1189         ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
1190
1191         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1192         if (ret)
1193                 return ret;
1194
1195         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
1196                 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
1197                         break;
1198
1199                 msleep(100);
1200                 if (fatal_signal_pending(current))
1201                         return -EINTR;
1202                 if (time_after(jiffies, timeout)) {
1203                         dev_err(ctrl->device,
1204                                 "Device shutdown incomplete; abort shutdown\n");
1205                         return -ENODEV;
1206                 }
1207         }
1208
1209         return ret;
1210 }
1211 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1212
1213 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1214                 struct request_queue *q)
1215 {
1216         bool vwc = false;
1217
1218         if (ctrl->max_hw_sectors) {
1219                 u32 max_segments =
1220                         (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
1221
1222                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1223                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1224         }
1225         if (ctrl->quirks & NVME_QUIRK_STRIPE_SIZE)
1226                 blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1227         blk_queue_virt_boundary(q, ctrl->page_size - 1);
1228         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
1229                 vwc = true;
1230         blk_queue_write_cache(q, vwc, vwc);
1231 }
1232
1233 /*
1234  * Initialize the cached copies of the Identify data and various controller
1235  * register in our nvme_ctrl structure.  This should be called as soon as
1236  * the admin queue is fully up and running.
1237  */
1238 int nvme_init_identify(struct nvme_ctrl *ctrl)
1239 {
1240         struct nvme_id_ctrl *id;
1241         u64 cap;
1242         int ret, page_shift;
1243         u32 max_hw_sectors;
1244
1245         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
1246         if (ret) {
1247                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
1248                 return ret;
1249         }
1250
1251         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
1252         if (ret) {
1253                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
1254                 return ret;
1255         }
1256         page_shift = NVME_CAP_MPSMIN(cap) + 12;
1257
1258         if (ctrl->vs >= NVME_VS(1, 1, 0))
1259                 ctrl->subsystem = NVME_CAP_NSSRC(cap);
1260
1261         ret = nvme_identify_ctrl(ctrl, &id);
1262         if (ret) {
1263                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
1264                 return -EIO;
1265         }
1266
1267         ctrl->vid = le16_to_cpu(id->vid);
1268         ctrl->oncs = le16_to_cpup(&id->oncs);
1269         atomic_set(&ctrl->abort_limit, id->acl + 1);
1270         ctrl->vwc = id->vwc;
1271         ctrl->cntlid = le16_to_cpup(&id->cntlid);
1272         memcpy(ctrl->serial, id->sn, sizeof(id->sn));
1273         memcpy(ctrl->model, id->mn, sizeof(id->mn));
1274         memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
1275         if (id->mdts)
1276                 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
1277         else
1278                 max_hw_sectors = UINT_MAX;
1279         ctrl->max_hw_sectors =
1280                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
1281
1282         nvme_set_queue_limits(ctrl, ctrl->admin_q);
1283         ctrl->sgls = le32_to_cpu(id->sgls);
1284         ctrl->kas = le16_to_cpu(id->kas);
1285
1286         if (ctrl->ops->is_fabrics) {
1287                 ctrl->icdoff = le16_to_cpu(id->icdoff);
1288                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
1289                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
1290                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
1291
1292                 /*
1293                  * In fabrics we need to verify the cntlid matches the
1294                  * admin connect
1295                  */
1296                 if (ctrl->cntlid != le16_to_cpu(id->cntlid))
1297                         ret = -EINVAL;
1298
1299                 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
1300                         dev_err(ctrl->dev,
1301                                 "keep-alive support is mandatory for fabrics\n");
1302                         ret = -EINVAL;
1303                 }
1304         } else {
1305                 ctrl->cntlid = le16_to_cpu(id->cntlid);
1306         }
1307
1308         kfree(id);
1309         return ret;
1310 }
1311 EXPORT_SYMBOL_GPL(nvme_init_identify);
1312
1313 static int nvme_dev_open(struct inode *inode, struct file *file)
1314 {
1315         struct nvme_ctrl *ctrl;
1316         int instance = iminor(inode);
1317         int ret = -ENODEV;
1318
1319         spin_lock(&dev_list_lock);
1320         list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
1321                 if (ctrl->instance != instance)
1322                         continue;
1323
1324                 if (!ctrl->admin_q) {
1325                         ret = -EWOULDBLOCK;
1326                         break;
1327                 }
1328                 if (!kref_get_unless_zero(&ctrl->kref))
1329                         break;
1330                 file->private_data = ctrl;
1331                 ret = 0;
1332                 break;
1333         }
1334         spin_unlock(&dev_list_lock);
1335
1336         return ret;
1337 }
1338
1339 static int nvme_dev_release(struct inode *inode, struct file *file)
1340 {
1341         nvme_put_ctrl(file->private_data);
1342         return 0;
1343 }
1344
1345 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
1346 {
1347         struct nvme_ns *ns;
1348         int ret;
1349
1350         mutex_lock(&ctrl->namespaces_mutex);
1351         if (list_empty(&ctrl->namespaces)) {
1352                 ret = -ENOTTY;
1353                 goto out_unlock;
1354         }
1355
1356         ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
1357         if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
1358                 dev_warn(ctrl->device,
1359                         "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
1360                 ret = -EINVAL;
1361                 goto out_unlock;
1362         }
1363
1364         dev_warn(ctrl->device,
1365                 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
1366         kref_get(&ns->kref);
1367         mutex_unlock(&ctrl->namespaces_mutex);
1368
1369         ret = nvme_user_cmd(ctrl, ns, argp);
1370         nvme_put_ns(ns);
1371         return ret;
1372
1373 out_unlock:
1374         mutex_unlock(&ctrl->namespaces_mutex);
1375         return ret;
1376 }
1377
1378 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
1379                 unsigned long arg)
1380 {
1381         struct nvme_ctrl *ctrl = file->private_data;
1382         void __user *argp = (void __user *)arg;
1383
1384         switch (cmd) {
1385         case NVME_IOCTL_ADMIN_CMD:
1386                 return nvme_user_cmd(ctrl, NULL, argp);
1387         case NVME_IOCTL_IO_CMD:
1388                 return nvme_dev_user_cmd(ctrl, argp);
1389         case NVME_IOCTL_RESET:
1390                 dev_warn(ctrl->device, "resetting controller\n");
1391                 return ctrl->ops->reset_ctrl(ctrl);
1392         case NVME_IOCTL_SUBSYS_RESET:
1393                 return nvme_reset_subsystem(ctrl);
1394         case NVME_IOCTL_RESCAN:
1395                 nvme_queue_scan(ctrl);
1396                 return 0;
1397         default:
1398                 return -ENOTTY;
1399         }
1400 }
1401
1402 static const struct file_operations nvme_dev_fops = {
1403         .owner          = THIS_MODULE,
1404         .open           = nvme_dev_open,
1405         .release        = nvme_dev_release,
1406         .unlocked_ioctl = nvme_dev_ioctl,
1407         .compat_ioctl   = nvme_dev_ioctl,
1408 };
1409
1410 static ssize_t nvme_sysfs_reset(struct device *dev,
1411                                 struct device_attribute *attr, const char *buf,
1412                                 size_t count)
1413 {
1414         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1415         int ret;
1416
1417         ret = ctrl->ops->reset_ctrl(ctrl);
1418         if (ret < 0)
1419                 return ret;
1420         return count;
1421 }
1422 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
1423
1424 static ssize_t nvme_sysfs_rescan(struct device *dev,
1425                                 struct device_attribute *attr, const char *buf,
1426                                 size_t count)
1427 {
1428         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1429
1430         nvme_queue_scan(ctrl);
1431         return count;
1432 }
1433 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
1434
1435 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
1436                                                                 char *buf)
1437 {
1438         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1439         struct nvme_ctrl *ctrl = ns->ctrl;
1440         int serial_len = sizeof(ctrl->serial);
1441         int model_len = sizeof(ctrl->model);
1442
1443         if (memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
1444                 return sprintf(buf, "eui.%16phN\n", ns->uuid);
1445
1446         if (memchr_inv(ns->eui, 0, sizeof(ns->eui)))
1447                 return sprintf(buf, "eui.%8phN\n", ns->eui);
1448
1449         while (ctrl->serial[serial_len - 1] == ' ')
1450                 serial_len--;
1451         while (ctrl->model[model_len - 1] == ' ')
1452                 model_len--;
1453
1454         return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", ctrl->vid,
1455                 serial_len, ctrl->serial, model_len, ctrl->model, ns->ns_id);
1456 }
1457 static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);
1458
1459 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
1460                                                                 char *buf)
1461 {
1462         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1463         return sprintf(buf, "%pU\n", ns->uuid);
1464 }
1465 static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
1466
1467 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
1468                                                                 char *buf)
1469 {
1470         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1471         return sprintf(buf, "%8phd\n", ns->eui);
1472 }
1473 static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
1474
1475 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
1476                                                                 char *buf)
1477 {
1478         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1479         return sprintf(buf, "%d\n", ns->ns_id);
1480 }
1481 static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
1482
1483 static struct attribute *nvme_ns_attrs[] = {
1484         &dev_attr_wwid.attr,
1485         &dev_attr_uuid.attr,
1486         &dev_attr_eui.attr,
1487         &dev_attr_nsid.attr,
1488         NULL,
1489 };
1490
1491 static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
1492                 struct attribute *a, int n)
1493 {
1494         struct device *dev = container_of(kobj, struct device, kobj);
1495         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1496
1497         if (a == &dev_attr_uuid.attr) {
1498                 if (!memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
1499                         return 0;
1500         }
1501         if (a == &dev_attr_eui.attr) {
1502                 if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
1503                         return 0;
1504         }
1505         return a->mode;
1506 }
1507
1508 static const struct attribute_group nvme_ns_attr_group = {
1509         .attrs          = nvme_ns_attrs,
1510         .is_visible     = nvme_ns_attrs_are_visible,
1511 };
1512
1513 #define nvme_show_str_function(field)                                           \
1514 static ssize_t  field##_show(struct device *dev,                                \
1515                             struct device_attribute *attr, char *buf)           \
1516 {                                                                               \
1517         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
1518         return sprintf(buf, "%.*s\n", (int)sizeof(ctrl->field), ctrl->field);   \
1519 }                                                                               \
1520 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
1521
1522 #define nvme_show_int_function(field)                                           \
1523 static ssize_t  field##_show(struct device *dev,                                \
1524                             struct device_attribute *attr, char *buf)           \
1525 {                                                                               \
1526         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
1527         return sprintf(buf, "%d\n", ctrl->field);       \
1528 }                                                                               \
1529 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
1530
1531 nvme_show_str_function(model);
1532 nvme_show_str_function(serial);
1533 nvme_show_str_function(firmware_rev);
1534 nvme_show_int_function(cntlid);
1535
1536 static ssize_t nvme_sysfs_delete(struct device *dev,
1537                                 struct device_attribute *attr, const char *buf,
1538                                 size_t count)
1539 {
1540         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1541
1542         if (device_remove_file_self(dev, attr))
1543                 ctrl->ops->delete_ctrl(ctrl);
1544         return count;
1545 }
1546 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
1547
1548 static ssize_t nvme_sysfs_show_transport(struct device *dev,
1549                                          struct device_attribute *attr,
1550                                          char *buf)
1551 {
1552         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1553
1554         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
1555 }
1556 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
1557
1558 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
1559                                          struct device_attribute *attr,
1560                                          char *buf)
1561 {
1562         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1563
1564         return snprintf(buf, PAGE_SIZE, "%s\n",
1565                         ctrl->ops->get_subsysnqn(ctrl));
1566 }
1567 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
1568
1569 static ssize_t nvme_sysfs_show_address(struct device *dev,
1570                                          struct device_attribute *attr,
1571                                          char *buf)
1572 {
1573         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1574
1575         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
1576 }
1577 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
1578
1579 static struct attribute *nvme_dev_attrs[] = {
1580         &dev_attr_reset_controller.attr,
1581         &dev_attr_rescan_controller.attr,
1582         &dev_attr_model.attr,
1583         &dev_attr_serial.attr,
1584         &dev_attr_firmware_rev.attr,
1585         &dev_attr_cntlid.attr,
1586         &dev_attr_delete_controller.attr,
1587         &dev_attr_transport.attr,
1588         &dev_attr_subsysnqn.attr,
1589         &dev_attr_address.attr,
1590         NULL
1591 };
1592
1593 #define CHECK_ATTR(ctrl, a, name)               \
1594         if ((a) == &dev_attr_##name.attr &&     \
1595             !(ctrl)->ops->get_##name)           \
1596                 return 0
1597
1598 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
1599                 struct attribute *a, int n)
1600 {
1601         struct device *dev = container_of(kobj, struct device, kobj);
1602         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1603
1604         if (a == &dev_attr_delete_controller.attr) {
1605                 if (!ctrl->ops->delete_ctrl)
1606                         return 0;
1607         }
1608
1609         CHECK_ATTR(ctrl, a, subsysnqn);
1610         CHECK_ATTR(ctrl, a, address);
1611
1612         return a->mode;
1613 }
1614
1615 static struct attribute_group nvme_dev_attrs_group = {
1616         .attrs          = nvme_dev_attrs,
1617         .is_visible     = nvme_dev_attrs_are_visible,
1618 };
1619
1620 static const struct attribute_group *nvme_dev_attr_groups[] = {
1621         &nvme_dev_attrs_group,
1622         NULL,
1623 };
1624
1625 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
1626 {
1627         struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
1628         struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
1629
1630         return nsa->ns_id - nsb->ns_id;
1631 }
1632
1633 static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1634 {
1635         struct nvme_ns *ns, *ret = NULL;
1636
1637         mutex_lock(&ctrl->namespaces_mutex);
1638         list_for_each_entry(ns, &ctrl->namespaces, list) {
1639                 if (ns->ns_id == nsid) {
1640                         kref_get(&ns->kref);
1641                         ret = ns;
1642                         break;
1643                 }
1644                 if (ns->ns_id > nsid)
1645                         break;
1646         }
1647         mutex_unlock(&ctrl->namespaces_mutex);
1648         return ret;
1649 }
1650
1651 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1652 {
1653         struct nvme_ns *ns;
1654         struct gendisk *disk;
1655         struct nvme_id_ns *id;
1656         char disk_name[DISK_NAME_LEN];
1657         int node = dev_to_node(ctrl->dev);
1658
1659         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
1660         if (!ns)
1661                 return;
1662
1663         ns->instance = ida_simple_get(&ctrl->ns_ida, 1, 0, GFP_KERNEL);
1664         if (ns->instance < 0)
1665                 goto out_free_ns;
1666
1667         ns->queue = blk_mq_init_queue(ctrl->tagset);
1668         if (IS_ERR(ns->queue))
1669                 goto out_release_instance;
1670         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1671         ns->queue->queuedata = ns;
1672         ns->ctrl = ctrl;
1673
1674         kref_init(&ns->kref);
1675         ns->ns_id = nsid;
1676         ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
1677
1678         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1679         nvme_set_queue_limits(ctrl, ns->queue);
1680
1681         sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
1682
1683         if (nvme_revalidate_ns(ns, &id))
1684                 goto out_free_queue;
1685
1686         if (nvme_nvm_ns_supported(ns, id) &&
1687                                 nvme_nvm_register(ns, disk_name, node)) {
1688                 dev_warn(ctrl->dev, "%s: LightNVM init failure\n", __func__);
1689                 goto out_free_id;
1690         }
1691
1692         disk = alloc_disk_node(0, node);
1693         if (!disk)
1694                 goto out_free_id;
1695
1696         disk->fops = &nvme_fops;
1697         disk->private_data = ns;
1698         disk->queue = ns->queue;
1699         disk->flags = GENHD_FL_EXT_DEVT;
1700         memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
1701         ns->disk = disk;
1702
1703         __nvme_revalidate_disk(disk, id);
1704
1705         mutex_lock(&ctrl->namespaces_mutex);
1706         list_add_tail(&ns->list, &ctrl->namespaces);
1707         mutex_unlock(&ctrl->namespaces_mutex);
1708
1709         kref_get(&ctrl->kref);
1710
1711         kfree(id);
1712
1713         device_add_disk(ctrl->device, ns->disk);
1714         if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
1715                                         &nvme_ns_attr_group))
1716                 pr_warn("%s: failed to create sysfs group for identification\n",
1717                         ns->disk->disk_name);
1718         if (ns->ndev && nvme_nvm_register_sysfs(ns))
1719                 pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
1720                         ns->disk->disk_name);
1721         return;
1722  out_free_id:
1723         kfree(id);
1724  out_free_queue:
1725         blk_cleanup_queue(ns->queue);
1726  out_release_instance:
1727         ida_simple_remove(&ctrl->ns_ida, ns->instance);
1728  out_free_ns:
1729         kfree(ns);
1730 }
1731
1732 static void nvme_ns_remove(struct nvme_ns *ns)
1733 {
1734         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
1735                 return;
1736
1737         if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
1738                 if (blk_get_integrity(ns->disk))
1739                         blk_integrity_unregister(ns->disk);
1740                 sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
1741                                         &nvme_ns_attr_group);
1742                 if (ns->ndev)
1743                         nvme_nvm_unregister_sysfs(ns);
1744                 del_gendisk(ns->disk);
1745                 blk_mq_abort_requeue_list(ns->queue);
1746                 blk_cleanup_queue(ns->queue);
1747         }
1748
1749         mutex_lock(&ns->ctrl->namespaces_mutex);
1750         list_del_init(&ns->list);
1751         mutex_unlock(&ns->ctrl->namespaces_mutex);
1752
1753         nvme_put_ns(ns);
1754 }
1755
1756 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1757 {
1758         struct nvme_ns *ns;
1759
1760         ns = nvme_find_get_ns(ctrl, nsid);
1761         if (ns) {
1762                 if (ns->disk && revalidate_disk(ns->disk))
1763                         nvme_ns_remove(ns);
1764                 nvme_put_ns(ns);
1765         } else
1766                 nvme_alloc_ns(ctrl, nsid);
1767 }
1768
1769 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
1770                                         unsigned nsid)
1771 {
1772         struct nvme_ns *ns, *next;
1773
1774         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
1775                 if (ns->ns_id > nsid)
1776                         nvme_ns_remove(ns);
1777         }
1778 }
1779
1780 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
1781 {
1782         struct nvme_ns *ns;
1783         __le32 *ns_list;
1784         unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
1785         int ret = 0;
1786
1787         ns_list = kzalloc(0x1000, GFP_KERNEL);
1788         if (!ns_list)
1789                 return -ENOMEM;
1790
1791         for (i = 0; i < num_lists; i++) {
1792                 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
1793                 if (ret)
1794                         goto free;
1795
1796                 for (j = 0; j < min(nn, 1024U); j++) {
1797                         nsid = le32_to_cpu(ns_list[j]);
1798                         if (!nsid)
1799                                 goto out;
1800
1801                         nvme_validate_ns(ctrl, nsid);
1802
1803                         while (++prev < nsid) {
1804                                 ns = nvme_find_get_ns(ctrl, prev);
1805                                 if (ns) {
1806                                         nvme_ns_remove(ns);
1807                                         nvme_put_ns(ns);
1808                                 }
1809                         }
1810                 }
1811                 nn -= j;
1812         }
1813  out:
1814         nvme_remove_invalid_namespaces(ctrl, prev);
1815  free:
1816         kfree(ns_list);
1817         return ret;
1818 }
1819
1820 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
1821 {
1822         unsigned i;
1823
1824         for (i = 1; i <= nn; i++)
1825                 nvme_validate_ns(ctrl, i);
1826
1827         nvme_remove_invalid_namespaces(ctrl, nn);
1828 }
1829
1830 static void nvme_scan_work(struct work_struct *work)
1831 {
1832         struct nvme_ctrl *ctrl =
1833                 container_of(work, struct nvme_ctrl, scan_work);
1834         struct nvme_id_ctrl *id;
1835         unsigned nn;
1836
1837         if (ctrl->state != NVME_CTRL_LIVE)
1838                 return;
1839
1840         if (nvme_identify_ctrl(ctrl, &id))
1841                 return;
1842
1843         nn = le32_to_cpu(id->nn);
1844         if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1845             !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
1846                 if (!nvme_scan_ns_list(ctrl, nn))
1847                         goto done;
1848         }
1849         nvme_scan_ns_sequential(ctrl, nn);
1850  done:
1851         mutex_lock(&ctrl->namespaces_mutex);
1852         list_sort(NULL, &ctrl->namespaces, ns_cmp);
1853         mutex_unlock(&ctrl->namespaces_mutex);
1854         kfree(id);
1855 }
1856
1857 void nvme_queue_scan(struct nvme_ctrl *ctrl)
1858 {
1859         /*
1860          * Do not queue new scan work when a controller is reset during
1861          * removal.
1862          */
1863         if (ctrl->state == NVME_CTRL_LIVE)
1864                 schedule_work(&ctrl->scan_work);
1865 }
1866 EXPORT_SYMBOL_GPL(nvme_queue_scan);
1867
1868 /*
1869  * This function iterates the namespace list unlocked to allow recovery from
1870  * controller failure. It is up to the caller to ensure the namespace list is
1871  * not modified by scan work while this function is executing.
1872  */
1873 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
1874 {
1875         struct nvme_ns *ns, *next;
1876
1877         /*
1878          * The dead states indicates the controller was not gracefully
1879          * disconnected. In that case, we won't be able to flush any data while
1880          * removing the namespaces' disks; fail all the queues now to avoid
1881          * potentially having to clean up the failed sync later.
1882          */
1883         if (ctrl->state == NVME_CTRL_DEAD)
1884                 nvme_kill_queues(ctrl);
1885
1886         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
1887                 nvme_ns_remove(ns);
1888 }
1889 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
1890
1891 static void nvme_async_event_work(struct work_struct *work)
1892 {
1893         struct nvme_ctrl *ctrl =
1894                 container_of(work, struct nvme_ctrl, async_event_work);
1895
1896         spin_lock_irq(&ctrl->lock);
1897         while (ctrl->event_limit > 0) {
1898                 int aer_idx = --ctrl->event_limit;
1899
1900                 spin_unlock_irq(&ctrl->lock);
1901                 ctrl->ops->submit_async_event(ctrl, aer_idx);
1902                 spin_lock_irq(&ctrl->lock);
1903         }
1904         spin_unlock_irq(&ctrl->lock);
1905 }
1906
1907 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
1908                 union nvme_result *res)
1909 {
1910         u32 result = le32_to_cpu(res->u32);
1911         bool done = true;
1912
1913         switch (le16_to_cpu(status) >> 1) {
1914         case NVME_SC_SUCCESS:
1915                 done = false;
1916                 /*FALLTHRU*/
1917         case NVME_SC_ABORT_REQ:
1918                 ++ctrl->event_limit;
1919                 schedule_work(&ctrl->async_event_work);
1920                 break;
1921         default:
1922                 break;
1923         }
1924
1925         if (done)
1926                 return;
1927
1928         switch (result & 0xff07) {
1929         case NVME_AER_NOTICE_NS_CHANGED:
1930                 dev_info(ctrl->device, "rescanning\n");
1931                 nvme_queue_scan(ctrl);
1932                 break;
1933         default:
1934                 dev_warn(ctrl->device, "async event result %08x\n", result);
1935         }
1936 }
1937 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
1938
1939 void nvme_queue_async_events(struct nvme_ctrl *ctrl)
1940 {
1941         ctrl->event_limit = NVME_NR_AERS;
1942         schedule_work(&ctrl->async_event_work);
1943 }
1944 EXPORT_SYMBOL_GPL(nvme_queue_async_events);
1945
1946 static DEFINE_IDA(nvme_instance_ida);
1947
1948 static int nvme_set_instance(struct nvme_ctrl *ctrl)
1949 {
1950         int instance, error;
1951
1952         do {
1953                 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
1954                         return -ENODEV;
1955
1956                 spin_lock(&dev_list_lock);
1957                 error = ida_get_new(&nvme_instance_ida, &instance);
1958                 spin_unlock(&dev_list_lock);
1959         } while (error == -EAGAIN);
1960
1961         if (error)
1962                 return -ENODEV;
1963
1964         ctrl->instance = instance;
1965         return 0;
1966 }
1967
1968 static void nvme_release_instance(struct nvme_ctrl *ctrl)
1969 {
1970         spin_lock(&dev_list_lock);
1971         ida_remove(&nvme_instance_ida, ctrl->instance);
1972         spin_unlock(&dev_list_lock);
1973 }
1974
1975 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
1976 {
1977         flush_work(&ctrl->async_event_work);
1978         flush_work(&ctrl->scan_work);
1979         nvme_remove_namespaces(ctrl);
1980
1981         device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
1982
1983         spin_lock(&dev_list_lock);
1984         list_del(&ctrl->node);
1985         spin_unlock(&dev_list_lock);
1986 }
1987 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
1988
1989 static void nvme_free_ctrl(struct kref *kref)
1990 {
1991         struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
1992
1993         put_device(ctrl->device);
1994         nvme_release_instance(ctrl);
1995         ida_destroy(&ctrl->ns_ida);
1996
1997         ctrl->ops->free_ctrl(ctrl);
1998 }
1999
2000 void nvme_put_ctrl(struct nvme_ctrl *ctrl)
2001 {
2002         kref_put(&ctrl->kref, nvme_free_ctrl);
2003 }
2004 EXPORT_SYMBOL_GPL(nvme_put_ctrl);
2005
2006 /*
2007  * Initialize a NVMe controller structures.  This needs to be called during
2008  * earliest initialization so that we have the initialized structured around
2009  * during probing.
2010  */
2011 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
2012                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
2013 {
2014         int ret;
2015
2016         ctrl->state = NVME_CTRL_NEW;
2017         spin_lock_init(&ctrl->lock);
2018         INIT_LIST_HEAD(&ctrl->namespaces);
2019         mutex_init(&ctrl->namespaces_mutex);
2020         kref_init(&ctrl->kref);
2021         ctrl->dev = dev;
2022         ctrl->ops = ops;
2023         ctrl->quirks = quirks;
2024         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
2025         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
2026
2027         ret = nvme_set_instance(ctrl);
2028         if (ret)
2029                 goto out;
2030
2031         ctrl->device = device_create_with_groups(nvme_class, ctrl->dev,
2032                                 MKDEV(nvme_char_major, ctrl->instance),
2033                                 ctrl, nvme_dev_attr_groups,
2034                                 "nvme%d", ctrl->instance);
2035         if (IS_ERR(ctrl->device)) {
2036                 ret = PTR_ERR(ctrl->device);
2037                 goto out_release_instance;
2038         }
2039         get_device(ctrl->device);
2040         ida_init(&ctrl->ns_ida);
2041
2042         spin_lock(&dev_list_lock);
2043         list_add_tail(&ctrl->node, &nvme_ctrl_list);
2044         spin_unlock(&dev_list_lock);
2045
2046         return 0;
2047 out_release_instance:
2048         nvme_release_instance(ctrl);
2049 out:
2050         return ret;
2051 }
2052 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
2053
2054 /**
2055  * nvme_kill_queues(): Ends all namespace queues
2056  * @ctrl: the dead controller that needs to end
2057  *
2058  * Call this function when the driver determines it is unable to get the
2059  * controller in a state capable of servicing IO.
2060  */
2061 void nvme_kill_queues(struct nvme_ctrl *ctrl)
2062 {
2063         struct nvme_ns *ns;
2064
2065         mutex_lock(&ctrl->namespaces_mutex);
2066         list_for_each_entry(ns, &ctrl->namespaces, list) {
2067                 /*
2068                  * Revalidating a dead namespace sets capacity to 0. This will
2069                  * end buffered writers dirtying pages that can't be synced.
2070                  */
2071                 if (ns->disk && !test_and_set_bit(NVME_NS_DEAD, &ns->flags))
2072                         revalidate_disk(ns->disk);
2073
2074                 blk_set_queue_dying(ns->queue);
2075                 blk_mq_abort_requeue_list(ns->queue);
2076                 blk_mq_start_stopped_hw_queues(ns->queue, true);
2077         }
2078         mutex_unlock(&ctrl->namespaces_mutex);
2079 }
2080 EXPORT_SYMBOL_GPL(nvme_kill_queues);
2081
2082 void nvme_stop_queues(struct nvme_ctrl *ctrl)
2083 {
2084         struct nvme_ns *ns;
2085
2086         mutex_lock(&ctrl->namespaces_mutex);
2087         list_for_each_entry(ns, &ctrl->namespaces, list)
2088                 blk_mq_quiesce_queue(ns->queue);
2089         mutex_unlock(&ctrl->namespaces_mutex);
2090 }
2091 EXPORT_SYMBOL_GPL(nvme_stop_queues);
2092
2093 void nvme_start_queues(struct nvme_ctrl *ctrl)
2094 {
2095         struct nvme_ns *ns;
2096
2097         mutex_lock(&ctrl->namespaces_mutex);
2098         list_for_each_entry(ns, &ctrl->namespaces, list) {
2099                 blk_mq_start_stopped_hw_queues(ns->queue, true);
2100                 blk_mq_kick_requeue_list(ns->queue);
2101         }
2102         mutex_unlock(&ctrl->namespaces_mutex);
2103 }
2104 EXPORT_SYMBOL_GPL(nvme_start_queues);
2105
2106 int __init nvme_core_init(void)
2107 {
2108         int result;
2109
2110         result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
2111                                                         &nvme_dev_fops);
2112         if (result < 0)
2113                 return result;
2114         else if (result > 0)
2115                 nvme_char_major = result;
2116
2117         nvme_class = class_create(THIS_MODULE, "nvme");
2118         if (IS_ERR(nvme_class)) {
2119                 result = PTR_ERR(nvme_class);
2120                 goto unregister_chrdev;
2121         }
2122
2123         return 0;
2124
2125  unregister_chrdev:
2126         __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
2127         return result;
2128 }
2129
2130 void nvme_core_exit(void)
2131 {
2132         class_destroy(nvme_class);
2133         __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
2134 }
2135
2136 MODULE_LICENSE("GPL");
2137 MODULE_VERSION("1.0");
2138 module_init(nvme_core_init);
2139 module_exit(nvme_core_exit);