drivers/nvme/target/core.c

   1 /*
   2  * Common code for the NVMe target.
   3  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  15 #include <linux/module.h>
  16 #include <linux/random.h>
  17 #include "nvmet.h"
  18
  19 static struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX];
  20 static DEFINE_IDA(cntlid_ida);
  21
  22 /*
  23  * This read/write semaphore is used to synchronize access to configuration
  24  * information on a target system that will result in discovery log page
  25  * information change for at least one host.
  26  * The full list of resources to protected by this semaphore is:
  27  *
  28  *  - subsystems list
  29  *  - per-subsystem allowed hosts list
  30  *  - allow_any_host subsystem attribute
  31  *  - nvmet_genctr
  32  *  - the nvmet_transports array
  33  *
  34  * When updating any of those lists/structures write lock should be obtained,
  35  * while when reading (popolating discovery log page or checking host-subsystem
  36  * link) read lock is obtained to allow concurrent reads.
  37  */
  38 DECLARE_RWSEM(nvmet_config_sem);
  39
  40 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
  41                 const char *subsysnqn);
  42
  43 u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf,
  44                 size_t len)
  45 {
  46         if (sg_pcopy_from_buffer(req->sg, req->sg_cnt, buf, len, off) != len)
  47                 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
  48         return 0;
  49 }
  50
  51 u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, size_t len)
  52 {
  53         if (sg_pcopy_to_buffer(req->sg, req->sg_cnt, buf, len, off) != len)
  54                 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR;
  55         return 0;
  56 }
  57
  58 static u32 nvmet_async_event_result(struct nvmet_async_event *aen)
  59 {
  60         return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16);
  61 }
  62
  63 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl)
  64 {
  65         struct nvmet_req *req;
  66
  67         while (1) {
  68                 mutex_lock(&ctrl->lock);
  69                 if (!ctrl->nr_async_event_cmds) {
  70                         mutex_unlock(&ctrl->lock);
  71                         return;
  72                 }
  73
  74                 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
  75                 mutex_unlock(&ctrl->lock);
  76                 nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR);
  77         }
  78 }
  79
  80 static void nvmet_async_event_work(struct work_struct *work)
  81 {
  82         struct nvmet_ctrl *ctrl =
  83                 container_of(work, struct nvmet_ctrl, async_event_work);
  84         struct nvmet_async_event *aen;
  85         struct nvmet_req *req;
  86
  87         while (1) {
  88                 mutex_lock(&ctrl->lock);
  89                 aen = list_first_entry_or_null(&ctrl->async_events,
  90                                 struct nvmet_async_event, entry);
  91                 if (!aen || !ctrl->nr_async_event_cmds) {
  92                         mutex_unlock(&ctrl->lock);
  93                         return;
  94                 }
  95
  96                 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
  97                 nvmet_set_result(req, nvmet_async_event_result(aen));
  98
  99                 list_del(&aen->entry);
 100                 kfree(aen);
 101
 102                 mutex_unlock(&ctrl->lock);
 103                 nvmet_req_complete(req, 0);
 104         }
 105 }
 106
 107 static void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
 108                 u8 event_info, u8 log_page)
 109 {
 110         struct nvmet_async_event *aen;
 111
 112         aen = kmalloc(sizeof(*aen), GFP_KERNEL);
 113         if (!aen)
 114                 return;
 115
 116         aen->event_type = event_type;
 117         aen->event_info = event_info;
 118         aen->log_page = log_page;
 119
 120         mutex_lock(&ctrl->lock);
 121         list_add_tail(&aen->entry, &ctrl->async_events);
 122         mutex_unlock(&ctrl->lock);
 123
 124         schedule_work(&ctrl->async_event_work);
 125 }
 126
 127 int nvmet_register_transport(struct nvmet_fabrics_ops *ops)
 128 {
 129         int ret = 0;
 130
 131         down_write(&nvmet_config_sem);
 132         if (nvmet_transports[ops->type])
 133                 ret = -EINVAL;
 134         else
 135                 nvmet_transports[ops->type] = ops;
 136         up_write(&nvmet_config_sem);
 137
 138         return ret;
 139 }
 140 EXPORT_SYMBOL_GPL(nvmet_register_transport);
 141
 142 void nvmet_unregister_transport(struct nvmet_fabrics_ops *ops)
 143 {
 144         down_write(&nvmet_config_sem);
 145         nvmet_transports[ops->type] = NULL;
 146         up_write(&nvmet_config_sem);
 147 }
 148 EXPORT_SYMBOL_GPL(nvmet_unregister_transport);
 149
 150 int nvmet_enable_port(struct nvmet_port *port)
 151 {
 152         struct nvmet_fabrics_ops *ops;
 153         int ret;
 154
 155         lockdep_assert_held(&nvmet_config_sem);
 156
 157         ops = nvmet_transports[port->disc_addr.trtype];
 158         if (!ops) {
 159                 up_write(&nvmet_config_sem);
 160                 request_module("nvmet-transport-%d", port->disc_addr.trtype);
 161                 down_write(&nvmet_config_sem);
 162                 ops = nvmet_transports[port->disc_addr.trtype];
 163                 if (!ops) {
 164                         pr_err("transport type %d not supported\n",
 165                                 port->disc_addr.trtype);
 166                         return -EINVAL;
 167                 }
 168         }
 169
 170         if (!try_module_get(ops->owner))
 171                 return -EINVAL;
 172
 173         ret = ops->add_port(port);
 174         if (ret) {
 175                 module_put(ops->owner);
 176                 return ret;
 177         }
 178
 179         port->enabled = true;
 180         return 0;
 181 }
 182
 183 void nvmet_disable_port(struct nvmet_port *port)
 184 {
 185         struct nvmet_fabrics_ops *ops;
 186
 187         lockdep_assert_held(&nvmet_config_sem);
 188
 189         port->enabled = false;
 190
 191         ops = nvmet_transports[port->disc_addr.trtype];
 192         ops->remove_port(port);
 193         module_put(ops->owner);
 194 }
 195
 196 static void nvmet_keep_alive_timer(struct work_struct *work)
 197 {
 198         struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work),
 199                         struct nvmet_ctrl, ka_work);
 200
 201         pr_err("ctrl %d keep-alive timer (%d seconds) expired!\n",
 202                 ctrl->cntlid, ctrl->kato);
 203
 204         nvmet_ctrl_fatal_error(ctrl);
 205 }
 206
 207 static void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl)
 208 {
 209         pr_debug("ctrl %d start keep-alive timer for %d secs\n",
 210                 ctrl->cntlid, ctrl->kato);
 211
 212         INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer);
 213         schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
 214 }
 215
 216 static void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl)
 217 {
 218         pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid);
 219
 220         cancel_delayed_work_sync(&ctrl->ka_work);
 221 }
 222
 223 static struct nvmet_ns *__nvmet_find_namespace(struct nvmet_ctrl *ctrl,
 224                 __le32 nsid)
 225 {
 226         struct nvmet_ns *ns;
 227
 228         list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) {
 229                 if (ns->nsid == le32_to_cpu(nsid))
 230                         return ns;
 231         }
 232
 233         return NULL;
 234 }
 235
 236 struct nvmet_ns *nvmet_find_namespace(struct nvmet_ctrl *ctrl, __le32 nsid)
 237 {
 238         struct nvmet_ns *ns;
 239
 240         rcu_read_lock();
 241         ns = __nvmet_find_namespace(ctrl, nsid);
 242         if (ns)
 243                 percpu_ref_get(&ns->ref);
 244         rcu_read_unlock();
 245
 246         return ns;
 247 }
 248
 249 static void nvmet_destroy_namespace(struct percpu_ref *ref)
 250 {
 251         struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref);
 252
 253         complete(&ns->disable_done);
 254 }
 255
 256 void nvmet_put_namespace(struct nvmet_ns *ns)
 257 {
 258         percpu_ref_put(&ns->ref);
 259 }
 260
 261 int nvmet_ns_enable(struct nvmet_ns *ns)
 262 {
 263         struct nvmet_subsys *subsys = ns->subsys;
 264         struct nvmet_ctrl *ctrl;
 265         int ret = 0;
 266
 267         mutex_lock(&subsys->lock);
 268         if (ns->enabled)
 269                 goto out_unlock;
 270
 271         ns->bdev = blkdev_get_by_path(ns->device_path, FMODE_READ | FMODE_WRITE,
 272                         NULL);
 273         if (IS_ERR(ns->bdev)) {
 274                 pr_err("nvmet: failed to open block device %s: (%ld)\n",
 275                         ns->device_path, PTR_ERR(ns->bdev));
 276                 ret = PTR_ERR(ns->bdev);
 277                 ns->bdev = NULL;
 278                 goto out_unlock;
 279         }
 280
 281         ns->size = i_size_read(ns->bdev->bd_inode);
 282         ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev));
 283
 284         ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace,
 285                                 0, GFP_KERNEL);
 286         if (ret)
 287                 goto out_blkdev_put;
 288
 289         if (ns->nsid > subsys->max_nsid)
 290                 subsys->max_nsid = ns->nsid;
 291
 292         /*
 293          * The namespaces list needs to be sorted to simplify the implementation
 294          * of the Identify Namepace List subcommand.
 295          */
 296         if (list_empty(&subsys->namespaces)) {
 297                 list_add_tail_rcu(&ns->dev_link, &subsys->namespaces);
 298         } else {
 299                 struct nvmet_ns *old;
 300
 301                 list_for_each_entry_rcu(old, &subsys->namespaces, dev_link) {
 302                         BUG_ON(ns->nsid == old->nsid);
 303                         if (ns->nsid < old->nsid)
 304                                 break;
 305                 }
 306
 307                 list_add_tail_rcu(&ns->dev_link, &old->dev_link);
 308         }
 309
 310         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
 311                 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0);
 312
 313         ns->enabled = true;
 314         ret = 0;
 315 out_unlock:
 316         mutex_unlock(&subsys->lock);
 317         return ret;
 318 out_blkdev_put:
 319         blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ);
 320         ns->bdev = NULL;
 321         goto out_unlock;
 322 }
 323
 324 void nvmet_ns_disable(struct nvmet_ns *ns)
 325 {
 326         struct nvmet_subsys *subsys = ns->subsys;
 327         struct nvmet_ctrl *ctrl;
 328
 329         mutex_lock(&subsys->lock);
 330         if (!ns->enabled)
 331                 goto out_unlock;
 332
 333         ns->enabled = false;
 334         list_del_rcu(&ns->dev_link);
 335         mutex_unlock(&subsys->lock);
 336
 337         /*
 338          * Now that we removed the namespaces from the lookup list, we
 339          * can kill the per_cpu ref and wait for any remaining references
 340          * to be dropped, as well as a RCU grace period for anyone only
 341          * using the namepace under rcu_read_lock().  Note that we can't
 342          * use call_rcu here as we need to ensure the namespaces have
 343          * been fully destroyed before unloading the module.
 344          */
 345         percpu_ref_kill(&ns->ref);
 346         synchronize_rcu();
 347         wait_for_completion(&ns->disable_done);
 348         percpu_ref_exit(&ns->ref);
 349
 350         mutex_lock(&subsys->lock);
 351         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
 352                 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0);
 353
 354         if (ns->bdev)
 355                 blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ);
 356 out_unlock:
 357         mutex_unlock(&subsys->lock);
 358 }
 359
 360 void nvmet_ns_free(struct nvmet_ns *ns)
 361 {
 362         nvmet_ns_disable(ns);
 363
 364         kfree(ns->device_path);
 365         kfree(ns);
 366 }
 367
 368 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid)
 369 {
 370         struct nvmet_ns *ns;
 371
 372         ns = kzalloc(sizeof(*ns), GFP_KERNEL);
 373         if (!ns)
 374                 return NULL;
 375
 376         INIT_LIST_HEAD(&ns->dev_link);
 377         init_completion(&ns->disable_done);
 378
 379         ns->nsid = nsid;
 380         ns->subsys = subsys;
 381
 382         return ns;
 383 }
 384
 385 static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
 386 {
 387         if (status)
 388                 nvmet_set_status(req, status);
 389
 390         /* XXX: need to fill in something useful for sq_head */
 391         req->rsp->sq_head = 0;
 392         if (likely(req->sq)) /* may happen during early failure */
 393                 req->rsp->sq_id = cpu_to_le16(req->sq->qid);
 394         req->rsp->command_id = req->cmd->common.command_id;
 395
 396         if (req->ns)
 397                 nvmet_put_namespace(req->ns);
 398         req->ops->queue_response(req);
 399 }
 400
 401 void nvmet_req_complete(struct nvmet_req *req, u16 status)
 402 {
 403         __nvmet_req_complete(req, status);
 404         percpu_ref_put(&req->sq->ref);
 405 }
 406 EXPORT_SYMBOL_GPL(nvmet_req_complete);
 407
 408 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq,
 409                 u16 qid, u16 size)
 410 {
 411         cq->qid = qid;
 412         cq->size = size;
 413
 414         ctrl->cqs[qid] = cq;
 415 }
 416
 417 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq,
 418                 u16 qid, u16 size)
 419 {
 420         sq->qid = qid;
 421         sq->size = size;
 422
 423         ctrl->sqs[qid] = sq;
 424 }
 425
 426 void nvmet_sq_destroy(struct nvmet_sq *sq)
 427 {
 428         /*
 429          * If this is the admin queue, complete all AERs so that our
 430          * queue doesn't have outstanding requests on it.
 431          */
 432         if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq)
 433                 nvmet_async_events_free(sq->ctrl);
 434         percpu_ref_kill(&sq->ref);
 435         wait_for_completion(&sq->free_done);
 436         percpu_ref_exit(&sq->ref);
 437
 438         if (sq->ctrl) {
 439                 nvmet_ctrl_put(sq->ctrl);
 440                 sq->ctrl = NULL; /* allows reusing the queue later */
 441         }
 442 }
 443 EXPORT_SYMBOL_GPL(nvmet_sq_destroy);
 444
 445 static void nvmet_sq_free(struct percpu_ref *ref)
 446 {
 447         struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref);
 448
 449         complete(&sq->free_done);
 450 }
 451
 452 int nvmet_sq_init(struct nvmet_sq *sq)
 453 {
 454         int ret;
 455
 456         ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL);
 457         if (ret) {
 458                 pr_err("percpu_ref init failed!\n");
 459                 return ret;
 460         }
 461         init_completion(&sq->free_done);
 462
 463         return 0;
 464 }
 465 EXPORT_SYMBOL_GPL(nvmet_sq_init);
 466
 467 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
 468                 struct nvmet_sq *sq, struct nvmet_fabrics_ops *ops)
 469 {
 470         u8 flags = req->cmd->common.flags;
 471         u16 status;
 472
 473         req->cq = cq;
 474         req->sq = sq;
 475         req->ops = ops;
 476         req->sg = NULL;
 477         req->sg_cnt = 0;
 478         req->rsp->status = 0;
 479
 480         /* no support for fused commands yet */
 481         if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) {
 482                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 483                 goto fail;
 484         }
 485
 486         /* either variant of SGLs is fine, as we don't support metadata */
 487         if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF &&
 488                      (flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METASEG)) {
 489                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 490                 goto fail;
 491         }
 492
 493         if (unlikely(!req->sq->ctrl))
 494                 /* will return an error for any Non-connect command: */
 495                 status = nvmet_parse_connect_cmd(req);
 496         else if (likely(req->sq->qid != 0))
 497                 status = nvmet_parse_io_cmd(req);
 498         else if (req->cmd->common.opcode == nvme_fabrics_command)
 499                 status = nvmet_parse_fabrics_cmd(req);
 500         else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC)
 501                 status = nvmet_parse_discovery_cmd(req);
 502         else
 503                 status = nvmet_parse_admin_cmd(req);
 504
 505         if (status)
 506                 goto fail;
 507
 508         if (unlikely(!percpu_ref_tryget_live(&sq->ref))) {
 509                 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 510                 goto fail;
 511         }
 512
 513         return true;
 514
 515 fail:
 516         __nvmet_req_complete(req, status);
 517         return false;
 518 }
 519 EXPORT_SYMBOL_GPL(nvmet_req_init);
 520
 521 static inline bool nvmet_cc_en(u32 cc)
 522 {
 523         return cc & 0x1;
 524 }
 525
 526 static inline u8 nvmet_cc_css(u32 cc)
 527 {
 528         return (cc >> 4) & 0x7;
 529 }
 530
 531 static inline u8 nvmet_cc_mps(u32 cc)
 532 {
 533         return (cc >> 7) & 0xf;
 534 }
 535
 536 static inline u8 nvmet_cc_ams(u32 cc)
 537 {
 538         return (cc >> 11) & 0x7;
 539 }
 540
 541 static inline u8 nvmet_cc_shn(u32 cc)
 542 {
 543         return (cc >> 14) & 0x3;
 544 }
 545
 546 static inline u8 nvmet_cc_iosqes(u32 cc)
 547 {
 548         return (cc >> 16) & 0xf;
 549 }
 550
 551 static inline u8 nvmet_cc_iocqes(u32 cc)
 552 {
 553         return (cc >> 20) & 0xf;
 554 }
 555
 556 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl)
 557 {
 558         lockdep_assert_held(&ctrl->lock);
 559
 560         if (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES ||
 561             nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES ||
 562             nvmet_cc_mps(ctrl->cc) != 0 ||
 563             nvmet_cc_ams(ctrl->cc) != 0 ||
 564             nvmet_cc_css(ctrl->cc) != 0) {
 565                 ctrl->csts = NVME_CSTS_CFS;
 566                 return;
 567         }
 568
 569         ctrl->csts = NVME_CSTS_RDY;
 570 }
 571
 572 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl)
 573 {
 574         lockdep_assert_held(&ctrl->lock);
 575
 576         /* XXX: tear down queues? */
 577         ctrl->csts &= ~NVME_CSTS_RDY;
 578         ctrl->cc = 0;
 579 }
 580
 581 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new)
 582 {
 583         u32 old;
 584
 585         mutex_lock(&ctrl->lock);
 586         old = ctrl->cc;
 587         ctrl->cc = new;
 588
 589         if (nvmet_cc_en(new) && !nvmet_cc_en(old))
 590                 nvmet_start_ctrl(ctrl);
 591         if (!nvmet_cc_en(new) && nvmet_cc_en(old))
 592                 nvmet_clear_ctrl(ctrl);
 593         if (nvmet_cc_shn(new) && !nvmet_cc_shn(old)) {
 594                 nvmet_clear_ctrl(ctrl);
 595                 ctrl->csts |= NVME_CSTS_SHST_CMPLT;
 596         }
 597         if (!nvmet_cc_shn(new) && nvmet_cc_shn(old))
 598                 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT;
 599         mutex_unlock(&ctrl->lock);
 600 }
 601
 602 static void nvmet_init_cap(struct nvmet_ctrl *ctrl)
 603 {
 604         /* command sets supported: NVMe command set: */
 605         ctrl->cap = (1ULL << 37);
 606         /* CC.EN timeout in 500msec units: */
 607         ctrl->cap |= (15ULL << 24);
 608         /* maximum queue entries supported: */
 609         ctrl->cap |= NVMET_QUEUE_SIZE - 1;
 610 }
 611
 612 u16 nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid,
 613                 struct nvmet_req *req, struct nvmet_ctrl **ret)
 614 {
 615         struct nvmet_subsys *subsys;
 616         struct nvmet_ctrl *ctrl;
 617         u16 status = 0;
 618
 619         subsys = nvmet_find_get_subsys(req->port, subsysnqn);
 620         if (!subsys) {
 621                 pr_warn("connect request for invalid subsystem %s!\n",
 622                         subsysnqn);
 623                 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
 624                 return NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 625         }
 626
 627         mutex_lock(&subsys->lock);
 628         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
 629                 if (ctrl->cntlid == cntlid) {
 630                         if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) {
 631                                 pr_warn("hostnqn mismatch.\n");
 632                                 continue;
 633                         }
 634                         if (!kref_get_unless_zero(&ctrl->ref))
 635                                 continue;
 636
 637                         *ret = ctrl;
 638                         goto out;
 639                 }
 640         }
 641
 642         pr_warn("could not find controller %d for subsys %s / host %s\n",
 643                 cntlid, subsysnqn, hostnqn);
 644         req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid);
 645         status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 646
 647 out:
 648         mutex_unlock(&subsys->lock);
 649         nvmet_subsys_put(subsys);
 650         return status;
 651 }
 652
 653 static bool __nvmet_host_allowed(struct nvmet_subsys *subsys,
 654                 const char *hostnqn)
 655 {
 656         struct nvmet_host_link *p;
 657
 658         if (subsys->allow_any_host)
 659                 return true;
 660
 661         list_for_each_entry(p, &subsys->hosts, entry) {
 662                 if (!strcmp(nvmet_host_name(p->host), hostnqn))
 663                         return true;
 664         }
 665
 666         return false;
 667 }
 668
 669 static bool nvmet_host_discovery_allowed(struct nvmet_req *req,
 670                 const char *hostnqn)
 671 {
 672         struct nvmet_subsys_link *s;
 673
 674         list_for_each_entry(s, &req->port->subsystems, entry) {
 675                 if (__nvmet_host_allowed(s->subsys, hostnqn))
 676                         return true;
 677         }
 678
 679         return false;
 680 }
 681
 682 bool nvmet_host_allowed(struct nvmet_req *req, struct nvmet_subsys *subsys,
 683                 const char *hostnqn)
 684 {
 685         lockdep_assert_held(&nvmet_config_sem);
 686
 687         if (subsys->type == NVME_NQN_DISC)
 688                 return nvmet_host_discovery_allowed(req, hostnqn);
 689         else
 690                 return __nvmet_host_allowed(subsys, hostnqn);
 691 }
 692
 693 u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
 694                 struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp)
 695 {
 696         struct nvmet_subsys *subsys;
 697         struct nvmet_ctrl *ctrl;
 698         int ret;
 699         u16 status;
 700
 701         status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 702         subsys = nvmet_find_get_subsys(req->port, subsysnqn);
 703         if (!subsys) {
 704                 pr_warn("connect request for invalid subsystem %s!\n",
 705                         subsysnqn);
 706                 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
 707                 goto out;
 708         }
 709
 710         status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 711         down_read(&nvmet_config_sem);
 712         if (!nvmet_host_allowed(req, subsys, hostnqn)) {
 713                 pr_info("connect by host %s for subsystem %s not allowed\n",
 714                         hostnqn, subsysnqn);
 715                 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn);
 716                 up_read(&nvmet_config_sem);
 717                 goto out_put_subsystem;
 718         }
 719         up_read(&nvmet_config_sem);
 720
 721         status = NVME_SC_INTERNAL;
 722         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
 723         if (!ctrl)
 724                 goto out_put_subsystem;
 725         mutex_init(&ctrl->lock);
 726
 727         nvmet_init_cap(ctrl);
 728
 729         INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work);
 730         INIT_LIST_HEAD(&ctrl->async_events);
 731
 732         memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE);
 733         memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE);
 734
 735         /* generate a random serial number as our controllers are ephemeral: */
 736         get_random_bytes(&ctrl->serial, sizeof(ctrl->serial));
 737
 738         kref_init(&ctrl->ref);
 739         ctrl->subsys = subsys;
 740
 741         ctrl->cqs = kcalloc(subsys->max_qid + 1,
 742                         sizeof(struct nvmet_cq *),
 743                         GFP_KERNEL);
 744         if (!ctrl->cqs)
 745                 goto out_free_ctrl;
 746
 747         ctrl->sqs = kcalloc(subsys->max_qid + 1,
 748                         sizeof(struct nvmet_sq *),
 749                         GFP_KERNEL);
 750         if (!ctrl->sqs)
 751                 goto out_free_cqs;
 752
 753         ret = ida_simple_get(&cntlid_ida,
 754                              NVME_CNTLID_MIN, NVME_CNTLID_MAX,
 755                              GFP_KERNEL);
 756         if (ret < 0) {
 757                 status = NVME_SC_CONNECT_CTRL_BUSY | NVME_SC_DNR;
 758                 goto out_free_sqs;
 759         }
 760         ctrl->cntlid = ret;
 761
 762         ctrl->ops = req->ops;
 763         if (ctrl->subsys->type == NVME_NQN_DISC) {
 764                 /* Don't accept keep-alive timeout for discovery controllers */
 765                 if (kato) {
 766                         status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 767                         goto out_free_sqs;
 768                 }
 769
 770                 /*
 771                  * Discovery controllers use some arbitrary high value in order
 772                  * to cleanup stale discovery sessions
 773                  *
 774                  * From the latest base diff RC:
 775                  * "The Keep Alive command is not supported by
 776                  * Discovery controllers. A transport may specify a
 777                  * fixed Discovery controller activity timeout value
 778                  * (e.g., 2 minutes).  If no commands are received
 779                  * by a Discovery controller within that time
 780                  * period, the controller may perform the
 781                  * actions for Keep Alive Timer expiration".
 782                  */
 783                 ctrl->kato = NVMET_DISC_KATO;
 784         } else {
 785                 /* keep-alive timeout in seconds */
 786                 ctrl->kato = DIV_ROUND_UP(kato, 1000);
 787         }
 788         nvmet_start_keep_alive_timer(ctrl);
 789
 790         mutex_lock(&subsys->lock);
 791         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
 792         mutex_unlock(&subsys->lock);
 793
 794         *ctrlp = ctrl;
 795         return 0;
 796
 797 out_free_sqs:
 798         kfree(ctrl->sqs);
 799 out_free_cqs:
 800         kfree(ctrl->cqs);
 801 out_free_ctrl:
 802         kfree(ctrl);
 803 out_put_subsystem:
 804         nvmet_subsys_put(subsys);
 805 out:
 806         return status;
 807 }
 808
 809 static void nvmet_ctrl_free(struct kref *ref)
 810 {
 811         struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref);
 812         struct nvmet_subsys *subsys = ctrl->subsys;
 813
 814         nvmet_stop_keep_alive_timer(ctrl);
 815
 816         mutex_lock(&subsys->lock);
 817         list_del(&ctrl->subsys_entry);
 818         mutex_unlock(&subsys->lock);
 819
 820         flush_work(&ctrl->async_event_work);
 821         cancel_work_sync(&ctrl->fatal_err_work);
 822
 823         ida_simple_remove(&cntlid_ida, ctrl->cntlid);
 824         nvmet_subsys_put(subsys);
 825
 826         kfree(ctrl->sqs);
 827         kfree(ctrl->cqs);
 828         kfree(ctrl);
 829 }
 830
 831 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl)
 832 {
 833         kref_put(&ctrl->ref, nvmet_ctrl_free);
 834 }
 835
 836 static void nvmet_fatal_error_handler(struct work_struct *work)
 837 {
 838         struct nvmet_ctrl *ctrl =
 839                         container_of(work, struct nvmet_ctrl, fatal_err_work);
 840
 841         pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid);
 842         ctrl->ops->delete_ctrl(ctrl);
 843 }
 844
 845 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl)
 846 {
 847         mutex_lock(&ctrl->lock);
 848         if (!(ctrl->csts & NVME_CSTS_CFS)) {
 849                 ctrl->csts |= NVME_CSTS_CFS;
 850                 INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler);
 851                 schedule_work(&ctrl->fatal_err_work);
 852         }
 853         mutex_unlock(&ctrl->lock);
 854 }
 855 EXPORT_SYMBOL_GPL(nvmet_ctrl_fatal_error);
 856
 857 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
 858                 const char *subsysnqn)
 859 {
 860         struct nvmet_subsys_link *p;
 861
 862         if (!port)
 863                 return NULL;
 864
 865         if (!strncmp(NVME_DISC_SUBSYS_NAME, subsysnqn,
 866                         NVMF_NQN_SIZE)) {
 867                 if (!kref_get_unless_zero(&nvmet_disc_subsys->ref))
 868                         return NULL;
 869                 return nvmet_disc_subsys;
 870         }
 871
 872         down_read(&nvmet_config_sem);
 873         list_for_each_entry(p, &port->subsystems, entry) {
 874                 if (!strncmp(p->subsys->subsysnqn, subsysnqn,
 875                                 NVMF_NQN_SIZE)) {
 876                         if (!kref_get_unless_zero(&p->subsys->ref))
 877                                 break;
 878                         up_read(&nvmet_config_sem);
 879                         return p->subsys;
 880                 }
 881         }
 882         up_read(&nvmet_config_sem);
 883         return NULL;
 884 }
 885
 886 struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn,
 887                 enum nvme_subsys_type type)
 888 {
 889         struct nvmet_subsys *subsys;
 890
 891         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
 892         if (!subsys)
 893                 return NULL;
 894
 895         subsys->ver = NVME_VS(1, 2, 1); /* NVMe 1.2.1 */
 896
 897         switch (type) {
 898         case NVME_NQN_NVME:
 899                 subsys->max_qid = NVMET_NR_QUEUES;
 900                 break;
 901         case NVME_NQN_DISC:
 902                 subsys->max_qid = 0;
 903                 break;
 904         default:
 905                 pr_err("%s: Unknown Subsystem type - %d\n", __func__, type);
 906                 kfree(subsys);
 907                 return NULL;
 908         }
 909         subsys->type = type;
 910         subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE,
 911                         GFP_KERNEL);
 912         if (!subsys->subsysnqn) {
 913                 kfree(subsys);
 914                 return NULL;
 915         }
 916
 917         kref_init(&subsys->ref);
 918
 919         mutex_init(&subsys->lock);
 920         INIT_LIST_HEAD(&subsys->namespaces);
 921         INIT_LIST_HEAD(&subsys->ctrls);
 922         INIT_LIST_HEAD(&subsys->hosts);
 923
 924         return subsys;
 925 }
 926
 927 static void nvmet_subsys_free(struct kref *ref)
 928 {
 929         struct nvmet_subsys *subsys =
 930                 container_of(ref, struct nvmet_subsys, ref);
 931
 932         WARN_ON_ONCE(!list_empty(&subsys->namespaces));
 933
 934         kfree(subsys->subsysnqn);
 935         kfree(subsys);
 936 }
 937
 938 void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys)
 939 {
 940         struct nvmet_ctrl *ctrl;
 941
 942         mutex_lock(&subsys->lock);
 943         list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
 944                 ctrl->ops->delete_ctrl(ctrl);
 945         mutex_unlock(&subsys->lock);
 946 }
 947
 948 void nvmet_subsys_put(struct nvmet_subsys *subsys)
 949 {
 950         kref_put(&subsys->ref, nvmet_subsys_free);
 951 }
 952
 953 static int __init nvmet_init(void)
 954 {
 955         int error;
 956
 957         error = nvmet_init_discovery();
 958         if (error)
 959                 goto out;
 960
 961         error = nvmet_init_configfs();
 962         if (error)
 963                 goto out_exit_discovery;
 964         return 0;
 965
 966 out_exit_discovery:
 967         nvmet_exit_discovery();
 968 out:
 969         return error;
 970 }
 971
 972 static void __exit nvmet_exit(void)
 973 {
 974         nvmet_exit_configfs();
 975         nvmet_exit_discovery();
 976         ida_destroy(&cntlid_ida);
 977
 978         BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024);
 979         BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024);
 980 }
 981
 982 module_init(nvmet_init);
 983 module_exit(nvmet_exit);
 984
 985 MODULE_LICENSE("GPL v2");