drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43 #include <rdma/ib_cache.h>
  44
  45 #include <linux/atomic.h>
  46
  47 #include <scsi/scsi.h>
  48 #include <scsi/scsi_device.h>
  49 #include <scsi/scsi_dbg.h>
  50 #include <scsi/scsi_tcq.h>
  51 #include <scsi/srp.h>
  52 #include <scsi/scsi_transport_srp.h>
  53
  54 #include "ib_srp.h"
  55
  56 #define DRV_NAME        "ib_srp"
  57 #define PFX             DRV_NAME ": "
  58 #define DRV_VERSION     "2.0"
  59 #define DRV_RELDATE     "July 26, 2015"
  60
  61 MODULE_AUTHOR("Roland Dreier");
  62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
  63 MODULE_LICENSE("Dual BSD/GPL");
  64 MODULE_VERSION(DRV_VERSION);
  65 MODULE_INFO(release_date, DRV_RELDATE);
  66
  67 #if !defined(CONFIG_DYNAMIC_DEBUG)
  68 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
  69 #define DYNAMIC_DEBUG_BRANCH(descriptor) false
  70 #endif
  71
  72 static unsigned int srp_sg_tablesize;
  73 static unsigned int cmd_sg_entries;
  74 static unsigned int indirect_sg_entries;
  75 static bool allow_ext_sg;
  76 static bool prefer_fr = true;
  77 static bool register_always = true;
  78 static bool never_register;
  79 static int topspin_workarounds = 1;
  80
  81 module_param(srp_sg_tablesize, uint, 0444);
  82 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  83
  84 module_param(cmd_sg_entries, uint, 0444);
  85 MODULE_PARM_DESC(cmd_sg_entries,
  86                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  87
  88 module_param(indirect_sg_entries, uint, 0444);
  89 MODULE_PARM_DESC(indirect_sg_entries,
  90                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
  91
  92 module_param(allow_ext_sg, bool, 0444);
  93 MODULE_PARM_DESC(allow_ext_sg,
  94                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  95
  96 module_param(topspin_workarounds, int, 0444);
  97 MODULE_PARM_DESC(topspin_workarounds,
  98                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  99
 100 module_param(prefer_fr, bool, 0444);
 101 MODULE_PARM_DESC(prefer_fr,
 102 "Whether to use fast registration if both FMR and fast registration are supported");
 103
 104 module_param(register_always, bool, 0444);
 105 MODULE_PARM_DESC(register_always,
 106                  "Use memory registration even for contiguous memory regions");
 107
 108 module_param(never_register, bool, 0444);
 109 MODULE_PARM_DESC(never_register, "Never register memory");
 110
 111 static const struct kernel_param_ops srp_tmo_ops;
 112
 113 static int srp_reconnect_delay = 10;
 114 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 115                 S_IRUGO | S_IWUSR);
 116 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 117
 118 static int srp_fast_io_fail_tmo = 15;
 119 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 120                 S_IRUGO | S_IWUSR);
 121 MODULE_PARM_DESC(fast_io_fail_tmo,
 122                  "Number of seconds between the observation of a transport"
 123                  " layer error and failing all I/O. \"off\" means that this"
 124                  " functionality is disabled.");
 125
 126 static int srp_dev_loss_tmo = 600;
 127 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 128                 S_IRUGO | S_IWUSR);
 129 MODULE_PARM_DESC(dev_loss_tmo,
 130                  "Maximum number of seconds that the SRP transport should"
 131                  " insulate transport layer errors. After this time has been"
 132                  " exceeded the SCSI host is removed. Should be"
 133                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 134                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 135                  " this functionality is disabled.");
 136
 137 static unsigned ch_count;
 138 module_param(ch_count, uint, 0444);
 139 MODULE_PARM_DESC(ch_count,
 140                  "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
 141
 142 static void srp_add_one(struct ib_device *device);
 143 static void srp_remove_one(struct ib_device *device, void *client_data);
 144 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 145 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
 146                 const char *opname);
 147 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 148
 149 static struct scsi_transport_template *ib_srp_transport_template;
 150 static struct workqueue_struct *srp_remove_wq;
 151
 152 static struct ib_client srp_client = {
 153         .name   = "srp",
 154         .add    = srp_add_one,
 155         .remove = srp_remove_one
 156 };
 157
 158 static struct ib_sa_client srp_sa_client;
 159
 160 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 161 {
 162         int tmo = *(int *)kp->arg;
 163
 164         if (tmo >= 0)
 165                 return sprintf(buffer, "%d", tmo);
 166         else
 167                 return sprintf(buffer, "off");
 168 }
 169
 170 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 171 {
 172         int tmo, res;
 173
 174         res = srp_parse_tmo(&tmo, val);
 175         if (res)
 176                 goto out;
 177
 178         if (kp->arg == &srp_reconnect_delay)
 179                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 180                                     srp_dev_loss_tmo);
 181         else if (kp->arg == &srp_fast_io_fail_tmo)
 182                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 183         else
 184                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 185                                     tmo);
 186         if (res)
 187                 goto out;
 188         *(int *)kp->arg = tmo;
 189
 190 out:
 191         return res;
 192 }
 193
 194 static const struct kernel_param_ops srp_tmo_ops = {
 195         .get = srp_tmo_get,
 196         .set = srp_tmo_set,
 197 };
 198
 199 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 200 {
 201         return (struct srp_target_port *) host->hostdata;
 202 }
 203
 204 static const char *srp_target_info(struct Scsi_Host *host)
 205 {
 206         return host_to_target(host)->target_name;
 207 }
 208
 209 static int srp_target_is_topspin(struct srp_target_port *target)
 210 {
 211         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 212         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 213
 214         return topspin_workarounds &&
 215                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 216                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 217 }
 218
 219 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 220                                    gfp_t gfp_mask,
 221                                    enum dma_data_direction direction)
 222 {
 223         struct srp_iu *iu;
 224
 225         iu = kmalloc(sizeof *iu, gfp_mask);
 226         if (!iu)
 227                 goto out;
 228
 229         iu->buf = kzalloc(size, gfp_mask);
 230         if (!iu->buf)
 231                 goto out_free_iu;
 232
 233         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 234                                     direction);
 235         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 236                 goto out_free_buf;
 237
 238         iu->size      = size;
 239         iu->direction = direction;
 240
 241         return iu;
 242
 243 out_free_buf:
 244         kfree(iu->buf);
 245 out_free_iu:
 246         kfree(iu);
 247 out:
 248         return NULL;
 249 }
 250
 251 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 252 {
 253         if (!iu)
 254                 return;
 255
 256         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 257                             iu->direction);
 258         kfree(iu->buf);
 259         kfree(iu);
 260 }
 261
 262 static void srp_qp_event(struct ib_event *event, void *context)
 263 {
 264         pr_debug("QP event %s (%d)\n",
 265                  ib_event_msg(event->event), event->event);
 266 }
 267
 268 static int srp_init_qp(struct srp_target_port *target,
 269                        struct ib_qp *qp)
 270 {
 271         struct ib_qp_attr *attr;
 272         int ret;
 273
 274         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 275         if (!attr)
 276                 return -ENOMEM;
 277
 278         ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
 279                                   target->srp_host->port,
 280                                   be16_to_cpu(target->pkey),
 281                                   &attr->pkey_index);
 282         if (ret)
 283                 goto out;
 284
 285         attr->qp_state        = IB_QPS_INIT;
 286         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 287                                     IB_ACCESS_REMOTE_WRITE);
 288         attr->port_num        = target->srp_host->port;
 289
 290         ret = ib_modify_qp(qp, attr,
 291                            IB_QP_STATE          |
 292                            IB_QP_PKEY_INDEX     |
 293                            IB_QP_ACCESS_FLAGS   |
 294                            IB_QP_PORT);
 295
 296 out:
 297         kfree(attr);
 298         return ret;
 299 }
 300
 301 static int srp_new_cm_id(struct srp_rdma_ch *ch)
 302 {
 303         struct srp_target_port *target = ch->target;
 304         struct ib_cm_id *new_cm_id;
 305
 306         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 307                                     srp_cm_handler, ch);
 308         if (IS_ERR(new_cm_id))
 309                 return PTR_ERR(new_cm_id);
 310
 311         if (ch->cm_id)
 312                 ib_destroy_cm_id(ch->cm_id);
 313         ch->cm_id = new_cm_id;
 314         ch->path.sgid = target->sgid;
 315         ch->path.dgid = target->orig_dgid;
 316         ch->path.pkey = target->pkey;
 317         ch->path.service_id = target->service_id;
 318
 319         return 0;
 320 }
 321
 322 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 323 {
 324         struct srp_device *dev = target->srp_host->srp_dev;
 325         struct ib_fmr_pool_param fmr_param;
 326
 327         memset(&fmr_param, 0, sizeof(fmr_param));
 328         fmr_param.pool_size         = target->mr_pool_size;
 329         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 330         fmr_param.cache             = 1;
 331         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 332         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 333         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 334                                        IB_ACCESS_REMOTE_WRITE |
 335                                        IB_ACCESS_REMOTE_READ);
 336
 337         return ib_create_fmr_pool(dev->pd, &fmr_param);
 338 }
 339
 340 /**
 341  * srp_destroy_fr_pool() - free the resources owned by a pool
 342  * @pool: Fast registration pool to be destroyed.
 343  */
 344 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 345 {
 346         int i;
 347         struct srp_fr_desc *d;
 348
 349         if (!pool)
 350                 return;
 351
 352         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 353                 if (d->mr)
 354                         ib_dereg_mr(d->mr);
 355         }
 356         kfree(pool);
 357 }
 358
 359 /**
 360  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 361  * @device:            IB device to allocate fast registration descriptors for.
 362  * @pd:                Protection domain associated with the FR descriptors.
 363  * @pool_size:         Number of descriptors to allocate.
 364  * @max_page_list_len: Maximum fast registration work request page list length.
 365  */
 366 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 367                                               struct ib_pd *pd, int pool_size,
 368                                               int max_page_list_len)
 369 {
 370         struct srp_fr_pool *pool;
 371         struct srp_fr_desc *d;
 372         struct ib_mr *mr;
 373         int i, ret = -EINVAL;
 374         enum ib_mr_type mr_type;
 375
 376         if (pool_size <= 0)
 377                 goto err;
 378         ret = -ENOMEM;
 379         pool = kzalloc(sizeof(struct srp_fr_pool) +
 380                        pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
 381         if (!pool)
 382                 goto err;
 383         pool->size = pool_size;
 384         pool->max_page_list_len = max_page_list_len;
 385         spin_lock_init(&pool->lock);
 386         INIT_LIST_HEAD(&pool->free_list);
 387
 388         if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
 389                 mr_type = IB_MR_TYPE_SG_GAPS;
 390         else
 391                 mr_type = IB_MR_TYPE_MEM_REG;
 392
 393         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 394                 mr = ib_alloc_mr(pd, mr_type, max_page_list_len);
 395                 if (IS_ERR(mr)) {
 396                         ret = PTR_ERR(mr);
 397                         if (ret == -ENOMEM)
 398                                 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
 399                                         dev_name(&device->dev));
 400                         goto destroy_pool;
 401                 }
 402                 d->mr = mr;
 403                 list_add_tail(&d->entry, &pool->free_list);
 404         }
 405
 406 out:
 407         return pool;
 408
 409 destroy_pool:
 410         srp_destroy_fr_pool(pool);
 411
 412 err:
 413         pool = ERR_PTR(ret);
 414         goto out;
 415 }
 416
 417 /**
 418  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 419  * @pool: Pool to obtain descriptor from.
 420  */
 421 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 422 {
 423         struct srp_fr_desc *d = NULL;
 424         unsigned long flags;
 425
 426         spin_lock_irqsave(&pool->lock, flags);
 427         if (!list_empty(&pool->free_list)) {
 428                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 429                 list_del(&d->entry);
 430         }
 431         spin_unlock_irqrestore(&pool->lock, flags);
 432
 433         return d;
 434 }
 435
 436 /**
 437  * srp_fr_pool_put() - put an FR descriptor back in the free list
 438  * @pool: Pool the descriptor was allocated from.
 439  * @desc: Pointer to an array of fast registration descriptor pointers.
 440  * @n:    Number of descriptors to put back.
 441  *
 442  * Note: The caller must already have queued an invalidation request for
 443  * desc->mr->rkey before calling this function.
 444  */
 445 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 446                             int n)
 447 {
 448         unsigned long flags;
 449         int i;
 450
 451         spin_lock_irqsave(&pool->lock, flags);
 452         for (i = 0; i < n; i++)
 453                 list_add(&desc[i]->entry, &pool->free_list);
 454         spin_unlock_irqrestore(&pool->lock, flags);
 455 }
 456
 457 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 458 {
 459         struct srp_device *dev = target->srp_host->srp_dev;
 460
 461         return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
 462                                   dev->max_pages_per_mr);
 463 }
 464
 465 /**
 466  * srp_destroy_qp() - destroy an RDMA queue pair
 467  * @qp: RDMA queue pair.
 468  *
 469  * Drain the qp before destroying it.  This avoids that the receive
 470  * completion handler can access the queue pair while it is
 471  * being destroyed.
 472  */
 473 static void srp_destroy_qp(struct ib_qp *qp)
 474 {
 475         ib_drain_rq(qp);
 476         ib_destroy_qp(qp);
 477 }
 478
 479 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 480 {
 481         struct srp_target_port *target = ch->target;
 482         struct srp_device *dev = target->srp_host->srp_dev;
 483         struct ib_qp_init_attr *init_attr;
 484         struct ib_cq *recv_cq, *send_cq;
 485         struct ib_qp *qp;
 486         struct ib_fmr_pool *fmr_pool = NULL;
 487         struct srp_fr_pool *fr_pool = NULL;
 488         const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
 489         int ret;
 490
 491         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 492         if (!init_attr)
 493                 return -ENOMEM;
 494
 495         /* queue_size + 1 for ib_drain_rq() */
 496         recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
 497                                 ch->comp_vector, IB_POLL_SOFTIRQ);
 498         if (IS_ERR(recv_cq)) {
 499                 ret = PTR_ERR(recv_cq);
 500                 goto err;
 501         }
 502
 503         send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
 504                                 ch->comp_vector, IB_POLL_DIRECT);
 505         if (IS_ERR(send_cq)) {
 506                 ret = PTR_ERR(send_cq);
 507                 goto err_recv_cq;
 508         }
 509
 510         init_attr->event_handler       = srp_qp_event;
 511         init_attr->cap.max_send_wr     = m * target->queue_size;
 512         init_attr->cap.max_recv_wr     = target->queue_size + 1;
 513         init_attr->cap.max_recv_sge    = 1;
 514         init_attr->cap.max_send_sge    = 1;
 515         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 516         init_attr->qp_type             = IB_QPT_RC;
 517         init_attr->send_cq             = send_cq;
 518         init_attr->recv_cq             = recv_cq;
 519
 520         qp = ib_create_qp(dev->pd, init_attr);
 521         if (IS_ERR(qp)) {
 522                 ret = PTR_ERR(qp);
 523                 goto err_send_cq;
 524         }
 525
 526         ret = srp_init_qp(target, qp);
 527         if (ret)
 528                 goto err_qp;
 529
 530         if (dev->use_fast_reg) {
 531                 fr_pool = srp_alloc_fr_pool(target);
 532                 if (IS_ERR(fr_pool)) {
 533                         ret = PTR_ERR(fr_pool);
 534                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 535                                      "FR pool allocation failed (%d)\n", ret);
 536                         goto err_qp;
 537                 }
 538         } else if (dev->use_fmr) {
 539                 fmr_pool = srp_alloc_fmr_pool(target);
 540                 if (IS_ERR(fmr_pool)) {
 541                         ret = PTR_ERR(fmr_pool);
 542                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 543                                      "FMR pool allocation failed (%d)\n", ret);
 544                         goto err_qp;
 545                 }
 546         }
 547
 548         if (ch->qp)
 549                 srp_destroy_qp(ch->qp);
 550         if (ch->recv_cq)
 551                 ib_free_cq(ch->recv_cq);
 552         if (ch->send_cq)
 553                 ib_free_cq(ch->send_cq);
 554
 555         ch->qp = qp;
 556         ch->recv_cq = recv_cq;
 557         ch->send_cq = send_cq;
 558
 559         if (dev->use_fast_reg) {
 560                 if (ch->fr_pool)
 561                         srp_destroy_fr_pool(ch->fr_pool);
 562                 ch->fr_pool = fr_pool;
 563         } else if (dev->use_fmr) {
 564                 if (ch->fmr_pool)
 565                         ib_destroy_fmr_pool(ch->fmr_pool);
 566                 ch->fmr_pool = fmr_pool;
 567         }
 568
 569         kfree(init_attr);
 570         return 0;
 571
 572 err_qp:
 573         srp_destroy_qp(qp);
 574
 575 err_send_cq:
 576         ib_free_cq(send_cq);
 577
 578 err_recv_cq:
 579         ib_free_cq(recv_cq);
 580
 581 err:
 582         kfree(init_attr);
 583         return ret;
 584 }
 585
 586 /*
 587  * Note: this function may be called without srp_alloc_iu_bufs() having been
 588  * invoked. Hence the ch->[rt]x_ring checks.
 589  */
 590 static void srp_free_ch_ib(struct srp_target_port *target,
 591                            struct srp_rdma_ch *ch)
 592 {
 593         struct srp_device *dev = target->srp_host->srp_dev;
 594         int i;
 595
 596         if (!ch->target)
 597                 return;
 598
 599         if (ch->cm_id) {
 600                 ib_destroy_cm_id(ch->cm_id);
 601                 ch->cm_id = NULL;
 602         }
 603
 604         /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
 605         if (!ch->qp)
 606                 return;
 607
 608         if (dev->use_fast_reg) {
 609                 if (ch->fr_pool)
 610                         srp_destroy_fr_pool(ch->fr_pool);
 611         } else if (dev->use_fmr) {
 612                 if (ch->fmr_pool)
 613                         ib_destroy_fmr_pool(ch->fmr_pool);
 614         }
 615
 616         srp_destroy_qp(ch->qp);
 617         ib_free_cq(ch->send_cq);
 618         ib_free_cq(ch->recv_cq);
 619
 620         /*
 621          * Avoid that the SCSI error handler tries to use this channel after
 622          * it has been freed. The SCSI error handler can namely continue
 623          * trying to perform recovery actions after scsi_remove_host()
 624          * returned.
 625          */
 626         ch->target = NULL;
 627
 628         ch->qp = NULL;
 629         ch->send_cq = ch->recv_cq = NULL;
 630
 631         if (ch->rx_ring) {
 632                 for (i = 0; i < target->queue_size; ++i)
 633                         srp_free_iu(target->srp_host, ch->rx_ring[i]);
 634                 kfree(ch->rx_ring);
 635                 ch->rx_ring = NULL;
 636         }
 637         if (ch->tx_ring) {
 638                 for (i = 0; i < target->queue_size; ++i)
 639                         srp_free_iu(target->srp_host, ch->tx_ring[i]);
 640                 kfree(ch->tx_ring);
 641                 ch->tx_ring = NULL;
 642         }
 643 }
 644
 645 static void srp_path_rec_completion(int status,
 646                                     struct ib_sa_path_rec *pathrec,
 647                                     void *ch_ptr)
 648 {
 649         struct srp_rdma_ch *ch = ch_ptr;
 650         struct srp_target_port *target = ch->target;
 651
 652         ch->status = status;
 653         if (status)
 654                 shost_printk(KERN_ERR, target->scsi_host,
 655                              PFX "Got failed path rec status %d\n", status);
 656         else
 657                 ch->path = *pathrec;
 658         complete(&ch->done);
 659 }
 660
 661 static int srp_lookup_path(struct srp_rdma_ch *ch)
 662 {
 663         struct srp_target_port *target = ch->target;
 664         int ret;
 665
 666         ch->path.numb_path = 1;
 667
 668         init_completion(&ch->done);
 669
 670         ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 671                                                target->srp_host->srp_dev->dev,
 672                                                target->srp_host->port,
 673                                                &ch->path,
 674                                                IB_SA_PATH_REC_SERVICE_ID |
 675                                                IB_SA_PATH_REC_DGID       |
 676                                                IB_SA_PATH_REC_SGID       |
 677                                                IB_SA_PATH_REC_NUMB_PATH  |
 678                                                IB_SA_PATH_REC_PKEY,
 679                                                SRP_PATH_REC_TIMEOUT_MS,
 680                                                GFP_KERNEL,
 681                                                srp_path_rec_completion,
 682                                                ch, &ch->path_query);
 683         if (ch->path_query_id < 0)
 684                 return ch->path_query_id;
 685
 686         ret = wait_for_completion_interruptible(&ch->done);
 687         if (ret < 0)
 688                 return ret;
 689
 690         if (ch->status < 0)
 691                 shost_printk(KERN_WARNING, target->scsi_host,
 692                              PFX "Path record query failed\n");
 693
 694         return ch->status;
 695 }
 696
 697 static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 698 {
 699         struct srp_target_port *target = ch->target;
 700         struct {
 701                 struct ib_cm_req_param param;
 702                 struct srp_login_req   priv;
 703         } *req = NULL;
 704         int status;
 705
 706         req = kzalloc(sizeof *req, GFP_KERNEL);
 707         if (!req)
 708                 return -ENOMEM;
 709
 710         req->param.primary_path               = &ch->path;
 711         req->param.alternate_path             = NULL;
 712         req->param.service_id                 = target->service_id;
 713         req->param.qp_num                     = ch->qp->qp_num;
 714         req->param.qp_type                    = ch->qp->qp_type;
 715         req->param.private_data               = &req->priv;
 716         req->param.private_data_len           = sizeof req->priv;
 717         req->param.flow_control               = 1;
 718
 719         get_random_bytes(&req->param.starting_psn, 4);
 720         req->param.starting_psn              &= 0xffffff;
 721
 722         /*
 723          * Pick some arbitrary defaults here; we could make these
 724          * module parameters if anyone cared about setting them.
 725          */
 726         req->param.responder_resources        = 4;
 727         req->param.remote_cm_response_timeout = 20;
 728         req->param.local_cm_response_timeout  = 20;
 729         req->param.retry_count                = target->tl_retry_count;
 730         req->param.rnr_retry_count            = 7;
 731         req->param.max_cm_retries             = 15;
 732
 733         req->priv.opcode        = SRP_LOGIN_REQ;
 734         req->priv.tag           = 0;
 735         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 736         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 737                                               SRP_BUF_FORMAT_INDIRECT);
 738         req->priv.req_flags     = (multich ? SRP_MULTICHAN_MULTI :
 739                                    SRP_MULTICHAN_SINGLE);
 740         /*
 741          * In the published SRP specification (draft rev. 16a), the
 742          * port identifier format is 8 bytes of ID extension followed
 743          * by 8 bytes of GUID.  Older drafts put the two halves in the
 744          * opposite order, so that the GUID comes first.
 745          *
 746          * Targets conforming to these obsolete drafts can be
 747          * recognized by the I/O Class they report.
 748          */
 749         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 750                 memcpy(req->priv.initiator_port_id,
 751                        &target->sgid.global.interface_id, 8);
 752                 memcpy(req->priv.initiator_port_id + 8,
 753                        &target->initiator_ext, 8);
 754                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 755                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 756         } else {
 757                 memcpy(req->priv.initiator_port_id,
 758                        &target->initiator_ext, 8);
 759                 memcpy(req->priv.initiator_port_id + 8,
 760                        &target->sgid.global.interface_id, 8);
 761                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 762                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 763         }
 764
 765         /*
 766          * Topspin/Cisco SRP targets will reject our login unless we
 767          * zero out the first 8 bytes of our initiator port ID and set
 768          * the second 8 bytes to the local node GUID.
 769          */
 770         if (srp_target_is_topspin(target)) {
 771                 shost_printk(KERN_DEBUG, target->scsi_host,
 772                              PFX "Topspin/Cisco initiator port ID workaround "
 773                              "activated for target GUID %016llx\n",
 774                              be64_to_cpu(target->ioc_guid));
 775                 memset(req->priv.initiator_port_id, 0, 8);
 776                 memcpy(req->priv.initiator_port_id + 8,
 777                        &target->srp_host->srp_dev->dev->node_guid, 8);
 778         }
 779
 780         status = ib_send_cm_req(ch->cm_id, &req->param);
 781
 782         kfree(req);
 783
 784         return status;
 785 }
 786
 787 static bool srp_queue_remove_work(struct srp_target_port *target)
 788 {
 789         bool changed = false;
 790
 791         spin_lock_irq(&target->lock);
 792         if (target->state != SRP_TARGET_REMOVED) {
 793                 target->state = SRP_TARGET_REMOVED;
 794                 changed = true;
 795         }
 796         spin_unlock_irq(&target->lock);
 797
 798         if (changed)
 799                 queue_work(srp_remove_wq, &target->remove_work);
 800
 801         return changed;
 802 }
 803
 804 static void srp_disconnect_target(struct srp_target_port *target)
 805 {
 806         struct srp_rdma_ch *ch;
 807         int i;
 808
 809         /* XXX should send SRP_I_LOGOUT request */
 810
 811         for (i = 0; i < target->ch_count; i++) {
 812                 ch = &target->ch[i];
 813                 ch->connected = false;
 814                 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
 815                         shost_printk(KERN_DEBUG, target->scsi_host,
 816                                      PFX "Sending CM DREQ failed\n");
 817                 }
 818         }
 819 }
 820
 821 static void srp_free_req_data(struct srp_target_port *target,
 822                               struct srp_rdma_ch *ch)
 823 {
 824         struct srp_device *dev = target->srp_host->srp_dev;
 825         struct ib_device *ibdev = dev->dev;
 826         struct srp_request *req;
 827         int i;
 828
 829         if (!ch->req_ring)
 830                 return;
 831
 832         for (i = 0; i < target->req_ring_size; ++i) {
 833                 req = &ch->req_ring[i];
 834                 if (dev->use_fast_reg) {
 835                         kfree(req->fr_list);
 836                 } else {
 837                         kfree(req->fmr_list);
 838                         kfree(req->map_page);
 839                 }
 840                 if (req->indirect_dma_addr) {
 841                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 842                                             target->indirect_size,
 843                                             DMA_TO_DEVICE);
 844                 }
 845                 kfree(req->indirect_desc);
 846         }
 847
 848         kfree(ch->req_ring);
 849         ch->req_ring = NULL;
 850 }
 851
 852 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
 853 {
 854         struct srp_target_port *target = ch->target;
 855         struct srp_device *srp_dev = target->srp_host->srp_dev;
 856         struct ib_device *ibdev = srp_dev->dev;
 857         struct srp_request *req;
 858         void *mr_list;
 859         dma_addr_t dma_addr;
 860         int i, ret = -ENOMEM;
 861
 862         ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
 863                                GFP_KERNEL);
 864         if (!ch->req_ring)
 865                 goto out;
 866
 867         for (i = 0; i < target->req_ring_size; ++i) {
 868                 req = &ch->req_ring[i];
 869                 mr_list = kmalloc(target->mr_per_cmd * sizeof(void *),
 870                                   GFP_KERNEL);
 871                 if (!mr_list)
 872                         goto out;
 873                 if (srp_dev->use_fast_reg) {
 874                         req->fr_list = mr_list;
 875                 } else {
 876                         req->fmr_list = mr_list;
 877                         req->map_page = kmalloc(srp_dev->max_pages_per_mr *
 878                                                 sizeof(void *), GFP_KERNEL);
 879                         if (!req->map_page)
 880                                 goto out;
 881                 }
 882                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 883                 if (!req->indirect_desc)
 884                         goto out;
 885
 886                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 887                                              target->indirect_size,
 888                                              DMA_TO_DEVICE);
 889                 if (ib_dma_mapping_error(ibdev, dma_addr))
 890                         goto out;
 891
 892                 req->indirect_dma_addr = dma_addr;
 893         }
 894         ret = 0;
 895
 896 out:
 897         return ret;
 898 }
 899
 900 /**
 901  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 902  * @shost: SCSI host whose attributes to remove from sysfs.
 903  *
 904  * Note: Any attributes defined in the host template and that did not exist
 905  * before invocation of this function will be ignored.
 906  */
 907 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 908 {
 909         struct device_attribute **attr;
 910
 911         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 912                 device_remove_file(&shost->shost_dev, *attr);
 913 }
 914
 915 static void srp_remove_target(struct srp_target_port *target)
 916 {
 917         struct srp_rdma_ch *ch;
 918         int i;
 919
 920         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 921
 922         srp_del_scsi_host_attr(target->scsi_host);
 923         srp_rport_get(target->rport);
 924         srp_remove_host(target->scsi_host);
 925         scsi_remove_host(target->scsi_host);
 926         srp_stop_rport_timers(target->rport);
 927         srp_disconnect_target(target);
 928         for (i = 0; i < target->ch_count; i++) {
 929                 ch = &target->ch[i];
 930                 srp_free_ch_ib(target, ch);
 931         }
 932         cancel_work_sync(&target->tl_err_work);
 933         srp_rport_put(target->rport);
 934         for (i = 0; i < target->ch_count; i++) {
 935                 ch = &target->ch[i];
 936                 srp_free_req_data(target, ch);
 937         }
 938         kfree(target->ch);
 939         target->ch = NULL;
 940
 941         spin_lock(&target->srp_host->target_lock);
 942         list_del(&target->list);
 943         spin_unlock(&target->srp_host->target_lock);
 944
 945         scsi_host_put(target->scsi_host);
 946 }
 947
 948 static void srp_remove_work(struct work_struct *work)
 949 {
 950         struct srp_target_port *target =
 951                 container_of(work, struct srp_target_port, remove_work);
 952
 953         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 954
 955         srp_remove_target(target);
 956 }
 957
 958 static void srp_rport_delete(struct srp_rport *rport)
 959 {
 960         struct srp_target_port *target = rport->lld_data;
 961
 962         srp_queue_remove_work(target);
 963 }
 964
 965 /**
 966  * srp_connected_ch() - number of connected channels
 967  * @target: SRP target port.
 968  */
 969 static int srp_connected_ch(struct srp_target_port *target)
 970 {
 971         int i, c = 0;
 972
 973         for (i = 0; i < target->ch_count; i++)
 974                 c += target->ch[i].connected;
 975
 976         return c;
 977 }
 978
 979 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
 980 {
 981         struct srp_target_port *target = ch->target;
 982         int ret;
 983
 984         WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
 985
 986         ret = srp_lookup_path(ch);
 987         if (ret)
 988                 goto out;
 989
 990         while (1) {
 991                 init_completion(&ch->done);
 992                 ret = srp_send_req(ch, multich);
 993                 if (ret)
 994                         goto out;
 995                 ret = wait_for_completion_interruptible(&ch->done);
 996                 if (ret < 0)
 997                         goto out;
 998
 999                 /*
1000                  * The CM event handling code will set status to
1001                  * SRP_PORT_REDIRECT if we get a port redirect REJ
1002                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
1003                  * redirect REJ back.
1004                  */
1005                 ret = ch->status;
1006                 switch (ret) {
1007                 case 0:
1008                         ch->connected = true;
1009                         goto out;
1010
1011                 case SRP_PORT_REDIRECT:
1012                         ret = srp_lookup_path(ch);
1013                         if (ret)
1014                                 goto out;
1015                         break;
1016
1017                 case SRP_DLID_REDIRECT:
1018                         break;
1019
1020                 case SRP_STALE_CONN:
1021                         shost_printk(KERN_ERR, target->scsi_host, PFX
1022                                      "giving up on stale connection\n");
1023                         ret = -ECONNRESET;
1024                         goto out;
1025
1026                 default:
1027                         goto out;
1028                 }
1029         }
1030
1031 out:
1032         return ret <= 0 ? ret : -ENODEV;
1033 }
1034
1035 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1036 {
1037         srp_handle_qp_err(cq, wc, "INV RKEY");
1038 }
1039
1040 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1041                 u32 rkey)
1042 {
1043         struct ib_send_wr *bad_wr;
1044         struct ib_send_wr wr = {
1045                 .opcode             = IB_WR_LOCAL_INV,
1046                 .next               = NULL,
1047                 .num_sge            = 0,
1048                 .send_flags         = 0,
1049                 .ex.invalidate_rkey = rkey,
1050         };
1051
1052         wr.wr_cqe = &req->reg_cqe;
1053         req->reg_cqe.done = srp_inv_rkey_err_done;
1054         return ib_post_send(ch->qp, &wr, &bad_wr);
1055 }
1056
1057 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1058                            struct srp_rdma_ch *ch,
1059                            struct srp_request *req)
1060 {
1061         struct srp_target_port *target = ch->target;
1062         struct srp_device *dev = target->srp_host->srp_dev;
1063         struct ib_device *ibdev = dev->dev;
1064         int i, res;
1065
1066         if (!scsi_sglist(scmnd) ||
1067             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1068              scmnd->sc_data_direction != DMA_FROM_DEVICE))
1069                 return;
1070
1071         if (dev->use_fast_reg) {
1072                 struct srp_fr_desc **pfr;
1073
1074                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1075                         res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1076                         if (res < 0) {
1077                                 shost_printk(KERN_ERR, target->scsi_host, PFX
1078                                   "Queueing INV WR for rkey %#x failed (%d)\n",
1079                                   (*pfr)->mr->rkey, res);
1080                                 queue_work(system_long_wq,
1081                                            &target->tl_err_work);
1082                         }
1083                 }
1084                 if (req->nmdesc)
1085                         srp_fr_pool_put(ch->fr_pool, req->fr_list,
1086                                         req->nmdesc);
1087         } else if (dev->use_fmr) {
1088                 struct ib_pool_fmr **pfmr;
1089
1090                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1091                         ib_fmr_pool_unmap(*pfmr);
1092         }
1093
1094         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1095                         scmnd->sc_data_direction);
1096 }
1097
1098 /**
1099  * srp_claim_req - Take ownership of the scmnd associated with a request.
1100  * @ch: SRP RDMA channel.
1101  * @req: SRP request.
1102  * @sdev: If not NULL, only take ownership for this SCSI device.
1103  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1104  *         ownership of @req->scmnd if it equals @scmnd.
1105  *
1106  * Return value:
1107  * Either NULL or a pointer to the SCSI command the caller became owner of.
1108  */
1109 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1110                                        struct srp_request *req,
1111                                        struct scsi_device *sdev,
1112                                        struct scsi_cmnd *scmnd)
1113 {
1114         unsigned long flags;
1115
1116         spin_lock_irqsave(&ch->lock, flags);
1117         if (req->scmnd &&
1118             (!sdev || req->scmnd->device == sdev) &&
1119             (!scmnd || req->scmnd == scmnd)) {
1120                 scmnd = req->scmnd;
1121                 req->scmnd = NULL;
1122         } else {
1123                 scmnd = NULL;
1124         }
1125         spin_unlock_irqrestore(&ch->lock, flags);
1126
1127         return scmnd;
1128 }
1129
1130 /**
1131  * srp_free_req() - Unmap data and adjust ch->req_lim.
1132  * @ch:     SRP RDMA channel.
1133  * @req:    Request to be freed.
1134  * @scmnd:  SCSI command associated with @req.
1135  * @req_lim_delta: Amount to be added to @target->req_lim.
1136  */
1137 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1138                          struct scsi_cmnd *scmnd, s32 req_lim_delta)
1139 {
1140         unsigned long flags;
1141
1142         srp_unmap_data(scmnd, ch, req);
1143
1144         spin_lock_irqsave(&ch->lock, flags);
1145         ch->req_lim += req_lim_delta;
1146         spin_unlock_irqrestore(&ch->lock, flags);
1147 }
1148
1149 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1150                            struct scsi_device *sdev, int result)
1151 {
1152         struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1153
1154         if (scmnd) {
1155                 srp_free_req(ch, req, scmnd, 0);
1156                 scmnd->result = result;
1157                 scmnd->scsi_done(scmnd);
1158         }
1159 }
1160
1161 static void srp_terminate_io(struct srp_rport *rport)
1162 {
1163         struct srp_target_port *target = rport->lld_data;
1164         struct srp_rdma_ch *ch;
1165         struct Scsi_Host *shost = target->scsi_host;
1166         struct scsi_device *sdev;
1167         int i, j;
1168
1169         /*
1170          * Invoking srp_terminate_io() while srp_queuecommand() is running
1171          * is not safe. Hence the warning statement below.
1172          */
1173         shost_for_each_device(sdev, shost)
1174                 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1175
1176         for (i = 0; i < target->ch_count; i++) {
1177                 ch = &target->ch[i];
1178
1179                 for (j = 0; j < target->req_ring_size; ++j) {
1180                         struct srp_request *req = &ch->req_ring[j];
1181
1182                         srp_finish_req(ch, req, NULL,
1183                                        DID_TRANSPORT_FAILFAST << 16);
1184                 }
1185         }
1186 }
1187
1188 /*
1189  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1190  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1191  * srp_reset_device() or srp_reset_host() calls will occur while this function
1192  * is in progress. One way to realize that is not to call this function
1193  * directly but to call srp_reconnect_rport() instead since that last function
1194  * serializes calls of this function via rport->mutex and also blocks
1195  * srp_queuecommand() calls before invoking this function.
1196  */
1197 static int srp_rport_reconnect(struct srp_rport *rport)
1198 {
1199         struct srp_target_port *target = rport->lld_data;
1200         struct srp_rdma_ch *ch;
1201         int i, j, ret = 0;
1202         bool multich = false;
1203
1204         srp_disconnect_target(target);
1205
1206         if (target->state == SRP_TARGET_SCANNING)
1207                 return -ENODEV;
1208
1209         /*
1210          * Now get a new local CM ID so that we avoid confusing the target in
1211          * case things are really fouled up. Doing so also ensures that all CM
1212          * callbacks will have finished before a new QP is allocated.
1213          */
1214         for (i = 0; i < target->ch_count; i++) {
1215                 ch = &target->ch[i];
1216                 ret += srp_new_cm_id(ch);
1217         }
1218         for (i = 0; i < target->ch_count; i++) {
1219                 ch = &target->ch[i];
1220                 for (j = 0; j < target->req_ring_size; ++j) {
1221                         struct srp_request *req = &ch->req_ring[j];
1222
1223                         srp_finish_req(ch, req, NULL, DID_RESET << 16);
1224                 }
1225         }
1226         for (i = 0; i < target->ch_count; i++) {
1227                 ch = &target->ch[i];
1228                 /*
1229                  * Whether or not creating a new CM ID succeeded, create a new
1230                  * QP. This guarantees that all completion callback function
1231                  * invocations have finished before request resetting starts.
1232                  */
1233                 ret += srp_create_ch_ib(ch);
1234
1235                 INIT_LIST_HEAD(&ch->free_tx);
1236                 for (j = 0; j < target->queue_size; ++j)
1237                         list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1238         }
1239
1240         target->qp_in_error = false;
1241
1242         for (i = 0; i < target->ch_count; i++) {
1243                 ch = &target->ch[i];
1244                 if (ret)
1245                         break;
1246                 ret = srp_connect_ch(ch, multich);
1247                 multich = true;
1248         }
1249
1250         if (ret == 0)
1251                 shost_printk(KERN_INFO, target->scsi_host,
1252                              PFX "reconnect succeeded\n");
1253
1254         return ret;
1255 }
1256
1257 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1258                          unsigned int dma_len, u32 rkey)
1259 {
1260         struct srp_direct_buf *desc = state->desc;
1261
1262         WARN_ON_ONCE(!dma_len);
1263
1264         desc->va = cpu_to_be64(dma_addr);
1265         desc->key = cpu_to_be32(rkey);
1266         desc->len = cpu_to_be32(dma_len);
1267
1268         state->total_len += dma_len;
1269         state->desc++;
1270         state->ndesc++;
1271 }
1272
1273 static int srp_map_finish_fmr(struct srp_map_state *state,
1274                               struct srp_rdma_ch *ch)
1275 {
1276         struct srp_target_port *target = ch->target;
1277         struct srp_device *dev = target->srp_host->srp_dev;
1278         struct ib_pd *pd = target->pd;
1279         struct ib_pool_fmr *fmr;
1280         u64 io_addr = 0;
1281
1282         if (state->fmr.next >= state->fmr.end) {
1283                 shost_printk(KERN_ERR, ch->target->scsi_host,
1284                              PFX "Out of MRs (mr_per_cmd = %d)\n",
1285                              ch->target->mr_per_cmd);
1286                 return -ENOMEM;
1287         }
1288
1289         WARN_ON_ONCE(!dev->use_fmr);
1290
1291         if (state->npages == 0)
1292                 return 0;
1293
1294         if (state->npages == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1295                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1296                              pd->unsafe_global_rkey);
1297                 goto reset_state;
1298         }
1299
1300         fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1301                                    state->npages, io_addr);
1302         if (IS_ERR(fmr))
1303                 return PTR_ERR(fmr);
1304
1305         *state->fmr.next++ = fmr;
1306         state->nmdesc++;
1307
1308         srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1309                      state->dma_len, fmr->fmr->rkey);
1310
1311 reset_state:
1312         state->npages = 0;
1313         state->dma_len = 0;
1314
1315         return 0;
1316 }
1317
1318 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1319 {
1320         srp_handle_qp_err(cq, wc, "FAST REG");
1321 }
1322
1323 /*
1324  * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
1325  * where to start in the first element. If sg_offset_p != NULL then
1326  * *sg_offset_p is updated to the offset in state->sg[retval] of the first
1327  * byte that has not yet been mapped.
1328  */
1329 static int srp_map_finish_fr(struct srp_map_state *state,
1330                              struct srp_request *req,
1331                              struct srp_rdma_ch *ch, int sg_nents,
1332                              unsigned int *sg_offset_p)
1333 {
1334         struct srp_target_port *target = ch->target;
1335         struct srp_device *dev = target->srp_host->srp_dev;
1336         struct ib_pd *pd = target->pd;
1337         struct ib_send_wr *bad_wr;
1338         struct ib_reg_wr wr;
1339         struct srp_fr_desc *desc;
1340         u32 rkey;
1341         int n, err;
1342
1343         if (state->fr.next >= state->fr.end) {
1344                 shost_printk(KERN_ERR, ch->target->scsi_host,
1345                              PFX "Out of MRs (mr_per_cmd = %d)\n",
1346                              ch->target->mr_per_cmd);
1347                 return -ENOMEM;
1348         }
1349
1350         WARN_ON_ONCE(!dev->use_fast_reg);
1351
1352         if (sg_nents == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1353                 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1354
1355                 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
1356                              sg_dma_len(state->sg) - sg_offset,
1357                              pd->unsafe_global_rkey);
1358                 if (sg_offset_p)
1359                         *sg_offset_p = 0;
1360                 return 1;
1361         }
1362
1363         desc = srp_fr_pool_get(ch->fr_pool);
1364         if (!desc)
1365                 return -ENOMEM;
1366
1367         rkey = ib_inc_rkey(desc->mr->rkey);
1368         ib_update_fast_reg_key(desc->mr, rkey);
1369
1370         n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
1371                          dev->mr_page_size);
1372         if (unlikely(n < 0)) {
1373                 srp_fr_pool_put(ch->fr_pool, &desc, 1);
1374                 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1375                          dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1376                          sg_offset_p ? *sg_offset_p : -1, n);
1377                 return n;
1378         }
1379
1380         WARN_ON_ONCE(desc->mr->length == 0);
1381
1382         req->reg_cqe.done = srp_reg_mr_err_done;
1383
1384         wr.wr.next = NULL;
1385         wr.wr.opcode = IB_WR_REG_MR;
1386         wr.wr.wr_cqe = &req->reg_cqe;
1387         wr.wr.num_sge = 0;
1388         wr.wr.send_flags = 0;
1389         wr.mr = desc->mr;
1390         wr.key = desc->mr->rkey;
1391         wr.access = (IB_ACCESS_LOCAL_WRITE |
1392                      IB_ACCESS_REMOTE_READ |
1393                      IB_ACCESS_REMOTE_WRITE);
1394
1395         *state->fr.next++ = desc;
1396         state->nmdesc++;
1397
1398         srp_map_desc(state, desc->mr->iova,
1399                      desc->mr->length, desc->mr->rkey);
1400
1401         err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1402         if (unlikely(err)) {
1403                 WARN_ON_ONCE(err == -ENOMEM);
1404                 return err;
1405         }
1406
1407         return n;
1408 }
1409
1410 static int srp_map_sg_entry(struct srp_map_state *state,
1411                             struct srp_rdma_ch *ch,
1412                             struct scatterlist *sg)
1413 {
1414         struct srp_target_port *target = ch->target;
1415         struct srp_device *dev = target->srp_host->srp_dev;
1416         struct ib_device *ibdev = dev->dev;
1417         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1418         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1419         unsigned int len = 0;
1420         int ret;
1421
1422         WARN_ON_ONCE(!dma_len);
1423
1424         while (dma_len) {
1425                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1426
1427                 if (state->npages == dev->max_pages_per_mr ||
1428                     (state->npages > 0 && offset != 0)) {
1429                         ret = srp_map_finish_fmr(state, ch);
1430                         if (ret)
1431                                 return ret;
1432                 }
1433
1434                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1435
1436                 if (!state->npages)
1437                         state->base_dma_addr = dma_addr;
1438                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1439                 state->dma_len += len;
1440                 dma_addr += len;
1441                 dma_len -= len;
1442         }
1443
1444         /*
1445          * If the end of the MR is not on a page boundary then we need to
1446          * close it out and start a new one -- we can only merge at page
1447          * boundaries.
1448          */
1449         ret = 0;
1450         if ((dma_addr & ~dev->mr_page_mask) != 0)
1451                 ret = srp_map_finish_fmr(state, ch);
1452         return ret;
1453 }
1454
1455 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1456                           struct srp_request *req, struct scatterlist *scat,
1457                           int count)
1458 {
1459         struct scatterlist *sg;
1460         int i, ret;
1461
1462         state->pages = req->map_page;
1463         state->fmr.next = req->fmr_list;
1464         state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
1465
1466         for_each_sg(scat, sg, count, i) {
1467                 ret = srp_map_sg_entry(state, ch, sg);
1468                 if (ret)
1469                         return ret;
1470         }
1471
1472         ret = srp_map_finish_fmr(state, ch);
1473         if (ret)
1474                 return ret;
1475
1476         return 0;
1477 }
1478
1479 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1480                          struct srp_request *req, struct scatterlist *scat,
1481                          int count)
1482 {
1483         unsigned int sg_offset = 0;
1484
1485         state->fr.next = req->fr_list;
1486         state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1487         state->sg = scat;
1488
1489         if (count == 0)
1490                 return 0;
1491
1492         while (count) {
1493                 int i, n;
1494
1495                 n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1496                 if (unlikely(n < 0))
1497                         return n;
1498
1499                 count -= n;
1500                 for (i = 0; i < n; i++)
1501                         state->sg = sg_next(state->sg);
1502         }
1503
1504         return 0;
1505 }
1506
1507 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1508                           struct srp_request *req, struct scatterlist *scat,
1509                           int count)
1510 {
1511         struct srp_target_port *target = ch->target;
1512         struct srp_device *dev = target->srp_host->srp_dev;
1513         struct scatterlist *sg;
1514         int i;
1515
1516         for_each_sg(scat, sg, count, i) {
1517                 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1518                              ib_sg_dma_len(dev->dev, sg),
1519                              target->pd->unsafe_global_rkey);
1520         }
1521
1522         return 0;
1523 }
1524
1525 /*
1526  * Register the indirect data buffer descriptor with the HCA.
1527  *
1528  * Note: since the indirect data buffer descriptor has been allocated with
1529  * kmalloc() it is guaranteed that this buffer is a physically contiguous
1530  * memory buffer.
1531  */
1532 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1533                        void **next_mr, void **end_mr, u32 idb_len,
1534                        __be32 *idb_rkey)
1535 {
1536         struct srp_target_port *target = ch->target;
1537         struct srp_device *dev = target->srp_host->srp_dev;
1538         struct srp_map_state state;
1539         struct srp_direct_buf idb_desc;
1540         u64 idb_pages[1];
1541         struct scatterlist idb_sg[1];
1542         int ret;
1543
1544         memset(&state, 0, sizeof(state));
1545         memset(&idb_desc, 0, sizeof(idb_desc));
1546         state.gen.next = next_mr;
1547         state.gen.end = end_mr;
1548         state.desc = &idb_desc;
1549         state.base_dma_addr = req->indirect_dma_addr;
1550         state.dma_len = idb_len;
1551
1552         if (dev->use_fast_reg) {
1553                 state.sg = idb_sg;
1554                 sg_init_one(idb_sg, req->indirect_desc, idb_len);
1555                 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1556 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1557                 idb_sg->dma_length = idb_sg->length;          /* hack^2 */
1558 #endif
1559                 ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1560                 if (ret < 0)
1561                         return ret;
1562                 WARN_ON_ONCE(ret < 1);
1563         } else if (dev->use_fmr) {
1564                 state.pages = idb_pages;
1565                 state.pages[0] = (req->indirect_dma_addr &
1566                                   dev->mr_page_mask);
1567                 state.npages = 1;
1568                 ret = srp_map_finish_fmr(&state, ch);
1569                 if (ret < 0)
1570                         return ret;
1571         } else {
1572                 return -EINVAL;
1573         }
1574
1575         *idb_rkey = idb_desc.key;
1576
1577         return 0;
1578 }
1579
1580 static void srp_check_mapping(struct srp_map_state *state,
1581                               struct srp_rdma_ch *ch, struct srp_request *req,
1582                               struct scatterlist *scat, int count)
1583 {
1584         struct srp_device *dev = ch->target->srp_host->srp_dev;
1585         struct srp_fr_desc **pfr;
1586         u64 desc_len = 0, mr_len = 0;
1587         int i;
1588
1589         for (i = 0; i < state->ndesc; i++)
1590                 desc_len += be32_to_cpu(req->indirect_desc[i].len);
1591         if (dev->use_fast_reg)
1592                 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
1593                         mr_len += (*pfr)->mr->length;
1594         else if (dev->use_fmr)
1595                 for (i = 0; i < state->nmdesc; i++)
1596                         mr_len += be32_to_cpu(req->indirect_desc[i].len);
1597         if (desc_len != scsi_bufflen(req->scmnd) ||
1598             mr_len > scsi_bufflen(req->scmnd))
1599                 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
1600                        scsi_bufflen(req->scmnd), desc_len, mr_len,
1601                        state->ndesc, state->nmdesc);
1602 }
1603
1604 /**
1605  * srp_map_data() - map SCSI data buffer onto an SRP request
1606  * @scmnd: SCSI command to map
1607  * @ch: SRP RDMA channel
1608  * @req: SRP request
1609  *
1610  * Returns the length in bytes of the SRP_CMD IU or a negative value if
1611  * mapping failed.
1612  */
1613 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1614                         struct srp_request *req)
1615 {
1616         struct srp_target_port *target = ch->target;
1617         struct ib_pd *pd = target->pd;
1618         struct scatterlist *scat;
1619         struct srp_cmd *cmd = req->cmd->buf;
1620         int len, nents, count, ret;
1621         struct srp_device *dev;
1622         struct ib_device *ibdev;
1623         struct srp_map_state state;
1624         struct srp_indirect_buf *indirect_hdr;
1625         u32 idb_len, table_len;
1626         __be32 idb_rkey;
1627         u8 fmt;
1628
1629         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1630                 return sizeof (struct srp_cmd);
1631
1632         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1633             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1634                 shost_printk(KERN_WARNING, target->scsi_host,
1635                              PFX "Unhandled data direction %d\n",
1636                              scmnd->sc_data_direction);
1637                 return -EINVAL;
1638         }
1639
1640         nents = scsi_sg_count(scmnd);
1641         scat  = scsi_sglist(scmnd);
1642
1643         dev = target->srp_host->srp_dev;
1644         ibdev = dev->dev;
1645
1646         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1647         if (unlikely(count == 0))
1648                 return -EIO;
1649
1650         fmt = SRP_DATA_DESC_DIRECT;
1651         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1652
1653         if (count == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1654                 /*
1655                  * The midlayer only generated a single gather/scatter
1656                  * entry, or DMA mapping coalesced everything to a
1657                  * single entry.  So a direct descriptor along with
1658                  * the DMA MR suffices.
1659                  */
1660                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1661
1662                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1663                 buf->key = cpu_to_be32(pd->unsafe_global_rkey);
1664                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1665
1666                 req->nmdesc = 0;
1667                 goto map_complete;
1668         }
1669
1670         /*
1671          * We have more than one scatter/gather entry, so build our indirect
1672          * descriptor table, trying to merge as many entries as we can.
1673          */
1674         indirect_hdr = (void *) cmd->add_data;
1675
1676         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1677                                    target->indirect_size, DMA_TO_DEVICE);
1678
1679         memset(&state, 0, sizeof(state));
1680         state.desc = req->indirect_desc;
1681         if (dev->use_fast_reg)
1682                 ret = srp_map_sg_fr(&state, ch, req, scat, count);
1683         else if (dev->use_fmr)
1684                 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
1685         else
1686                 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1687         req->nmdesc = state.nmdesc;
1688         if (ret < 0)
1689                 goto unmap;
1690
1691         {
1692                 DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
1693                         "Memory mapping consistency check");
1694                 if (DYNAMIC_DEBUG_BRANCH(ddm))
1695                         srp_check_mapping(&state, ch, req, scat, count);
1696         }
1697
1698         /* We've mapped the request, now pull as much of the indirect
1699          * descriptor table as we can into the command buffer. If this
1700          * target is not using an external indirect table, we are
1701          * guaranteed to fit into the command, as the SCSI layer won't
1702          * give us more S/G entries than we allow.
1703          */
1704         if (state.ndesc == 1) {
1705                 /*
1706                  * Memory registration collapsed the sg-list into one entry,
1707                  * so use a direct descriptor.
1708                  */
1709                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1710
1711                 *buf = req->indirect_desc[0];
1712                 goto map_complete;
1713         }
1714
1715         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1716                                                 !target->allow_ext_sg)) {
1717                 shost_printk(KERN_ERR, target->scsi_host,
1718                              "Could not fit S/G list into SRP_CMD\n");
1719                 ret = -EIO;
1720                 goto unmap;
1721         }
1722
1723         count = min(state.ndesc, target->cmd_sg_cnt);
1724         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1725         idb_len = sizeof(struct srp_indirect_buf) + table_len;
1726
1727         fmt = SRP_DATA_DESC_INDIRECT;
1728         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1729         len += count * sizeof (struct srp_direct_buf);
1730
1731         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1732                count * sizeof (struct srp_direct_buf));
1733
1734         if (!(pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1735                 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1736                                   idb_len, &idb_rkey);
1737                 if (ret < 0)
1738                         goto unmap;
1739                 req->nmdesc++;
1740         } else {
1741                 idb_rkey = cpu_to_be32(pd->unsafe_global_rkey);
1742         }
1743
1744         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1745         indirect_hdr->table_desc.key = idb_rkey;
1746         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1747         indirect_hdr->len = cpu_to_be32(state.total_len);
1748
1749         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1750                 cmd->data_out_desc_cnt = count;
1751         else
1752                 cmd->data_in_desc_cnt = count;
1753
1754         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1755                                       DMA_TO_DEVICE);
1756
1757 map_complete:
1758         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1759                 cmd->buf_fmt = fmt << 4;
1760         else
1761                 cmd->buf_fmt = fmt;
1762
1763         return len;
1764
1765 unmap:
1766         srp_unmap_data(scmnd, ch, req);
1767         if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
1768                 ret = -E2BIG;
1769         return ret;
1770 }
1771
1772 /*
1773  * Return an IU and possible credit to the free pool
1774  */
1775 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1776                           enum srp_iu_type iu_type)
1777 {
1778         unsigned long flags;
1779
1780         spin_lock_irqsave(&ch->lock, flags);
1781         list_add(&iu->list, &ch->free_tx);
1782         if (iu_type != SRP_IU_RSP)
1783                 ++ch->req_lim;
1784         spin_unlock_irqrestore(&ch->lock, flags);
1785 }
1786
1787 /*
1788  * Must be called with ch->lock held to protect req_lim and free_tx.
1789  * If IU is not sent, it must be returned using srp_put_tx_iu().
1790  *
1791  * Note:
1792  * An upper limit for the number of allocated information units for each
1793  * request type is:
1794  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1795  *   more than Scsi_Host.can_queue requests.
1796  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1797  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1798  *   one unanswered SRP request to an initiator.
1799  */
1800 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1801                                       enum srp_iu_type iu_type)
1802 {
1803         struct srp_target_port *target = ch->target;
1804         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1805         struct srp_iu *iu;
1806
1807         ib_process_cq_direct(ch->send_cq, -1);
1808
1809         if (list_empty(&ch->free_tx))
1810                 return NULL;
1811
1812         /* Initiator responses to target requests do not consume credits */
1813         if (iu_type != SRP_IU_RSP) {
1814                 if (ch->req_lim <= rsv) {
1815                         ++target->zero_req_lim;
1816                         return NULL;
1817                 }
1818
1819                 --ch->req_lim;
1820         }
1821
1822         iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1823         list_del(&iu->list);
1824         return iu;
1825 }
1826
1827 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1828 {
1829         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1830         struct srp_rdma_ch *ch = cq->cq_context;
1831
1832         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1833                 srp_handle_qp_err(cq, wc, "SEND");
1834                 return;
1835         }
1836
1837         list_add(&iu->list, &ch->free_tx);
1838 }
1839
1840 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1841 {
1842         struct srp_target_port *target = ch->target;
1843         struct ib_sge list;
1844         struct ib_send_wr wr, *bad_wr;
1845
1846         list.addr   = iu->dma;
1847         list.length = len;
1848         list.lkey   = target->lkey;
1849
1850         iu->cqe.done = srp_send_done;
1851
1852         wr.next       = NULL;
1853         wr.wr_cqe     = &iu->cqe;
1854         wr.sg_list    = &list;
1855         wr.num_sge    = 1;
1856         wr.opcode     = IB_WR_SEND;
1857         wr.send_flags = IB_SEND_SIGNALED;
1858
1859         return ib_post_send(ch->qp, &wr, &bad_wr);
1860 }
1861
1862 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1863 {
1864         struct srp_target_port *target = ch->target;
1865         struct ib_recv_wr wr, *bad_wr;
1866         struct ib_sge list;
1867
1868         list.addr   = iu->dma;
1869         list.length = iu->size;
1870         list.lkey   = target->lkey;
1871
1872         iu->cqe.done = srp_recv_done;
1873
1874         wr.next     = NULL;
1875         wr.wr_cqe   = &iu->cqe;
1876         wr.sg_list  = &list;
1877         wr.num_sge  = 1;
1878
1879         return ib_post_recv(ch->qp, &wr, &bad_wr);
1880 }
1881
1882 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1883 {
1884         struct srp_target_port *target = ch->target;
1885         struct srp_request *req;
1886         struct scsi_cmnd *scmnd;
1887         unsigned long flags;
1888
1889         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1890                 spin_lock_irqsave(&ch->lock, flags);
1891                 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1892                 spin_unlock_irqrestore(&ch->lock, flags);
1893
1894                 ch->tsk_mgmt_status = -1;
1895                 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1896                         ch->tsk_mgmt_status = rsp->data[3];
1897                 complete(&ch->tsk_mgmt_done);
1898         } else {
1899                 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1900                 if (scmnd) {
1901                         req = (void *)scmnd->host_scribble;
1902                         scmnd = srp_claim_req(ch, req, NULL, scmnd);
1903                 }
1904                 if (!scmnd) {
1905                         shost_printk(KERN_ERR, target->scsi_host,
1906                                      "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1907                                      rsp->tag, ch - target->ch, ch->qp->qp_num);
1908
1909                         spin_lock_irqsave(&ch->lock, flags);
1910                         ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1911                         spin_unlock_irqrestore(&ch->lock, flags);
1912
1913                         return;
1914                 }
1915                 scmnd->result = rsp->status;
1916
1917                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1918                         memcpy(scmnd->sense_buffer, rsp->data +
1919                                be32_to_cpu(rsp->resp_data_len),
1920                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1921                                      SCSI_SENSE_BUFFERSIZE));
1922                 }
1923
1924                 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1925                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1926                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1927                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1928                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1929                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1930                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1931                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1932
1933                 srp_free_req(ch, req, scmnd,
1934                              be32_to_cpu(rsp->req_lim_delta));
1935
1936                 scmnd->host_scribble = NULL;
1937                 scmnd->scsi_done(scmnd);
1938         }
1939 }
1940
1941 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1942                                void *rsp, int len)
1943 {
1944         struct srp_target_port *target = ch->target;
1945         struct ib_device *dev = target->srp_host->srp_dev->dev;
1946         unsigned long flags;
1947         struct srp_iu *iu;
1948         int err;
1949
1950         spin_lock_irqsave(&ch->lock, flags);
1951         ch->req_lim += req_delta;
1952         iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1953         spin_unlock_irqrestore(&ch->lock, flags);
1954
1955         if (!iu) {
1956                 shost_printk(KERN_ERR, target->scsi_host, PFX
1957                              "no IU available to send response\n");
1958                 return 1;
1959         }
1960
1961         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1962         memcpy(iu->buf, rsp, len);
1963         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1964
1965         err = srp_post_send(ch, iu, len);
1966         if (err) {
1967                 shost_printk(KERN_ERR, target->scsi_host, PFX
1968                              "unable to post response: %d\n", err);
1969                 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1970         }
1971
1972         return err;
1973 }
1974
1975 static void srp_process_cred_req(struct srp_rdma_ch *ch,
1976                                  struct srp_cred_req *req)
1977 {
1978         struct srp_cred_rsp rsp = {
1979                 .opcode = SRP_CRED_RSP,
1980                 .tag = req->tag,
1981         };
1982         s32 delta = be32_to_cpu(req->req_lim_delta);
1983
1984         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1985                 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1986                              "problems processing SRP_CRED_REQ\n");
1987 }
1988
1989 static void srp_process_aer_req(struct srp_rdma_ch *ch,
1990                                 struct srp_aer_req *req)
1991 {
1992         struct srp_target_port *target = ch->target;
1993         struct srp_aer_rsp rsp = {
1994                 .opcode = SRP_AER_RSP,
1995                 .tag = req->tag,
1996         };
1997         s32 delta = be32_to_cpu(req->req_lim_delta);
1998
1999         shost_printk(KERN_ERR, target->scsi_host, PFX
2000                      "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
2001
2002         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2003                 shost_printk(KERN_ERR, target->scsi_host, PFX
2004                              "problems processing SRP_AER_REQ\n");
2005 }
2006
2007 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2008 {
2009         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
2010         struct srp_rdma_ch *ch = cq->cq_context;
2011         struct srp_target_port *target = ch->target;
2012         struct ib_device *dev = target->srp_host->srp_dev->dev;
2013         int res;
2014         u8 opcode;
2015
2016         if (unlikely(wc->status != IB_WC_SUCCESS)) {
2017                 srp_handle_qp_err(cq, wc, "RECV");
2018                 return;
2019         }
2020
2021         ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2022                                    DMA_FROM_DEVICE);
2023
2024         opcode = *(u8 *) iu->buf;
2025
2026         if (0) {
2027                 shost_printk(KERN_ERR, target->scsi_host,
2028                              PFX "recv completion, opcode 0x%02x\n", opcode);
2029                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
2030                                iu->buf, wc->byte_len, true);
2031         }
2032
2033         switch (opcode) {
2034         case SRP_RSP:
2035                 srp_process_rsp(ch, iu->buf);
2036                 break;
2037
2038         case SRP_CRED_REQ:
2039                 srp_process_cred_req(ch, iu->buf);
2040                 break;
2041
2042         case SRP_AER_REQ:
2043                 srp_process_aer_req(ch, iu->buf);
2044                 break;
2045
2046         case SRP_T_LOGOUT:
2047                 /* XXX Handle target logout */
2048                 shost_printk(KERN_WARNING, target->scsi_host,
2049                              PFX "Got target logout request\n");
2050                 break;
2051
2052         default:
2053                 shost_printk(KERN_WARNING, target->scsi_host,
2054                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2055                 break;
2056         }
2057
2058         ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2059                                       DMA_FROM_DEVICE);
2060
2061         res = srp_post_recv(ch, iu);
2062         if (res != 0)
2063                 shost_printk(KERN_ERR, target->scsi_host,
2064                              PFX "Recv failed with error code %d\n", res);
2065 }
2066
2067 /**
2068  * srp_tl_err_work() - handle a transport layer error
2069  * @work: Work structure embedded in an SRP target port.
2070  *
2071  * Note: This function may get invoked before the rport has been created,
2072  * hence the target->rport test.
2073  */
2074 static void srp_tl_err_work(struct work_struct *work)
2075 {
2076         struct srp_target_port *target;
2077
2078         target = container_of(work, struct srp_target_port, tl_err_work);
2079         if (target->rport)
2080                 srp_start_tl_fail_timers(target->rport);
2081 }
2082
2083 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2084                 const char *opname)
2085 {
2086         struct srp_rdma_ch *ch = cq->cq_context;
2087         struct srp_target_port *target = ch->target;
2088
2089         if (ch->connected && !target->qp_in_error) {
2090                 shost_printk(KERN_ERR, target->scsi_host,
2091                              PFX "failed %s status %s (%d) for CQE %p\n",
2092                              opname, ib_wc_status_msg(wc->status), wc->status,
2093                              wc->wr_cqe);
2094                 queue_work(system_long_wq, &target->tl_err_work);
2095         }
2096         target->qp_in_error = true;
2097 }
2098
2099 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2100 {
2101         struct srp_target_port *target = host_to_target(shost);
2102         struct srp_rport *rport = target->rport;
2103         struct srp_rdma_ch *ch;
2104         struct srp_request *req;
2105         struct srp_iu *iu;
2106         struct srp_cmd *cmd;
2107         struct ib_device *dev;
2108         unsigned long flags;
2109         u32 tag;
2110         u16 idx;
2111         int len, ret;
2112         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2113
2114         /*
2115          * The SCSI EH thread is the only context from which srp_queuecommand()
2116          * can get invoked for blocked devices (SDEV_BLOCK /
2117          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2118          * locking the rport mutex if invoked from inside the SCSI EH.
2119          */
2120         if (in_scsi_eh)
2121                 mutex_lock(&rport->mutex);
2122
2123         scmnd->result = srp_chkready(target->rport);
2124         if (unlikely(scmnd->result))
2125                 goto err;
2126
2127         WARN_ON_ONCE(scmnd->request->tag < 0);
2128         tag = blk_mq_unique_tag(scmnd->request);
2129         ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2130         idx = blk_mq_unique_tag_to_tag(tag);
2131         WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2132                   dev_name(&shost->shost_gendev), tag, idx,
2133                   target->req_ring_size);
2134
2135         spin_lock_irqsave(&ch->lock, flags);
2136         iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2137         spin_unlock_irqrestore(&ch->lock, flags);
2138
2139         if (!iu)
2140                 goto err;
2141
2142         req = &ch->req_ring[idx];
2143         dev = target->srp_host->srp_dev->dev;
2144         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2145                                    DMA_TO_DEVICE);
2146
2147         scmnd->host_scribble = (void *) req;
2148
2149         cmd = iu->buf;
2150         memset(cmd, 0, sizeof *cmd);
2151
2152         cmd->opcode = SRP_CMD;
2153         int_to_scsilun(scmnd->device->lun, &cmd->lun);
2154         cmd->tag    = tag;
2155         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2156
2157         req->scmnd    = scmnd;
2158         req->cmd      = iu;
2159
2160         len = srp_map_data(scmnd, ch, req);
2161         if (len < 0) {
2162                 shost_printk(KERN_ERR, target->scsi_host,
2163                              PFX "Failed to map data (%d)\n", len);
2164                 /*
2165                  * If we ran out of memory descriptors (-ENOMEM) because an
2166                  * application is queuing many requests with more than
2167                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2168                  * to reduce queue depth temporarily.
2169                  */
2170                 scmnd->result = len == -ENOMEM ?
2171                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2172                 goto err_iu;
2173         }
2174
2175         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2176                                       DMA_TO_DEVICE);
2177
2178         if (srp_post_send(ch, iu, len)) {
2179                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2180                 goto err_unmap;
2181         }
2182
2183         ret = 0;
2184
2185 unlock_rport:
2186         if (in_scsi_eh)
2187                 mutex_unlock(&rport->mutex);
2188
2189         return ret;
2190
2191 err_unmap:
2192         srp_unmap_data(scmnd, ch, req);
2193
2194 err_iu:
2195         srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2196
2197         /*
2198          * Avoid that the loops that iterate over the request ring can
2199          * encounter a dangling SCSI command pointer.
2200          */
2201         req->scmnd = NULL;
2202
2203 err:
2204         if (scmnd->result) {
2205                 scmnd->scsi_done(scmnd);
2206                 ret = 0;
2207         } else {
2208                 ret = SCSI_MLQUEUE_HOST_BUSY;
2209         }
2210
2211         goto unlock_rport;
2212 }
2213
2214 /*
2215  * Note: the resources allocated in this function are freed in
2216  * srp_free_ch_ib().
2217  */
2218 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2219 {
2220         struct srp_target_port *target = ch->target;
2221         int i;
2222
2223         ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2224                               GFP_KERNEL);
2225         if (!ch->rx_ring)
2226                 goto err_no_ring;
2227         ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2228                               GFP_KERNEL);
2229         if (!ch->tx_ring)
2230                 goto err_no_ring;
2231
2232         for (i = 0; i < target->queue_size; ++i) {
2233                 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2234                                               ch->max_ti_iu_len,
2235                                               GFP_KERNEL, DMA_FROM_DEVICE);
2236                 if (!ch->rx_ring[i])
2237                         goto err;
2238         }
2239
2240         for (i = 0; i < target->queue_size; ++i) {
2241                 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2242                                               target->max_iu_len,
2243                                               GFP_KERNEL, DMA_TO_DEVICE);
2244                 if (!ch->tx_ring[i])
2245                         goto err;
2246
2247                 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2248         }
2249
2250         return 0;
2251
2252 err:
2253         for (i = 0; i < target->queue_size; ++i) {
2254                 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2255                 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2256         }
2257
2258
2259 err_no_ring:
2260         kfree(ch->tx_ring);
2261         ch->tx_ring = NULL;
2262         kfree(ch->rx_ring);
2263         ch->rx_ring = NULL;
2264
2265         return -ENOMEM;
2266 }
2267
2268 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2269 {
2270         uint64_t T_tr_ns, max_compl_time_ms;
2271         uint32_t rq_tmo_jiffies;
2272
2273         /*
2274          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2275          * table 91), both the QP timeout and the retry count have to be set
2276          * for RC QP's during the RTR to RTS transition.
2277          */
2278         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2279                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2280
2281         /*
2282          * Set target->rq_tmo_jiffies to one second more than the largest time
2283          * it can take before an error completion is generated. See also
2284          * C9-140..142 in the IBTA spec for more information about how to
2285          * convert the QP Local ACK Timeout value to nanoseconds.
2286          */
2287         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2288         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2289         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2290         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2291
2292         return rq_tmo_jiffies;
2293 }
2294
2295 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2296                                const struct srp_login_rsp *lrsp,
2297                                struct srp_rdma_ch *ch)
2298 {
2299         struct srp_target_port *target = ch->target;
2300         struct ib_qp_attr *qp_attr = NULL;
2301         int attr_mask = 0;
2302         int ret;
2303         int i;
2304
2305         if (lrsp->opcode == SRP_LOGIN_RSP) {
2306                 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2307                 ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2308
2309                 /*
2310                  * Reserve credits for task management so we don't
2311                  * bounce requests back to the SCSI mid-layer.
2312                  */
2313                 target->scsi_host->can_queue
2314                         = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2315                               target->scsi_host->can_queue);
2316                 target->scsi_host->cmd_per_lun
2317                         = min_t(int, target->scsi_host->can_queue,
2318                                 target->scsi_host->cmd_per_lun);
2319         } else {
2320                 shost_printk(KERN_WARNING, target->scsi_host,
2321                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2322                 ret = -ECONNRESET;
2323                 goto error;
2324         }
2325
2326         if (!ch->rx_ring) {
2327                 ret = srp_alloc_iu_bufs(ch);
2328                 if (ret)
2329                         goto error;
2330         }
2331
2332         ret = -ENOMEM;
2333         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2334         if (!qp_attr)
2335                 goto error;
2336
2337         qp_attr->qp_state = IB_QPS_RTR;
2338         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2339         if (ret)
2340                 goto error_free;
2341
2342         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2343         if (ret)
2344                 goto error_free;
2345
2346         for (i = 0; i < target->queue_size; i++) {
2347                 struct srp_iu *iu = ch->rx_ring[i];
2348
2349                 ret = srp_post_recv(ch, iu);
2350                 if (ret)
2351                         goto error_free;
2352         }
2353
2354         qp_attr->qp_state = IB_QPS_RTS;
2355         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2356         if (ret)
2357                 goto error_free;
2358
2359         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2360
2361         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2362         if (ret)
2363                 goto error_free;
2364
2365         ret = ib_send_cm_rtu(cm_id, NULL, 0);
2366
2367 error_free:
2368         kfree(qp_attr);
2369
2370 error:
2371         ch->status = ret;
2372 }
2373
2374 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2375                                struct ib_cm_event *event,
2376                                struct srp_rdma_ch *ch)
2377 {
2378         struct srp_target_port *target = ch->target;
2379         struct Scsi_Host *shost = target->scsi_host;
2380         struct ib_class_port_info *cpi;
2381         int opcode;
2382
2383         switch (event->param.rej_rcvd.reason) {
2384         case IB_CM_REJ_PORT_CM_REDIRECT:
2385                 cpi = event->param.rej_rcvd.ari;
2386                 ch->path.dlid = cpi->redirect_lid;
2387                 ch->path.pkey = cpi->redirect_pkey;
2388                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2389                 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2390
2391                 ch->status = ch->path.dlid ?
2392                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2393                 break;
2394
2395         case IB_CM_REJ_PORT_REDIRECT:
2396                 if (srp_target_is_topspin(target)) {
2397                         /*
2398                          * Topspin/Cisco SRP gateways incorrectly send
2399                          * reject reason code 25 when they mean 24
2400                          * (port redirect).
2401                          */
2402                         memcpy(ch->path.dgid.raw,
2403                                event->param.rej_rcvd.ari, 16);
2404
2405                         shost_printk(KERN_DEBUG, shost,
2406                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2407                                      be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2408                                      be64_to_cpu(ch->path.dgid.global.interface_id));
2409
2410                         ch->status = SRP_PORT_REDIRECT;
2411                 } else {
2412                         shost_printk(KERN_WARNING, shost,
2413                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2414                         ch->status = -ECONNRESET;
2415                 }
2416                 break;
2417
2418         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2419                 shost_printk(KERN_WARNING, shost,
2420                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2421                 ch->status = -ECONNRESET;
2422                 break;
2423
2424         case IB_CM_REJ_CONSUMER_DEFINED:
2425                 opcode = *(u8 *) event->private_data;
2426                 if (opcode == SRP_LOGIN_REJ) {
2427                         struct srp_login_rej *rej = event->private_data;
2428                         u32 reason = be32_to_cpu(rej->reason);
2429
2430                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2431                                 shost_printk(KERN_WARNING, shost,
2432                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2433                         else
2434                                 shost_printk(KERN_WARNING, shost, PFX
2435                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2436                                              target->sgid.raw,
2437                                              target->orig_dgid.raw, reason);
2438                 } else
2439                         shost_printk(KERN_WARNING, shost,
2440                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2441                                      " opcode 0x%02x\n", opcode);
2442                 ch->status = -ECONNRESET;
2443                 break;
2444
2445         case IB_CM_REJ_STALE_CONN:
2446                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2447                 ch->status = SRP_STALE_CONN;
2448                 break;
2449
2450         default:
2451                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2452                              event->param.rej_rcvd.reason);
2453                 ch->status = -ECONNRESET;
2454         }
2455 }
2456
2457 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2458 {
2459         struct srp_rdma_ch *ch = cm_id->context;
2460         struct srp_target_port *target = ch->target;
2461         int comp = 0;
2462
2463         switch (event->event) {
2464         case IB_CM_REQ_ERROR:
2465                 shost_printk(KERN_DEBUG, target->scsi_host,
2466                              PFX "Sending CM REQ failed\n");
2467                 comp = 1;
2468                 ch->status = -ECONNRESET;
2469                 break;
2470
2471         case IB_CM_REP_RECEIVED:
2472                 comp = 1;
2473                 srp_cm_rep_handler(cm_id, event->private_data, ch);
2474                 break;
2475
2476         case IB_CM_REJ_RECEIVED:
2477                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2478                 comp = 1;
2479
2480                 srp_cm_rej_handler(cm_id, event, ch);
2481                 break;
2482
2483         case IB_CM_DREQ_RECEIVED:
2484                 shost_printk(KERN_WARNING, target->scsi_host,
2485                              PFX "DREQ received - connection closed\n");
2486                 ch->connected = false;
2487                 if (ib_send_cm_drep(cm_id, NULL, 0))
2488                         shost_printk(KERN_ERR, target->scsi_host,
2489                                      PFX "Sending CM DREP failed\n");
2490                 queue_work(system_long_wq, &target->tl_err_work);
2491                 break;
2492
2493         case IB_CM_TIMEWAIT_EXIT:
2494                 shost_printk(KERN_ERR, target->scsi_host,
2495                              PFX "connection closed\n");
2496                 comp = 1;
2497
2498                 ch->status = 0;
2499                 break;
2500
2501         case IB_CM_MRA_RECEIVED:
2502         case IB_CM_DREQ_ERROR:
2503         case IB_CM_DREP_RECEIVED:
2504                 break;
2505
2506         default:
2507                 shost_printk(KERN_WARNING, target->scsi_host,
2508                              PFX "Unhandled CM event %d\n", event->event);
2509                 break;
2510         }
2511
2512         if (comp)
2513                 complete(&ch->done);
2514
2515         return 0;
2516 }
2517
2518 /**
2519  * srp_change_queue_depth - setting device queue depth
2520  * @sdev: scsi device struct
2521  * @qdepth: requested queue depth
2522  *
2523  * Returns queue depth.
2524  */
2525 static int
2526 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2527 {
2528         if (!sdev->tagged_supported)
2529                 qdepth = 1;
2530         return scsi_change_queue_depth(sdev, qdepth);
2531 }
2532
2533 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2534                              u8 func)
2535 {
2536         struct srp_target_port *target = ch->target;
2537         struct srp_rport *rport = target->rport;
2538         struct ib_device *dev = target->srp_host->srp_dev->dev;
2539         struct srp_iu *iu;
2540         struct srp_tsk_mgmt *tsk_mgmt;
2541
2542         if (!ch->connected || target->qp_in_error)
2543                 return -1;
2544
2545         init_completion(&ch->tsk_mgmt_done);
2546
2547         /*
2548          * Lock the rport mutex to avoid that srp_create_ch_ib() is
2549          * invoked while a task management function is being sent.
2550          */
2551         mutex_lock(&rport->mutex);
2552         spin_lock_irq(&ch->lock);
2553         iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2554         spin_unlock_irq(&ch->lock);
2555
2556         if (!iu) {
2557                 mutex_unlock(&rport->mutex);
2558
2559                 return -1;
2560         }
2561
2562         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2563                                    DMA_TO_DEVICE);
2564         tsk_mgmt = iu->buf;
2565         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2566
2567         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2568         int_to_scsilun(lun, &tsk_mgmt->lun);
2569         tsk_mgmt->tag           = req_tag | SRP_TAG_TSK_MGMT;
2570         tsk_mgmt->tsk_mgmt_func = func;
2571         tsk_mgmt->task_tag      = req_tag;
2572
2573         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2574                                       DMA_TO_DEVICE);
2575         if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2576                 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2577                 mutex_unlock(&rport->mutex);
2578
2579                 return -1;
2580         }
2581         mutex_unlock(&rport->mutex);
2582
2583         if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2584                                          msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2585                 return -1;
2586
2587         return 0;
2588 }
2589
2590 static int srp_abort(struct scsi_cmnd *scmnd)
2591 {
2592         struct srp_target_port *target = host_to_target(scmnd->device->host);
2593         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2594         u32 tag;
2595         u16 ch_idx;
2596         struct srp_rdma_ch *ch;
2597         int ret;
2598
2599         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2600
2601         if (!req)
2602                 return SUCCESS;
2603         tag = blk_mq_unique_tag(scmnd->request);
2604         ch_idx = blk_mq_unique_tag_to_hwq(tag);
2605         if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2606                 return SUCCESS;
2607         ch = &target->ch[ch_idx];
2608         if (!srp_claim_req(ch, req, NULL, scmnd))
2609                 return SUCCESS;
2610         shost_printk(KERN_ERR, target->scsi_host,
2611                      "Sending SRP abort for tag %#x\n", tag);
2612         if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2613                               SRP_TSK_ABORT_TASK) == 0)
2614                 ret = SUCCESS;
2615         else if (target->rport->state == SRP_RPORT_LOST)
2616                 ret = FAST_IO_FAIL;
2617         else
2618                 ret = FAILED;
2619         srp_free_req(ch, req, scmnd, 0);
2620         scmnd->result = DID_ABORT << 16;
2621         scmnd->scsi_done(scmnd);
2622
2623         return ret;
2624 }
2625
2626 static int srp_reset_device(struct scsi_cmnd *scmnd)
2627 {
2628         struct srp_target_port *target = host_to_target(scmnd->device->host);
2629         struct srp_rdma_ch *ch;
2630         int i;
2631
2632         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2633
2634         ch = &target->ch[0];
2635         if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2636                               SRP_TSK_LUN_RESET))
2637                 return FAILED;
2638         if (ch->tsk_mgmt_status)
2639                 return FAILED;
2640
2641         for (i = 0; i < target->ch_count; i++) {
2642                 ch = &target->ch[i];
2643                 for (i = 0; i < target->req_ring_size; ++i) {
2644                         struct srp_request *req = &ch->req_ring[i];
2645
2646                         srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2647                 }
2648         }
2649
2650         return SUCCESS;
2651 }
2652
2653 static int srp_reset_host(struct scsi_cmnd *scmnd)
2654 {
2655         struct srp_target_port *target = host_to_target(scmnd->device->host);
2656
2657         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2658
2659         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2660 }
2661
2662 static int srp_slave_alloc(struct scsi_device *sdev)
2663 {
2664         struct Scsi_Host *shost = sdev->host;
2665         struct srp_target_port *target = host_to_target(shost);
2666         struct srp_device *srp_dev = target->srp_host->srp_dev;
2667         struct ib_device *ibdev = srp_dev->dev;
2668
2669         if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
2670                 blk_queue_virt_boundary(sdev->request_queue,
2671                                         ~srp_dev->mr_page_mask);
2672
2673         return 0;
2674 }
2675
2676 static int srp_slave_configure(struct scsi_device *sdev)
2677 {
2678         struct Scsi_Host *shost = sdev->host;
2679         struct srp_target_port *target = host_to_target(shost);
2680         struct request_queue *q = sdev->request_queue;
2681         unsigned long timeout;
2682
2683         if (sdev->type == TYPE_DISK) {
2684                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2685                 blk_queue_rq_timeout(q, timeout);
2686         }
2687
2688         return 0;
2689 }
2690
2691 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2692                            char *buf)
2693 {
2694         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2695
2696         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2697 }
2698
2699 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2700                              char *buf)
2701 {
2702         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2703
2704         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2705 }
2706
2707 static ssize_t show_service_id(struct device *dev,
2708                                struct device_attribute *attr, char *buf)
2709 {
2710         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2711
2712         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2713 }
2714
2715 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2716                          char *buf)
2717 {
2718         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2719
2720         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2721 }
2722
2723 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2724                          char *buf)
2725 {
2726         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2727
2728         return sprintf(buf, "%pI6\n", target->sgid.raw);
2729 }
2730
2731 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2732                          char *buf)
2733 {
2734         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2735         struct srp_rdma_ch *ch = &target->ch[0];
2736
2737         return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2738 }
2739
2740 static ssize_t show_orig_dgid(struct device *dev,
2741                               struct device_attribute *attr, char *buf)
2742 {
2743         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2744
2745         return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2746 }
2747
2748 static ssize_t show_req_lim(struct device *dev,
2749                             struct device_attribute *attr, char *buf)
2750 {
2751         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2752         struct srp_rdma_ch *ch;
2753         int i, req_lim = INT_MAX;
2754
2755         for (i = 0; i < target->ch_count; i++) {
2756                 ch = &target->ch[i];
2757                 req_lim = min(req_lim, ch->req_lim);
2758         }
2759         return sprintf(buf, "%d\n", req_lim);
2760 }
2761
2762 static ssize_t show_zero_req_lim(struct device *dev,
2763                                  struct device_attribute *attr, char *buf)
2764 {
2765         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2766
2767         return sprintf(buf, "%d\n", target->zero_req_lim);
2768 }
2769
2770 static ssize_t show_local_ib_port(struct device *dev,
2771                                   struct device_attribute *attr, char *buf)
2772 {
2773         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2774
2775         return sprintf(buf, "%d\n", target->srp_host->port);
2776 }
2777
2778 static ssize_t show_local_ib_device(struct device *dev,
2779                                     struct device_attribute *attr, char *buf)
2780 {
2781         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2782
2783         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2784 }
2785
2786 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2787                              char *buf)
2788 {
2789         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2790
2791         return sprintf(buf, "%d\n", target->ch_count);
2792 }
2793
2794 static ssize_t show_comp_vector(struct device *dev,
2795                                 struct device_attribute *attr, char *buf)
2796 {
2797         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2798
2799         return sprintf(buf, "%d\n", target->comp_vector);
2800 }
2801
2802 static ssize_t show_tl_retry_count(struct device *dev,
2803                                    struct device_attribute *attr, char *buf)
2804 {
2805         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2806
2807         return sprintf(buf, "%d\n", target->tl_retry_count);
2808 }
2809
2810 static ssize_t show_cmd_sg_entries(struct device *dev,
2811                                    struct device_attribute *attr, char *buf)
2812 {
2813         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2814
2815         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2816 }
2817
2818 static ssize_t show_allow_ext_sg(struct device *dev,
2819                                  struct device_attribute *attr, char *buf)
2820 {
2821         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2822
2823         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2824 }
2825
2826 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2827 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2828 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2829 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2830 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2831 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2832 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2833 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2834 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2835 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2836 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2837 static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
2838 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2839 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2840 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2841 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2842
2843 static struct device_attribute *srp_host_attrs[] = {
2844         &dev_attr_id_ext,
2845         &dev_attr_ioc_guid,
2846         &dev_attr_service_id,
2847         &dev_attr_pkey,
2848         &dev_attr_sgid,
2849         &dev_attr_dgid,
2850         &dev_attr_orig_dgid,
2851         &dev_attr_req_lim,
2852         &dev_attr_zero_req_lim,
2853         &dev_attr_local_ib_port,
2854         &dev_attr_local_ib_device,
2855         &dev_attr_ch_count,
2856         &dev_attr_comp_vector,
2857         &dev_attr_tl_retry_count,
2858         &dev_attr_cmd_sg_entries,
2859         &dev_attr_allow_ext_sg,
2860         NULL
2861 };
2862
2863 static struct scsi_host_template srp_template = {
2864         .module                         = THIS_MODULE,
2865         .name                           = "InfiniBand SRP initiator",
2866         .proc_name                      = DRV_NAME,
2867         .slave_alloc                    = srp_slave_alloc,
2868         .slave_configure                = srp_slave_configure,
2869         .info                           = srp_target_info,
2870         .queuecommand                   = srp_queuecommand,
2871         .change_queue_depth             = srp_change_queue_depth,
2872         .eh_abort_handler               = srp_abort,
2873         .eh_device_reset_handler        = srp_reset_device,
2874         .eh_host_reset_handler          = srp_reset_host,
2875         .skip_settle_delay              = true,
2876         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2877         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2878         .this_id                        = -1,
2879         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2880         .use_clustering                 = ENABLE_CLUSTERING,
2881         .shost_attrs                    = srp_host_attrs,
2882         .track_queue_depth              = 1,
2883 };
2884
2885 static int srp_sdev_count(struct Scsi_Host *host)
2886 {
2887         struct scsi_device *sdev;
2888         int c = 0;
2889
2890         shost_for_each_device(sdev, host)
2891                 c++;
2892
2893         return c;
2894 }
2895
2896 /*
2897  * Return values:
2898  * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2899  * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2900  *    removal has been scheduled.
2901  * 0 and target->state != SRP_TARGET_REMOVED upon success.
2902  */
2903 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2904 {
2905         struct srp_rport_identifiers ids;
2906         struct srp_rport *rport;
2907
2908         target->state = SRP_TARGET_SCANNING;
2909         sprintf(target->target_name, "SRP.T10:%016llX",
2910                 be64_to_cpu(target->id_ext));
2911
2912         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2913                 return -ENODEV;
2914
2915         memcpy(ids.port_id, &target->id_ext, 8);
2916         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2917         ids.roles = SRP_RPORT_ROLE_TARGET;
2918         rport = srp_rport_add(target->scsi_host, &ids);
2919         if (IS_ERR(rport)) {
2920                 scsi_remove_host(target->scsi_host);
2921                 return PTR_ERR(rport);
2922         }
2923
2924         rport->lld_data = target;
2925         target->rport = rport;
2926
2927         spin_lock(&host->target_lock);
2928         list_add_tail(&target->list, &host->target_list);
2929         spin_unlock(&host->target_lock);
2930
2931         scsi_scan_target(&target->scsi_host->shost_gendev,
2932                          0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
2933
2934         if (srp_connected_ch(target) < target->ch_count ||
2935             target->qp_in_error) {
2936                 shost_printk(KERN_INFO, target->scsi_host,
2937                              PFX "SCSI scan failed - removing SCSI host\n");
2938                 srp_queue_remove_work(target);
2939                 goto out;
2940         }
2941
2942         pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
2943                  dev_name(&target->scsi_host->shost_gendev),
2944                  srp_sdev_count(target->scsi_host));
2945
2946         spin_lock_irq(&target->lock);
2947         if (target->state == SRP_TARGET_SCANNING)
2948                 target->state = SRP_TARGET_LIVE;
2949         spin_unlock_irq(&target->lock);
2950
2951 out:
2952         return 0;
2953 }
2954
2955 static void srp_release_dev(struct device *dev)
2956 {
2957         struct srp_host *host =
2958                 container_of(dev, struct srp_host, dev);
2959
2960         complete(&host->released);
2961 }
2962
2963 static struct class srp_class = {
2964         .name    = "infiniband_srp",
2965         .dev_release = srp_release_dev
2966 };
2967
2968 /**
2969  * srp_conn_unique() - check whether the connection to a target is unique
2970  * @host:   SRP host.
2971  * @target: SRP target port.
2972  */
2973 static bool srp_conn_unique(struct srp_host *host,
2974                             struct srp_target_port *target)
2975 {
2976         struct srp_target_port *t;
2977         bool ret = false;
2978
2979         if (target->state == SRP_TARGET_REMOVED)
2980                 goto out;
2981
2982         ret = true;
2983
2984         spin_lock(&host->target_lock);
2985         list_for_each_entry(t, &host->target_list, list) {
2986                 if (t != target &&
2987                     target->id_ext == t->id_ext &&
2988                     target->ioc_guid == t->ioc_guid &&
2989                     target->initiator_ext == t->initiator_ext) {
2990                         ret = false;
2991                         break;
2992                 }
2993         }
2994         spin_unlock(&host->target_lock);
2995
2996 out:
2997         return ret;
2998 }
2999
3000 /*
3001  * Target ports are added by writing
3002  *
3003  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
3004  *     pkey=<P_Key>,service_id=<service ID>
3005  *
3006  * to the add_target sysfs attribute.
3007  */
3008 enum {
3009         SRP_OPT_ERR             = 0,
3010         SRP_OPT_ID_EXT          = 1 << 0,
3011         SRP_OPT_IOC_GUID        = 1 << 1,
3012         SRP_OPT_DGID            = 1 << 2,
3013         SRP_OPT_PKEY            = 1 << 3,
3014         SRP_OPT_SERVICE_ID      = 1 << 4,
3015         SRP_OPT_MAX_SECT        = 1 << 5,
3016         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
3017         SRP_OPT_IO_CLASS        = 1 << 7,
3018         SRP_OPT_INITIATOR_EXT   = 1 << 8,
3019         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
3020         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
3021         SRP_OPT_SG_TABLESIZE    = 1 << 11,
3022         SRP_OPT_COMP_VECTOR     = 1 << 12,
3023         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
3024         SRP_OPT_QUEUE_SIZE      = 1 << 14,
3025         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
3026                                    SRP_OPT_IOC_GUID     |
3027                                    SRP_OPT_DGID         |
3028                                    SRP_OPT_PKEY         |
3029                                    SRP_OPT_SERVICE_ID),
3030 };
3031
3032 static const match_table_t srp_opt_tokens = {
3033         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
3034         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
3035         { SRP_OPT_DGID,                 "dgid=%s"               },
3036         { SRP_OPT_PKEY,                 "pkey=%x"               },
3037         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
3038         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
3039         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
3040         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
3041         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
3042         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
3043         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
3044         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
3045         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
3046         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
3047         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
3048         { SRP_OPT_ERR,                  NULL                    }
3049 };
3050
3051 static int srp_parse_options(const char *buf, struct srp_target_port *target)
3052 {
3053         char *options, *sep_opt;
3054         char *p;
3055         char dgid[3];
3056         substring_t args[MAX_OPT_ARGS];
3057         int opt_mask = 0;
3058         int token;
3059         int ret = -EINVAL;
3060         int i;
3061
3062         options = kstrdup(buf, GFP_KERNEL);
3063         if (!options)
3064                 return -ENOMEM;
3065
3066         sep_opt = options;
3067         while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3068                 if (!*p)
3069                         continue;
3070
3071                 token = match_token(p, srp_opt_tokens, args);
3072                 opt_mask |= token;
3073
3074                 switch (token) {
3075                 case SRP_OPT_ID_EXT:
3076                         p = match_strdup(args);
3077                         if (!p) {
3078                                 ret = -ENOMEM;
3079                                 goto out;
3080                         }
3081                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3082                         kfree(p);
3083                         break;
3084
3085                 case SRP_OPT_IOC_GUID:
3086                         p = match_strdup(args);
3087                         if (!p) {
3088                                 ret = -ENOMEM;
3089                                 goto out;
3090                         }
3091                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
3092                         kfree(p);
3093                         break;
3094
3095                 case SRP_OPT_DGID:
3096                         p = match_strdup(args);
3097                         if (!p) {
3098                                 ret = -ENOMEM;
3099                                 goto out;
3100                         }
3101                         if (strlen(p) != 32) {
3102                                 pr_warn("bad dest GID parameter '%s'\n", p);
3103                                 kfree(p);
3104                                 goto out;
3105                         }
3106
3107                         for (i = 0; i < 16; ++i) {
3108                                 strlcpy(dgid, p + i * 2, sizeof(dgid));
3109                                 if (sscanf(dgid, "%hhx",
3110                                            &target->orig_dgid.raw[i]) < 1) {
3111                                         ret = -EINVAL;
3112                                         kfree(p);
3113                                         goto out;
3114                                 }
3115                         }
3116                         kfree(p);
3117                         break;
3118
3119                 case SRP_OPT_PKEY:
3120                         if (match_hex(args, &token)) {
3121                                 pr_warn("bad P_Key parameter '%s'\n", p);
3122                                 goto out;
3123                         }
3124                         target->pkey = cpu_to_be16(token);
3125                         break;
3126
3127                 case SRP_OPT_SERVICE_ID:
3128                         p = match_strdup(args);
3129                         if (!p) {
3130                                 ret = -ENOMEM;
3131                                 goto out;
3132                         }
3133                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3134                         kfree(p);
3135                         break;
3136
3137                 case SRP_OPT_MAX_SECT:
3138                         if (match_int(args, &token)) {
3139                                 pr_warn("bad max sect parameter '%s'\n", p);
3140                                 goto out;
3141                         }
3142                         target->scsi_host->max_sectors = token;
3143                         break;
3144
3145                 case SRP_OPT_QUEUE_SIZE:
3146                         if (match_int(args, &token) || token < 1) {
3147                                 pr_warn("bad queue_size parameter '%s'\n", p);
3148                                 goto out;
3149                         }
3150                         target->scsi_host->can_queue = token;
3151                         target->queue_size = token + SRP_RSP_SQ_SIZE +
3152                                              SRP_TSK_MGMT_SQ_SIZE;
3153                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3154                                 target->scsi_host->cmd_per_lun = token;
3155                         break;
3156
3157                 case SRP_OPT_MAX_CMD_PER_LUN:
3158                         if (match_int(args, &token) || token < 1) {
3159                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3160                                         p);
3161                                 goto out;
3162                         }
3163                         target->scsi_host->cmd_per_lun = token;
3164                         break;
3165
3166                 case SRP_OPT_IO_CLASS:
3167                         if (match_hex(args, &token)) {
3168                                 pr_warn("bad IO class parameter '%s'\n", p);
3169                                 goto out;
3170                         }
3171                         if (token != SRP_REV10_IB_IO_CLASS &&
3172                             token != SRP_REV16A_IB_IO_CLASS) {
3173                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3174                                         token, SRP_REV10_IB_IO_CLASS,
3175                                         SRP_REV16A_IB_IO_CLASS);
3176                                 goto out;
3177                         }
3178                         target->io_class = token;
3179                         break;
3180
3181                 case SRP_OPT_INITIATOR_EXT:
3182                         p = match_strdup(args);
3183                         if (!p) {
3184                                 ret = -ENOMEM;
3185                                 goto out;
3186                         }
3187                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3188                         kfree(p);
3189                         break;
3190
3191                 case SRP_OPT_CMD_SG_ENTRIES:
3192                         if (match_int(args, &token) || token < 1 || token > 255) {
3193                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3194                                         p);
3195                                 goto out;
3196                         }
3197                         target->cmd_sg_cnt = token;
3198                         break;
3199
3200                 case SRP_OPT_ALLOW_EXT_SG:
3201                         if (match_int(args, &token)) {
3202                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3203                                 goto out;
3204                         }
3205                         target->allow_ext_sg = !!token;
3206                         break;
3207
3208                 case SRP_OPT_SG_TABLESIZE:
3209                         if (match_int(args, &token) || token < 1 ||
3210                                         token > SG_MAX_SEGMENTS) {
3211                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
3212                                         p);
3213                                 goto out;
3214                         }
3215                         target->sg_tablesize = token;
3216                         break;
3217
3218                 case SRP_OPT_COMP_VECTOR:
3219                         if (match_int(args, &token) || token < 0) {
3220                                 pr_warn("bad comp_vector parameter '%s'\n", p);
3221                                 goto out;
3222                         }
3223                         target->comp_vector = token;
3224                         break;
3225
3226                 case SRP_OPT_TL_RETRY_COUNT:
3227                         if (match_int(args, &token) || token < 2 || token > 7) {
3228                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3229                                         p);
3230                                 goto out;
3231                         }
3232                         target->tl_retry_count = token;
3233                         break;
3234
3235                 default:
3236                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3237                                 p);
3238                         goto out;
3239                 }
3240         }
3241
3242         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3243                 ret = 0;
3244         else
3245                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3246                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3247                             !(srp_opt_tokens[i].token & opt_mask))
3248                                 pr_warn("target creation request is missing parameter '%s'\n",
3249                                         srp_opt_tokens[i].pattern);
3250
3251         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3252             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3253                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3254                         target->scsi_host->cmd_per_lun,
3255                         target->scsi_host->can_queue);
3256
3257 out:
3258         kfree(options);
3259         return ret;
3260 }
3261
3262 static ssize_t srp_create_target(struct device *dev,
3263                                  struct device_attribute *attr,
3264                                  const char *buf, size_t count)
3265 {
3266         struct srp_host *host =
3267                 container_of(dev, struct srp_host, dev);
3268         struct Scsi_Host *target_host;
3269         struct srp_target_port *target;
3270         struct srp_rdma_ch *ch;
3271         struct srp_device *srp_dev = host->srp_dev;
3272         struct ib_device *ibdev = srp_dev->dev;
3273         int ret, node_idx, node, cpu, i;
3274         unsigned int max_sectors_per_mr, mr_per_cmd = 0;
3275         bool multich = false;
3276
3277         target_host = scsi_host_alloc(&srp_template,
3278                                       sizeof (struct srp_target_port));
3279         if (!target_host)
3280                 return -ENOMEM;
3281
3282         target_host->transportt  = ib_srp_transport_template;
3283         target_host->max_channel = 0;
3284         target_host->max_id      = 1;
3285         target_host->max_lun     = -1LL;
3286         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3287
3288         target = host_to_target(target_host);
3289
3290         target->io_class        = SRP_REV16A_IB_IO_CLASS;
3291         target->scsi_host       = target_host;
3292         target->srp_host        = host;
3293         target->pd              = host->srp_dev->pd;
3294         target->lkey            = host->srp_dev->pd->local_dma_lkey;
3295         target->cmd_sg_cnt      = cmd_sg_entries;
3296         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
3297         target->allow_ext_sg    = allow_ext_sg;
3298         target->tl_retry_count  = 7;
3299         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
3300
3301         /*
3302          * Avoid that the SCSI host can be removed by srp_remove_target()
3303          * before this function returns.
3304          */
3305         scsi_host_get(target->scsi_host);
3306
3307         ret = mutex_lock_interruptible(&host->add_target_mutex);
3308         if (ret < 0)
3309                 goto put;
3310
3311         ret = srp_parse_options(buf, target);
3312         if (ret)
3313                 goto out;
3314
3315         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3316
3317         if (!srp_conn_unique(target->srp_host, target)) {
3318                 shost_printk(KERN_INFO, target->scsi_host,
3319                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3320                              be64_to_cpu(target->id_ext),
3321                              be64_to_cpu(target->ioc_guid),
3322                              be64_to_cpu(target->initiator_ext));
3323                 ret = -EEXIST;
3324                 goto out;
3325         }
3326
3327         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3328             target->cmd_sg_cnt < target->sg_tablesize) {
3329                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3330                 target->sg_tablesize = target->cmd_sg_cnt;
3331         }
3332
3333         if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
3334                 /*
3335                  * FR and FMR can only map one HCA page per entry. If the
3336                  * start address is not aligned on a HCA page boundary two
3337                  * entries will be used for the head and the tail although
3338                  * these two entries combined contain at most one HCA page of
3339                  * data. Hence the "+ 1" in the calculation below.
3340                  *
3341                  * The indirect data buffer descriptor is contiguous so the
3342                  * memory for that buffer will only be registered if
3343                  * register_always is true. Hence add one to mr_per_cmd if
3344                  * register_always has been set.
3345                  */
3346                 max_sectors_per_mr = srp_dev->max_pages_per_mr <<
3347                                   (ilog2(srp_dev->mr_page_size) - 9);
3348                 mr_per_cmd = register_always +
3349                         (target->scsi_host->max_sectors + 1 +
3350                          max_sectors_per_mr - 1) / max_sectors_per_mr;
3351                 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3352                          target->scsi_host->max_sectors,
3353                          srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3354                          max_sectors_per_mr, mr_per_cmd);
3355         }
3356
3357         target_host->sg_tablesize = target->sg_tablesize;
3358         target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
3359         target->mr_per_cmd = mr_per_cmd;
3360         target->indirect_size = target->sg_tablesize *
3361                                 sizeof (struct srp_direct_buf);
3362         target->max_iu_len = sizeof (struct srp_cmd) +
3363                              sizeof (struct srp_indirect_buf) +
3364                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3365
3366         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3367         INIT_WORK(&target->remove_work, srp_remove_work);
3368         spin_lock_init(&target->lock);
3369         ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3370         if (ret)
3371                 goto out;
3372
3373         ret = -ENOMEM;
3374         target->ch_count = max_t(unsigned, num_online_nodes(),
3375                                  min(ch_count ? :
3376                                      min(4 * num_online_nodes(),
3377                                          ibdev->num_comp_vectors),
3378                                      num_online_cpus()));
3379         target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3380                              GFP_KERNEL);
3381         if (!target->ch)
3382                 goto out;
3383
3384         node_idx = 0;
3385         for_each_online_node(node) {
3386                 const int ch_start = (node_idx * target->ch_count /
3387                                       num_online_nodes());
3388                 const int ch_end = ((node_idx + 1) * target->ch_count /
3389                                     num_online_nodes());
3390                 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3391                                       num_online_nodes() + target->comp_vector)
3392                                      % ibdev->num_comp_vectors;
3393                 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3394                                     num_online_nodes() + target->comp_vector)
3395                                    % ibdev->num_comp_vectors;
3396                 int cpu_idx = 0;
3397
3398                 for_each_online_cpu(cpu) {
3399                         if (cpu_to_node(cpu) != node)
3400                                 continue;
3401                         if (ch_start + cpu_idx >= ch_end)
3402                                 continue;
3403                         ch = &target->ch[ch_start + cpu_idx];
3404                         ch->target = target;
3405                         ch->comp_vector = cv_start == cv_end ? cv_start :
3406                                 cv_start + cpu_idx % (cv_end - cv_start);
3407                         spin_lock_init(&ch->lock);
3408                         INIT_LIST_HEAD(&ch->free_tx);
3409                         ret = srp_new_cm_id(ch);
3410                         if (ret)
3411                                 goto err_disconnect;
3412
3413                         ret = srp_create_ch_ib(ch);
3414                         if (ret)
3415                                 goto err_disconnect;
3416
3417                         ret = srp_alloc_req_data(ch);
3418                         if (ret)
3419                                 goto err_disconnect;
3420
3421                         ret = srp_connect_ch(ch, multich);
3422                         if (ret) {
3423                                 shost_printk(KERN_ERR, target->scsi_host,
3424                                              PFX "Connection %d/%d failed\n",
3425                                              ch_start + cpu_idx,
3426                                              target->ch_count);
3427                                 if (node_idx == 0 && cpu_idx == 0) {
3428                                         goto err_disconnect;
3429                                 } else {
3430                                         srp_free_ch_ib(target, ch);
3431                                         srp_free_req_data(target, ch);
3432                                         target->ch_count = ch - target->ch;
3433                                         goto connected;
3434                                 }
3435                         }
3436
3437                         multich = true;
3438                         cpu_idx++;
3439                 }
3440                 node_idx++;
3441         }
3442
3443 connected:
3444         target->scsi_host->nr_hw_queues = target->ch_count;
3445
3446         ret = srp_add_target(host, target);
3447         if (ret)
3448                 goto err_disconnect;
3449
3450         if (target->state != SRP_TARGET_REMOVED) {
3451                 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3452                              "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3453                              be64_to_cpu(target->id_ext),
3454                              be64_to_cpu(target->ioc_guid),
3455                              be16_to_cpu(target->pkey),
3456                              be64_to_cpu(target->service_id),
3457                              target->sgid.raw, target->orig_dgid.raw);
3458         }
3459
3460         ret = count;
3461
3462 out:
3463         mutex_unlock(&host->add_target_mutex);
3464
3465 put:
3466         scsi_host_put(target->scsi_host);
3467         if (ret < 0)
3468                 scsi_host_put(target->scsi_host);
3469
3470         return ret;
3471
3472 err_disconnect:
3473         srp_disconnect_target(target);
3474
3475         for (i = 0; i < target->ch_count; i++) {
3476                 ch = &target->ch[i];
3477                 srp_free_ch_ib(target, ch);
3478                 srp_free_req_data(target, ch);
3479         }
3480
3481         kfree(target->ch);
3482         goto out;
3483 }
3484
3485 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3486
3487 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3488                           char *buf)
3489 {
3490         struct srp_host *host = container_of(dev, struct srp_host, dev);
3491
3492         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3493 }
3494
3495 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3496
3497 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3498                          char *buf)
3499 {
3500         struct srp_host *host = container_of(dev, struct srp_host, dev);
3501
3502         return sprintf(buf, "%d\n", host->port);
3503 }
3504
3505 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3506
3507 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3508 {
3509         struct srp_host *host;
3510
3511         host = kzalloc(sizeof *host, GFP_KERNEL);
3512         if (!host)
3513                 return NULL;
3514
3515         INIT_LIST_HEAD(&host->target_list);
3516         spin_lock_init(&host->target_lock);
3517         init_completion(&host->released);
3518         mutex_init(&host->add_target_mutex);
3519         host->srp_dev = device;
3520         host->port = port;
3521
3522         host->dev.class = &srp_class;
3523         host->dev.parent = device->dev->dma_device;
3524         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3525
3526         if (device_register(&host->dev))
3527                 goto free_host;
3528         if (device_create_file(&host->dev, &dev_attr_add_target))
3529                 goto err_class;
3530         if (device_create_file(&host->dev, &dev_attr_ibdev))
3531                 goto err_class;
3532         if (device_create_file(&host->dev, &dev_attr_port))
3533                 goto err_class;
3534
3535         return host;
3536
3537 err_class:
3538         device_unregister(&host->dev);
3539
3540 free_host:
3541         kfree(host);
3542
3543         return NULL;
3544 }
3545
3546 static void srp_add_one(struct ib_device *device)
3547 {
3548         struct srp_device *srp_dev;
3549         struct ib_device_attr *attr = &device->attrs;
3550         struct srp_host *host;
3551         int mr_page_shift, p;
3552         u64 max_pages_per_mr;
3553         unsigned int flags = 0;
3554
3555         srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
3556         if (!srp_dev)
3557                 return;
3558
3559         /*
3560          * Use the smallest page size supported by the HCA, down to a
3561          * minimum of 4096 bytes. We're unlikely to build large sglists
3562          * out of smaller entries.
3563          */
3564         mr_page_shift           = max(12, ffs(attr->page_size_cap) - 1);
3565         srp_dev->mr_page_size   = 1 << mr_page_shift;
3566         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
3567         max_pages_per_mr        = attr->max_mr_size;
3568         do_div(max_pages_per_mr, srp_dev->mr_page_size);
3569         pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
3570                  attr->max_mr_size, srp_dev->mr_page_size,
3571                  max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
3572         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3573                                           max_pages_per_mr);
3574
3575         srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3576                             device->map_phys_fmr && device->unmap_fmr);
3577         srp_dev->has_fr = (attr->device_cap_flags &
3578                            IB_DEVICE_MEM_MGT_EXTENSIONS);
3579         if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
3580                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3581         } else if (!never_register &&
3582                    attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
3583                 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3584                                          (!srp_dev->has_fmr || prefer_fr));
3585                 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3586         }
3587
3588         if (never_register || !register_always ||
3589             (!srp_dev->has_fmr && !srp_dev->has_fr))
3590                 flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
3591
3592         if (srp_dev->use_fast_reg) {
3593                 srp_dev->max_pages_per_mr =
3594                         min_t(u32, srp_dev->max_pages_per_mr,
3595                               attr->max_fast_reg_page_list_len);
3596         }
3597         srp_dev->mr_max_size    = srp_dev->mr_page_size *
3598                                    srp_dev->max_pages_per_mr;
3599         pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3600                  device->name, mr_page_shift, attr->max_mr_size,
3601                  attr->max_fast_reg_page_list_len,
3602                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3603
3604         INIT_LIST_HEAD(&srp_dev->dev_list);
3605
3606         srp_dev->dev = device;
3607         srp_dev->pd  = ib_alloc_pd(device, flags);
3608         if (IS_ERR(srp_dev->pd))
3609                 goto free_dev;
3610
3611
3612         for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3613                 host = srp_add_port(srp_dev, p);
3614                 if (host)
3615                         list_add_tail(&host->list, &srp_dev->dev_list);
3616         }
3617
3618         ib_set_client_data(device, &srp_client, srp_dev);
3619         return;
3620
3621 free_dev:
3622         kfree(srp_dev);
3623 }
3624
3625 static void srp_remove_one(struct ib_device *device, void *client_data)
3626 {
3627         struct srp_device *srp_dev;
3628         struct srp_host *host, *tmp_host;
3629         struct srp_target_port *target;
3630
3631         srp_dev = client_data;
3632         if (!srp_dev)
3633                 return;
3634
3635         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3636                 device_unregister(&host->dev);
3637                 /*
3638                  * Wait for the sysfs entry to go away, so that no new
3639                  * target ports can be created.
3640                  */
3641                 wait_for_completion(&host->released);
3642
3643                 /*
3644                  * Remove all target ports.
3645                  */
3646                 spin_lock(&host->target_lock);
3647                 list_for_each_entry(target, &host->target_list, list)
3648                         srp_queue_remove_work(target);
3649                 spin_unlock(&host->target_lock);
3650
3651                 /*
3652                  * Wait for tl_err and target port removal tasks.
3653                  */
3654                 flush_workqueue(system_long_wq);
3655                 flush_workqueue(srp_remove_wq);
3656
3657                 kfree(host);
3658         }
3659
3660         ib_dealloc_pd(srp_dev->pd);
3661
3662         kfree(srp_dev);
3663 }
3664
3665 static struct srp_function_template ib_srp_transport_functions = {
3666         .has_rport_state         = true,
3667         .reset_timer_if_blocked  = true,
3668         .reconnect_delay         = &srp_reconnect_delay,
3669         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
3670         .dev_loss_tmo            = &srp_dev_loss_tmo,
3671         .reconnect               = srp_rport_reconnect,
3672         .rport_delete            = srp_rport_delete,
3673         .terminate_rport_io      = srp_terminate_io,
3674 };
3675
3676 static int __init srp_init_module(void)
3677 {
3678         int ret;
3679
3680         if (srp_sg_tablesize) {
3681                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3682                 if (!cmd_sg_entries)
3683                         cmd_sg_entries = srp_sg_tablesize;
3684         }
3685
3686         if (!cmd_sg_entries)
3687                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3688
3689         if (cmd_sg_entries > 255) {
3690                 pr_warn("Clamping cmd_sg_entries to 255\n");
3691                 cmd_sg_entries = 255;
3692         }
3693
3694         if (!indirect_sg_entries)
3695                 indirect_sg_entries = cmd_sg_entries;
3696         else if (indirect_sg_entries < cmd_sg_entries) {
3697                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3698                         cmd_sg_entries);
3699                 indirect_sg_entries = cmd_sg_entries;
3700         }
3701
3702         if (indirect_sg_entries > SG_MAX_SEGMENTS) {
3703                 pr_warn("Clamping indirect_sg_entries to %u\n",
3704                         SG_MAX_SEGMENTS);
3705                 indirect_sg_entries = SG_MAX_SEGMENTS;
3706         }
3707
3708         srp_remove_wq = create_workqueue("srp_remove");
3709         if (!srp_remove_wq) {
3710                 ret = -ENOMEM;
3711                 goto out;
3712         }
3713
3714         ret = -ENOMEM;
3715         ib_srp_transport_template =
3716                 srp_attach_transport(&ib_srp_transport_functions);
3717         if (!ib_srp_transport_template)
3718                 goto destroy_wq;
3719
3720         ret = class_register(&srp_class);
3721         if (ret) {
3722                 pr_err("couldn't register class infiniband_srp\n");
3723                 goto release_tr;
3724         }
3725
3726         ib_sa_register_client(&srp_sa_client);
3727
3728         ret = ib_register_client(&srp_client);
3729         if (ret) {
3730                 pr_err("couldn't register IB client\n");
3731                 goto unreg_sa;
3732         }
3733
3734 out:
3735         return ret;
3736
3737 unreg_sa:
3738         ib_sa_unregister_client(&srp_sa_client);
3739         class_unregister(&srp_class);
3740
3741 release_tr:
3742         srp_release_transport(ib_srp_transport_template);
3743
3744 destroy_wq:
3745         destroy_workqueue(srp_remove_wq);
3746         goto out;
3747 }
3748
3749 static void __exit srp_cleanup_module(void)
3750 {
3751         ib_unregister_client(&srp_client);
3752         ib_sa_unregister_client(&srp_sa_client);
3753         class_unregister(&srp_class);
3754         srp_release_transport(ib_srp_transport_template);
3755         destroy_workqueue(srp_remove_wq);
3756 }
3757
3758 module_init(srp_init_module);
3759 module_exit(srp_cleanup_module);