drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43 #include <linux/lockdep.h>
  44 #include <rdma/ib_cache.h>
  45
  46 #include <linux/atomic.h>
  47
  48 #include <scsi/scsi.h>
  49 #include <scsi/scsi_device.h>
  50 #include <scsi/scsi_dbg.h>
  51 #include <scsi/scsi_tcq.h>
  52 #include <scsi/srp.h>
  53 #include <scsi/scsi_transport_srp.h>
  54
  55 #include "ib_srp.h"
  56
  57 #define DRV_NAME        "ib_srp"
  58 #define PFX             DRV_NAME ": "
  59 #define DRV_VERSION     "2.0"
  60 #define DRV_RELDATE     "July 26, 2015"
  61
  62 MODULE_AUTHOR("Roland Dreier");
  63 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
  64 MODULE_LICENSE("Dual BSD/GPL");
  65 MODULE_VERSION(DRV_VERSION);
  66 MODULE_INFO(release_date, DRV_RELDATE);
  67
  68 #if !defined(CONFIG_DYNAMIC_DEBUG)
  69 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
  70 #define DYNAMIC_DEBUG_BRANCH(descriptor) false
  71 #endif
  72
  73 static unsigned int srp_sg_tablesize;
  74 static unsigned int cmd_sg_entries;
  75 static unsigned int indirect_sg_entries;
  76 static bool allow_ext_sg;
  77 static bool prefer_fr = true;
  78 static bool register_always = true;
  79 static bool never_register;
  80 static int topspin_workarounds = 1;
  81
  82 module_param(srp_sg_tablesize, uint, 0444);
  83 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  84
  85 module_param(cmd_sg_entries, uint, 0444);
  86 MODULE_PARM_DESC(cmd_sg_entries,
  87                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  88
  89 module_param(indirect_sg_entries, uint, 0444);
  90 MODULE_PARM_DESC(indirect_sg_entries,
  91                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
  92
  93 module_param(allow_ext_sg, bool, 0444);
  94 MODULE_PARM_DESC(allow_ext_sg,
  95                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  96
  97 module_param(topspin_workarounds, int, 0444);
  98 MODULE_PARM_DESC(topspin_workarounds,
  99                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
 100
 101 module_param(prefer_fr, bool, 0444);
 102 MODULE_PARM_DESC(prefer_fr,
 103 "Whether to use fast registration if both FMR and fast registration are supported");
 104
 105 module_param(register_always, bool, 0444);
 106 MODULE_PARM_DESC(register_always,
 107                  "Use memory registration even for contiguous memory regions");
 108
 109 module_param(never_register, bool, 0444);
 110 MODULE_PARM_DESC(never_register, "Never register memory");
 111
 112 static const struct kernel_param_ops srp_tmo_ops;
 113
 114 static int srp_reconnect_delay = 10;
 115 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 116                 S_IRUGO | S_IWUSR);
 117 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 118
 119 static int srp_fast_io_fail_tmo = 15;
 120 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 121                 S_IRUGO | S_IWUSR);
 122 MODULE_PARM_DESC(fast_io_fail_tmo,
 123                  "Number of seconds between the observation of a transport"
 124                  " layer error and failing all I/O. \"off\" means that this"
 125                  " functionality is disabled.");
 126
 127 static int srp_dev_loss_tmo = 600;
 128 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 129                 S_IRUGO | S_IWUSR);
 130 MODULE_PARM_DESC(dev_loss_tmo,
 131                  "Maximum number of seconds that the SRP transport should"
 132                  " insulate transport layer errors. After this time has been"
 133                  " exceeded the SCSI host is removed. Should be"
 134                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 135                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 136                  " this functionality is disabled.");
 137
 138 static unsigned ch_count;
 139 module_param(ch_count, uint, 0444);
 140 MODULE_PARM_DESC(ch_count,
 141                  "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
 142
 143 static void srp_add_one(struct ib_device *device);
 144 static void srp_remove_one(struct ib_device *device, void *client_data);
 145 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 146 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
 147                 const char *opname);
 148 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 149
 150 static struct scsi_transport_template *ib_srp_transport_template;
 151 static struct workqueue_struct *srp_remove_wq;
 152
 153 static struct ib_client srp_client = {
 154         .name   = "srp",
 155         .add    = srp_add_one,
 156         .remove = srp_remove_one
 157 };
 158
 159 static struct ib_sa_client srp_sa_client;
 160
 161 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 162 {
 163         int tmo = *(int *)kp->arg;
 164
 165         if (tmo >= 0)
 166                 return sprintf(buffer, "%d", tmo);
 167         else
 168                 return sprintf(buffer, "off");
 169 }
 170
 171 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 172 {
 173         int tmo, res;
 174
 175         res = srp_parse_tmo(&tmo, val);
 176         if (res)
 177                 goto out;
 178
 179         if (kp->arg == &srp_reconnect_delay)
 180                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 181                                     srp_dev_loss_tmo);
 182         else if (kp->arg == &srp_fast_io_fail_tmo)
 183                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 184         else
 185                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 186                                     tmo);
 187         if (res)
 188                 goto out;
 189         *(int *)kp->arg = tmo;
 190
 191 out:
 192         return res;
 193 }
 194
 195 static const struct kernel_param_ops srp_tmo_ops = {
 196         .get = srp_tmo_get,
 197         .set = srp_tmo_set,
 198 };
 199
 200 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 201 {
 202         return (struct srp_target_port *) host->hostdata;
 203 }
 204
 205 static const char *srp_target_info(struct Scsi_Host *host)
 206 {
 207         return host_to_target(host)->target_name;
 208 }
 209
 210 static int srp_target_is_topspin(struct srp_target_port *target)
 211 {
 212         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 213         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 214
 215         return topspin_workarounds &&
 216                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 217                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 218 }
 219
 220 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 221                                    gfp_t gfp_mask,
 222                                    enum dma_data_direction direction)
 223 {
 224         struct srp_iu *iu;
 225
 226         iu = kmalloc(sizeof *iu, gfp_mask);
 227         if (!iu)
 228                 goto out;
 229
 230         iu->buf = kzalloc(size, gfp_mask);
 231         if (!iu->buf)
 232                 goto out_free_iu;
 233
 234         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 235                                     direction);
 236         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 237                 goto out_free_buf;
 238
 239         iu->size      = size;
 240         iu->direction = direction;
 241
 242         return iu;
 243
 244 out_free_buf:
 245         kfree(iu->buf);
 246 out_free_iu:
 247         kfree(iu);
 248 out:
 249         return NULL;
 250 }
 251
 252 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 253 {
 254         if (!iu)
 255                 return;
 256
 257         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 258                             iu->direction);
 259         kfree(iu->buf);
 260         kfree(iu);
 261 }
 262
 263 static void srp_qp_event(struct ib_event *event, void *context)
 264 {
 265         pr_debug("QP event %s (%d)\n",
 266                  ib_event_msg(event->event), event->event);
 267 }
 268
 269 static int srp_init_qp(struct srp_target_port *target,
 270                        struct ib_qp *qp)
 271 {
 272         struct ib_qp_attr *attr;
 273         int ret;
 274
 275         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 276         if (!attr)
 277                 return -ENOMEM;
 278
 279         ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
 280                                   target->srp_host->port,
 281                                   be16_to_cpu(target->pkey),
 282                                   &attr->pkey_index);
 283         if (ret)
 284                 goto out;
 285
 286         attr->qp_state        = IB_QPS_INIT;
 287         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 288                                     IB_ACCESS_REMOTE_WRITE);
 289         attr->port_num        = target->srp_host->port;
 290
 291         ret = ib_modify_qp(qp, attr,
 292                            IB_QP_STATE          |
 293                            IB_QP_PKEY_INDEX     |
 294                            IB_QP_ACCESS_FLAGS   |
 295                            IB_QP_PORT);
 296
 297 out:
 298         kfree(attr);
 299         return ret;
 300 }
 301
 302 static int srp_new_cm_id(struct srp_rdma_ch *ch)
 303 {
 304         struct srp_target_port *target = ch->target;
 305         struct ib_cm_id *new_cm_id;
 306
 307         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 308                                     srp_cm_handler, ch);
 309         if (IS_ERR(new_cm_id))
 310                 return PTR_ERR(new_cm_id);
 311
 312         if (ch->cm_id)
 313                 ib_destroy_cm_id(ch->cm_id);
 314         ch->cm_id = new_cm_id;
 315         ch->path.rec_type = SA_PATH_REC_TYPE_IB;
 316         ch->path.sgid = target->sgid;
 317         ch->path.dgid = target->orig_dgid;
 318         ch->path.pkey = target->pkey;
 319         sa_path_set_service_id(&ch->path, target->service_id);
 320
 321         return 0;
 322 }
 323
 324 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 325 {
 326         struct srp_device *dev = target->srp_host->srp_dev;
 327         struct ib_fmr_pool_param fmr_param;
 328
 329         memset(&fmr_param, 0, sizeof(fmr_param));
 330         fmr_param.pool_size         = target->mr_pool_size;
 331         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 332         fmr_param.cache             = 1;
 333         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 334         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 335         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 336                                        IB_ACCESS_REMOTE_WRITE |
 337                                        IB_ACCESS_REMOTE_READ);
 338
 339         return ib_create_fmr_pool(dev->pd, &fmr_param);
 340 }
 341
 342 /**
 343  * srp_destroy_fr_pool() - free the resources owned by a pool
 344  * @pool: Fast registration pool to be destroyed.
 345  */
 346 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 347 {
 348         int i;
 349         struct srp_fr_desc *d;
 350
 351         if (!pool)
 352                 return;
 353
 354         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 355                 if (d->mr)
 356                         ib_dereg_mr(d->mr);
 357         }
 358         kfree(pool);
 359 }
 360
 361 /**
 362  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 363  * @device:            IB device to allocate fast registration descriptors for.
 364  * @pd:                Protection domain associated with the FR descriptors.
 365  * @pool_size:         Number of descriptors to allocate.
 366  * @max_page_list_len: Maximum fast registration work request page list length.
 367  */
 368 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 369                                               struct ib_pd *pd, int pool_size,
 370                                               int max_page_list_len)
 371 {
 372         struct srp_fr_pool *pool;
 373         struct srp_fr_desc *d;
 374         struct ib_mr *mr;
 375         int i, ret = -EINVAL;
 376
 377         if (pool_size <= 0)
 378                 goto err;
 379         ret = -ENOMEM;
 380         pool = kzalloc(sizeof(struct srp_fr_pool) +
 381                        pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
 382         if (!pool)
 383                 goto err;
 384         pool->size = pool_size;
 385         pool->max_page_list_len = max_page_list_len;
 386         spin_lock_init(&pool->lock);
 387         INIT_LIST_HEAD(&pool->free_list);
 388
 389         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 390                 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
 391                                  max_page_list_len);
 392                 if (IS_ERR(mr)) {
 393                         ret = PTR_ERR(mr);
 394                         if (ret == -ENOMEM)
 395                                 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
 396                                         dev_name(&device->dev));
 397                         goto destroy_pool;
 398                 }
 399                 d->mr = mr;
 400                 list_add_tail(&d->entry, &pool->free_list);
 401         }
 402
 403 out:
 404         return pool;
 405
 406 destroy_pool:
 407         srp_destroy_fr_pool(pool);
 408
 409 err:
 410         pool = ERR_PTR(ret);
 411         goto out;
 412 }
 413
 414 /**
 415  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 416  * @pool: Pool to obtain descriptor from.
 417  */
 418 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 419 {
 420         struct srp_fr_desc *d = NULL;
 421         unsigned long flags;
 422
 423         spin_lock_irqsave(&pool->lock, flags);
 424         if (!list_empty(&pool->free_list)) {
 425                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 426                 list_del(&d->entry);
 427         }
 428         spin_unlock_irqrestore(&pool->lock, flags);
 429
 430         return d;
 431 }
 432
 433 /**
 434  * srp_fr_pool_put() - put an FR descriptor back in the free list
 435  * @pool: Pool the descriptor was allocated from.
 436  * @desc: Pointer to an array of fast registration descriptor pointers.
 437  * @n:    Number of descriptors to put back.
 438  *
 439  * Note: The caller must already have queued an invalidation request for
 440  * desc->mr->rkey before calling this function.
 441  */
 442 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 443                             int n)
 444 {
 445         unsigned long flags;
 446         int i;
 447
 448         spin_lock_irqsave(&pool->lock, flags);
 449         for (i = 0; i < n; i++)
 450                 list_add(&desc[i]->entry, &pool->free_list);
 451         spin_unlock_irqrestore(&pool->lock, flags);
 452 }
 453
 454 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 455 {
 456         struct srp_device *dev = target->srp_host->srp_dev;
 457
 458         return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
 459                                   dev->max_pages_per_mr);
 460 }
 461
 462 /**
 463  * srp_destroy_qp() - destroy an RDMA queue pair
 464  * @qp: RDMA queue pair.
 465  *
 466  * Drain the qp before destroying it.  This avoids that the receive
 467  * completion handler can access the queue pair while it is
 468  * being destroyed.
 469  */
 470 static void srp_destroy_qp(struct srp_rdma_ch *ch, struct ib_qp *qp)
 471 {
 472         spin_lock_irq(&ch->lock);
 473         ib_process_cq_direct(ch->send_cq, -1);
 474         spin_unlock_irq(&ch->lock);
 475
 476         ib_drain_qp(qp);
 477         ib_destroy_qp(qp);
 478 }
 479
 480 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 481 {
 482         struct srp_target_port *target = ch->target;
 483         struct srp_device *dev = target->srp_host->srp_dev;
 484         struct ib_qp_init_attr *init_attr;
 485         struct ib_cq *recv_cq, *send_cq;
 486         struct ib_qp *qp;
 487         struct ib_fmr_pool *fmr_pool = NULL;
 488         struct srp_fr_pool *fr_pool = NULL;
 489         const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
 490         int ret;
 491
 492         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 493         if (!init_attr)
 494                 return -ENOMEM;
 495
 496         /* queue_size + 1 for ib_drain_rq() */
 497         recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
 498                                 ch->comp_vector, IB_POLL_SOFTIRQ);
 499         if (IS_ERR(recv_cq)) {
 500                 ret = PTR_ERR(recv_cq);
 501                 goto err;
 502         }
 503
 504         send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
 505                                 ch->comp_vector, IB_POLL_DIRECT);
 506         if (IS_ERR(send_cq)) {
 507                 ret = PTR_ERR(send_cq);
 508                 goto err_recv_cq;
 509         }
 510
 511         init_attr->event_handler       = srp_qp_event;
 512         init_attr->cap.max_send_wr     = m * target->queue_size;
 513         init_attr->cap.max_recv_wr     = target->queue_size + 1;
 514         init_attr->cap.max_recv_sge    = 1;
 515         init_attr->cap.max_send_sge    = 1;
 516         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 517         init_attr->qp_type             = IB_QPT_RC;
 518         init_attr->send_cq             = send_cq;
 519         init_attr->recv_cq             = recv_cq;
 520
 521         qp = ib_create_qp(dev->pd, init_attr);
 522         if (IS_ERR(qp)) {
 523                 ret = PTR_ERR(qp);
 524                 goto err_send_cq;
 525         }
 526
 527         ret = srp_init_qp(target, qp);
 528         if (ret)
 529                 goto err_qp;
 530
 531         if (dev->use_fast_reg) {
 532                 fr_pool = srp_alloc_fr_pool(target);
 533                 if (IS_ERR(fr_pool)) {
 534                         ret = PTR_ERR(fr_pool);
 535                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 536                                      "FR pool allocation failed (%d)\n", ret);
 537                         goto err_qp;
 538                 }
 539         } else if (dev->use_fmr) {
 540                 fmr_pool = srp_alloc_fmr_pool(target);
 541                 if (IS_ERR(fmr_pool)) {
 542                         ret = PTR_ERR(fmr_pool);
 543                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 544                                      "FMR pool allocation failed (%d)\n", ret);
 545                         goto err_qp;
 546                 }
 547         }
 548
 549         if (ch->qp)
 550                 srp_destroy_qp(ch, ch->qp);
 551         if (ch->recv_cq)
 552                 ib_free_cq(ch->recv_cq);
 553         if (ch->send_cq)
 554                 ib_free_cq(ch->send_cq);
 555
 556         ch->qp = qp;
 557         ch->recv_cq = recv_cq;
 558         ch->send_cq = send_cq;
 559
 560         if (dev->use_fast_reg) {
 561                 if (ch->fr_pool)
 562                         srp_destroy_fr_pool(ch->fr_pool);
 563                 ch->fr_pool = fr_pool;
 564         } else if (dev->use_fmr) {
 565                 if (ch->fmr_pool)
 566                         ib_destroy_fmr_pool(ch->fmr_pool);
 567                 ch->fmr_pool = fmr_pool;
 568         }
 569
 570         kfree(init_attr);
 571         return 0;
 572
 573 err_qp:
 574         srp_destroy_qp(ch, qp);
 575
 576 err_send_cq:
 577         ib_free_cq(send_cq);
 578
 579 err_recv_cq:
 580         ib_free_cq(recv_cq);
 581
 582 err:
 583         kfree(init_attr);
 584         return ret;
 585 }
 586
 587 /*
 588  * Note: this function may be called without srp_alloc_iu_bufs() having been
 589  * invoked. Hence the ch->[rt]x_ring checks.
 590  */
 591 static void srp_free_ch_ib(struct srp_target_port *target,
 592                            struct srp_rdma_ch *ch)
 593 {
 594         struct srp_device *dev = target->srp_host->srp_dev;
 595         int i;
 596
 597         if (!ch->target)
 598                 return;
 599
 600         if (ch->cm_id) {
 601                 ib_destroy_cm_id(ch->cm_id);
 602                 ch->cm_id = NULL;
 603         }
 604
 605         /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
 606         if (!ch->qp)
 607                 return;
 608
 609         if (dev->use_fast_reg) {
 610                 if (ch->fr_pool)
 611                         srp_destroy_fr_pool(ch->fr_pool);
 612         } else if (dev->use_fmr) {
 613                 if (ch->fmr_pool)
 614                         ib_destroy_fmr_pool(ch->fmr_pool);
 615         }
 616
 617         srp_destroy_qp(ch, ch->qp);
 618         ib_free_cq(ch->send_cq);
 619         ib_free_cq(ch->recv_cq);
 620
 621         /*
 622          * Avoid that the SCSI error handler tries to use this channel after
 623          * it has been freed. The SCSI error handler can namely continue
 624          * trying to perform recovery actions after scsi_remove_host()
 625          * returned.
 626          */
 627         ch->target = NULL;
 628
 629         ch->qp = NULL;
 630         ch->send_cq = ch->recv_cq = NULL;
 631
 632         if (ch->rx_ring) {
 633                 for (i = 0; i < target->queue_size; ++i)
 634                         srp_free_iu(target->srp_host, ch->rx_ring[i]);
 635                 kfree(ch->rx_ring);
 636                 ch->rx_ring = NULL;
 637         }
 638         if (ch->tx_ring) {
 639                 for (i = 0; i < target->queue_size; ++i)
 640                         srp_free_iu(target->srp_host, ch->tx_ring[i]);
 641                 kfree(ch->tx_ring);
 642                 ch->tx_ring = NULL;
 643         }
 644 }
 645
 646 static void srp_path_rec_completion(int status,
 647                                     struct sa_path_rec *pathrec,
 648                                     void *ch_ptr)
 649 {
 650         struct srp_rdma_ch *ch = ch_ptr;
 651         struct srp_target_port *target = ch->target;
 652
 653         ch->status = status;
 654         if (status)
 655                 shost_printk(KERN_ERR, target->scsi_host,
 656                              PFX "Got failed path rec status %d\n", status);
 657         else
 658                 ch->path = *pathrec;
 659         complete(&ch->done);
 660 }
 661
 662 static int srp_lookup_path(struct srp_rdma_ch *ch)
 663 {
 664         struct srp_target_port *target = ch->target;
 665         int ret;
 666
 667         ch->path.numb_path = 1;
 668
 669         init_completion(&ch->done);
 670
 671         ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 672                                                target->srp_host->srp_dev->dev,
 673                                                target->srp_host->port,
 674                                                &ch->path,
 675                                                IB_SA_PATH_REC_SERVICE_ID |
 676                                                IB_SA_PATH_REC_DGID       |
 677                                                IB_SA_PATH_REC_SGID       |
 678                                                IB_SA_PATH_REC_NUMB_PATH  |
 679                                                IB_SA_PATH_REC_PKEY,
 680                                                SRP_PATH_REC_TIMEOUT_MS,
 681                                                GFP_KERNEL,
 682                                                srp_path_rec_completion,
 683                                                ch, &ch->path_query);
 684         if (ch->path_query_id < 0)
 685                 return ch->path_query_id;
 686
 687         ret = wait_for_completion_interruptible(&ch->done);
 688         if (ret < 0)
 689                 return ret;
 690
 691         if (ch->status < 0)
 692                 shost_printk(KERN_WARNING, target->scsi_host,
 693                              PFX "Path record query failed\n");
 694
 695         return ch->status;
 696 }
 697
 698 static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 699 {
 700         struct srp_target_port *target = ch->target;
 701         struct {
 702                 struct ib_cm_req_param param;
 703                 struct srp_login_req   priv;
 704         } *req = NULL;
 705         int status;
 706
 707         req = kzalloc(sizeof *req, GFP_KERNEL);
 708         if (!req)
 709                 return -ENOMEM;
 710
 711         req->param.primary_path               = &ch->path;
 712         req->param.alternate_path             = NULL;
 713         req->param.service_id                 = target->service_id;
 714         req->param.qp_num                     = ch->qp->qp_num;
 715         req->param.qp_type                    = ch->qp->qp_type;
 716         req->param.private_data               = &req->priv;
 717         req->param.private_data_len           = sizeof req->priv;
 718         req->param.flow_control               = 1;
 719
 720         get_random_bytes(&req->param.starting_psn, 4);
 721         req->param.starting_psn              &= 0xffffff;
 722
 723         /*
 724          * Pick some arbitrary defaults here; we could make these
 725          * module parameters if anyone cared about setting them.
 726          */
 727         req->param.responder_resources        = 4;
 728         req->param.remote_cm_response_timeout = 20;
 729         req->param.local_cm_response_timeout  = 20;
 730         req->param.retry_count                = target->tl_retry_count;
 731         req->param.rnr_retry_count            = 7;
 732         req->param.max_cm_retries             = 15;
 733
 734         req->priv.opcode        = SRP_LOGIN_REQ;
 735         req->priv.tag           = 0;
 736         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 737         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 738                                               SRP_BUF_FORMAT_INDIRECT);
 739         req->priv.req_flags     = (multich ? SRP_MULTICHAN_MULTI :
 740                                    SRP_MULTICHAN_SINGLE);
 741         /*
 742          * In the published SRP specification (draft rev. 16a), the
 743          * port identifier format is 8 bytes of ID extension followed
 744          * by 8 bytes of GUID.  Older drafts put the two halves in the
 745          * opposite order, so that the GUID comes first.
 746          *
 747          * Targets conforming to these obsolete drafts can be
 748          * recognized by the I/O Class they report.
 749          */
 750         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 751                 memcpy(req->priv.initiator_port_id,
 752                        &target->sgid.global.interface_id, 8);
 753                 memcpy(req->priv.initiator_port_id + 8,
 754                        &target->initiator_ext, 8);
 755                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 756                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 757         } else {
 758                 memcpy(req->priv.initiator_port_id,
 759                        &target->initiator_ext, 8);
 760                 memcpy(req->priv.initiator_port_id + 8,
 761                        &target->sgid.global.interface_id, 8);
 762                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 763                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 764         }
 765
 766         /*
 767          * Topspin/Cisco SRP targets will reject our login unless we
 768          * zero out the first 8 bytes of our initiator port ID and set
 769          * the second 8 bytes to the local node GUID.
 770          */
 771         if (srp_target_is_topspin(target)) {
 772                 shost_printk(KERN_DEBUG, target->scsi_host,
 773                              PFX "Topspin/Cisco initiator port ID workaround "
 774                              "activated for target GUID %016llx\n",
 775                              be64_to_cpu(target->ioc_guid));
 776                 memset(req->priv.initiator_port_id, 0, 8);
 777                 memcpy(req->priv.initiator_port_id + 8,
 778                        &target->srp_host->srp_dev->dev->node_guid, 8);
 779         }
 780
 781         status = ib_send_cm_req(ch->cm_id, &req->param);
 782
 783         kfree(req);
 784
 785         return status;
 786 }
 787
 788 static bool srp_queue_remove_work(struct srp_target_port *target)
 789 {
 790         bool changed = false;
 791
 792         spin_lock_irq(&target->lock);
 793         if (target->state != SRP_TARGET_REMOVED) {
 794                 target->state = SRP_TARGET_REMOVED;
 795                 changed = true;
 796         }
 797         spin_unlock_irq(&target->lock);
 798
 799         if (changed)
 800                 queue_work(srp_remove_wq, &target->remove_work);
 801
 802         return changed;
 803 }
 804
 805 static void srp_disconnect_target(struct srp_target_port *target)
 806 {
 807         struct srp_rdma_ch *ch;
 808         int i;
 809
 810         /* XXX should send SRP_I_LOGOUT request */
 811
 812         for (i = 0; i < target->ch_count; i++) {
 813                 ch = &target->ch[i];
 814                 ch->connected = false;
 815                 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
 816                         shost_printk(KERN_DEBUG, target->scsi_host,
 817                                      PFX "Sending CM DREQ failed\n");
 818                 }
 819         }
 820 }
 821
 822 static void srp_free_req_data(struct srp_target_port *target,
 823                               struct srp_rdma_ch *ch)
 824 {
 825         struct srp_device *dev = target->srp_host->srp_dev;
 826         struct ib_device *ibdev = dev->dev;
 827         struct srp_request *req;
 828         int i;
 829
 830         if (!ch->req_ring)
 831                 return;
 832
 833         for (i = 0; i < target->req_ring_size; ++i) {
 834                 req = &ch->req_ring[i];
 835                 if (dev->use_fast_reg) {
 836                         kfree(req->fr_list);
 837                 } else {
 838                         kfree(req->fmr_list);
 839                         kfree(req->map_page);
 840                 }
 841                 if (req->indirect_dma_addr) {
 842                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 843                                             target->indirect_size,
 844                                             DMA_TO_DEVICE);
 845                 }
 846                 kfree(req->indirect_desc);
 847         }
 848
 849         kfree(ch->req_ring);
 850         ch->req_ring = NULL;
 851 }
 852
 853 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
 854 {
 855         struct srp_target_port *target = ch->target;
 856         struct srp_device *srp_dev = target->srp_host->srp_dev;
 857         struct ib_device *ibdev = srp_dev->dev;
 858         struct srp_request *req;
 859         void *mr_list;
 860         dma_addr_t dma_addr;
 861         int i, ret = -ENOMEM;
 862
 863         ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
 864                                GFP_KERNEL);
 865         if (!ch->req_ring)
 866                 goto out;
 867
 868         for (i = 0; i < target->req_ring_size; ++i) {
 869                 req = &ch->req_ring[i];
 870                 mr_list = kmalloc(target->mr_per_cmd * sizeof(void *),
 871                                   GFP_KERNEL);
 872                 if (!mr_list)
 873                         goto out;
 874                 if (srp_dev->use_fast_reg) {
 875                         req->fr_list = mr_list;
 876                 } else {
 877                         req->fmr_list = mr_list;
 878                         req->map_page = kmalloc(srp_dev->max_pages_per_mr *
 879                                                 sizeof(void *), GFP_KERNEL);
 880                         if (!req->map_page)
 881                                 goto out;
 882                 }
 883                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 884                 if (!req->indirect_desc)
 885                         goto out;
 886
 887                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 888                                              target->indirect_size,
 889                                              DMA_TO_DEVICE);
 890                 if (ib_dma_mapping_error(ibdev, dma_addr))
 891                         goto out;
 892
 893                 req->indirect_dma_addr = dma_addr;
 894         }
 895         ret = 0;
 896
 897 out:
 898         return ret;
 899 }
 900
 901 /**
 902  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 903  * @shost: SCSI host whose attributes to remove from sysfs.
 904  *
 905  * Note: Any attributes defined in the host template and that did not exist
 906  * before invocation of this function will be ignored.
 907  */
 908 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 909 {
 910         struct device_attribute **attr;
 911
 912         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 913                 device_remove_file(&shost->shost_dev, *attr);
 914 }
 915
 916 static void srp_remove_target(struct srp_target_port *target)
 917 {
 918         struct srp_rdma_ch *ch;
 919         int i;
 920
 921         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 922
 923         srp_del_scsi_host_attr(target->scsi_host);
 924         srp_rport_get(target->rport);
 925         srp_remove_host(target->scsi_host);
 926         scsi_remove_host(target->scsi_host);
 927         srp_stop_rport_timers(target->rport);
 928         srp_disconnect_target(target);
 929         for (i = 0; i < target->ch_count; i++) {
 930                 ch = &target->ch[i];
 931                 srp_free_ch_ib(target, ch);
 932         }
 933         cancel_work_sync(&target->tl_err_work);
 934         srp_rport_put(target->rport);
 935         for (i = 0; i < target->ch_count; i++) {
 936                 ch = &target->ch[i];
 937                 srp_free_req_data(target, ch);
 938         }
 939         kfree(target->ch);
 940         target->ch = NULL;
 941
 942         spin_lock(&target->srp_host->target_lock);
 943         list_del(&target->list);
 944         spin_unlock(&target->srp_host->target_lock);
 945
 946         scsi_host_put(target->scsi_host);
 947 }
 948
 949 static void srp_remove_work(struct work_struct *work)
 950 {
 951         struct srp_target_port *target =
 952                 container_of(work, struct srp_target_port, remove_work);
 953
 954         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 955
 956         srp_remove_target(target);
 957 }
 958
 959 static void srp_rport_delete(struct srp_rport *rport)
 960 {
 961         struct srp_target_port *target = rport->lld_data;
 962
 963         srp_queue_remove_work(target);
 964 }
 965
 966 /**
 967  * srp_connected_ch() - number of connected channels
 968  * @target: SRP target port.
 969  */
 970 static int srp_connected_ch(struct srp_target_port *target)
 971 {
 972         int i, c = 0;
 973
 974         for (i = 0; i < target->ch_count; i++)
 975                 c += target->ch[i].connected;
 976
 977         return c;
 978 }
 979
 980 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
 981 {
 982         struct srp_target_port *target = ch->target;
 983         int ret;
 984
 985         WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
 986
 987         ret = srp_lookup_path(ch);
 988         if (ret)
 989                 goto out;
 990
 991         while (1) {
 992                 init_completion(&ch->done);
 993                 ret = srp_send_req(ch, multich);
 994                 if (ret)
 995                         goto out;
 996                 ret = wait_for_completion_interruptible(&ch->done);
 997                 if (ret < 0)
 998                         goto out;
 999
1000                 /*
1001                  * The CM event handling code will set status to
1002                  * SRP_PORT_REDIRECT if we get a port redirect REJ
1003                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
1004                  * redirect REJ back.
1005                  */
1006                 ret = ch->status;
1007                 switch (ret) {
1008                 case 0:
1009                         ch->connected = true;
1010                         goto out;
1011
1012                 case SRP_PORT_REDIRECT:
1013                         ret = srp_lookup_path(ch);
1014                         if (ret)
1015                                 goto out;
1016                         break;
1017
1018                 case SRP_DLID_REDIRECT:
1019                         break;
1020
1021                 case SRP_STALE_CONN:
1022                         shost_printk(KERN_ERR, target->scsi_host, PFX
1023                                      "giving up on stale connection\n");
1024                         ret = -ECONNRESET;
1025                         goto out;
1026
1027                 default:
1028                         goto out;
1029                 }
1030         }
1031
1032 out:
1033         return ret <= 0 ? ret : -ENODEV;
1034 }
1035
1036 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1037 {
1038         srp_handle_qp_err(cq, wc, "INV RKEY");
1039 }
1040
1041 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1042                 u32 rkey)
1043 {
1044         struct ib_send_wr *bad_wr;
1045         struct ib_send_wr wr = {
1046                 .opcode             = IB_WR_LOCAL_INV,
1047                 .next               = NULL,
1048                 .num_sge            = 0,
1049                 .send_flags         = 0,
1050                 .ex.invalidate_rkey = rkey,
1051         };
1052
1053         wr.wr_cqe = &req->reg_cqe;
1054         req->reg_cqe.done = srp_inv_rkey_err_done;
1055         return ib_post_send(ch->qp, &wr, &bad_wr);
1056 }
1057
1058 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1059                            struct srp_rdma_ch *ch,
1060                            struct srp_request *req)
1061 {
1062         struct srp_target_port *target = ch->target;
1063         struct srp_device *dev = target->srp_host->srp_dev;
1064         struct ib_device *ibdev = dev->dev;
1065         int i, res;
1066
1067         if (!scsi_sglist(scmnd) ||
1068             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1069              scmnd->sc_data_direction != DMA_FROM_DEVICE))
1070                 return;
1071
1072         if (dev->use_fast_reg) {
1073                 struct srp_fr_desc **pfr;
1074
1075                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1076                         res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1077                         if (res < 0) {
1078                                 shost_printk(KERN_ERR, target->scsi_host, PFX
1079                                   "Queueing INV WR for rkey %#x failed (%d)\n",
1080                                   (*pfr)->mr->rkey, res);
1081                                 queue_work(system_long_wq,
1082                                            &target->tl_err_work);
1083                         }
1084                 }
1085                 if (req->nmdesc)
1086                         srp_fr_pool_put(ch->fr_pool, req->fr_list,
1087                                         req->nmdesc);
1088         } else if (dev->use_fmr) {
1089                 struct ib_pool_fmr **pfmr;
1090
1091                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1092                         ib_fmr_pool_unmap(*pfmr);
1093         }
1094
1095         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1096                         scmnd->sc_data_direction);
1097 }
1098
1099 /**
1100  * srp_claim_req - Take ownership of the scmnd associated with a request.
1101  * @ch: SRP RDMA channel.
1102  * @req: SRP request.
1103  * @sdev: If not NULL, only take ownership for this SCSI device.
1104  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1105  *         ownership of @req->scmnd if it equals @scmnd.
1106  *
1107  * Return value:
1108  * Either NULL or a pointer to the SCSI command the caller became owner of.
1109  */
1110 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1111                                        struct srp_request *req,
1112                                        struct scsi_device *sdev,
1113                                        struct scsi_cmnd *scmnd)
1114 {
1115         unsigned long flags;
1116
1117         spin_lock_irqsave(&ch->lock, flags);
1118         if (req->scmnd &&
1119             (!sdev || req->scmnd->device == sdev) &&
1120             (!scmnd || req->scmnd == scmnd)) {
1121                 scmnd = req->scmnd;
1122                 req->scmnd = NULL;
1123         } else {
1124                 scmnd = NULL;
1125         }
1126         spin_unlock_irqrestore(&ch->lock, flags);
1127
1128         return scmnd;
1129 }
1130
1131 /**
1132  * srp_free_req() - Unmap data and adjust ch->req_lim.
1133  * @ch:     SRP RDMA channel.
1134  * @req:    Request to be freed.
1135  * @scmnd:  SCSI command associated with @req.
1136  * @req_lim_delta: Amount to be added to @target->req_lim.
1137  */
1138 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1139                          struct scsi_cmnd *scmnd, s32 req_lim_delta)
1140 {
1141         unsigned long flags;
1142
1143         srp_unmap_data(scmnd, ch, req);
1144
1145         spin_lock_irqsave(&ch->lock, flags);
1146         ch->req_lim += req_lim_delta;
1147         spin_unlock_irqrestore(&ch->lock, flags);
1148 }
1149
1150 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1151                            struct scsi_device *sdev, int result)
1152 {
1153         struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1154
1155         if (scmnd) {
1156                 srp_free_req(ch, req, scmnd, 0);
1157                 scmnd->result = result;
1158                 scmnd->scsi_done(scmnd);
1159         }
1160 }
1161
1162 static void srp_terminate_io(struct srp_rport *rport)
1163 {
1164         struct srp_target_port *target = rport->lld_data;
1165         struct srp_rdma_ch *ch;
1166         struct Scsi_Host *shost = target->scsi_host;
1167         struct scsi_device *sdev;
1168         int i, j;
1169
1170         /*
1171          * Invoking srp_terminate_io() while srp_queuecommand() is running
1172          * is not safe. Hence the warning statement below.
1173          */
1174         shost_for_each_device(sdev, shost)
1175                 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1176
1177         for (i = 0; i < target->ch_count; i++) {
1178                 ch = &target->ch[i];
1179
1180                 for (j = 0; j < target->req_ring_size; ++j) {
1181                         struct srp_request *req = &ch->req_ring[j];
1182
1183                         srp_finish_req(ch, req, NULL,
1184                                        DID_TRANSPORT_FAILFAST << 16);
1185                 }
1186         }
1187 }
1188
1189 /*
1190  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1191  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1192  * srp_reset_device() or srp_reset_host() calls will occur while this function
1193  * is in progress. One way to realize that is not to call this function
1194  * directly but to call srp_reconnect_rport() instead since that last function
1195  * serializes calls of this function via rport->mutex and also blocks
1196  * srp_queuecommand() calls before invoking this function.
1197  */
1198 static int srp_rport_reconnect(struct srp_rport *rport)
1199 {
1200         struct srp_target_port *target = rport->lld_data;
1201         struct srp_rdma_ch *ch;
1202         int i, j, ret = 0;
1203         bool multich = false;
1204
1205         srp_disconnect_target(target);
1206
1207         if (target->state == SRP_TARGET_SCANNING)
1208                 return -ENODEV;
1209
1210         /*
1211          * Now get a new local CM ID so that we avoid confusing the target in
1212          * case things are really fouled up. Doing so also ensures that all CM
1213          * callbacks will have finished before a new QP is allocated.
1214          */
1215         for (i = 0; i < target->ch_count; i++) {
1216                 ch = &target->ch[i];
1217                 ret += srp_new_cm_id(ch);
1218         }
1219         for (i = 0; i < target->ch_count; i++) {
1220                 ch = &target->ch[i];
1221                 for (j = 0; j < target->req_ring_size; ++j) {
1222                         struct srp_request *req = &ch->req_ring[j];
1223
1224                         srp_finish_req(ch, req, NULL, DID_RESET << 16);
1225                 }
1226         }
1227         for (i = 0; i < target->ch_count; i++) {
1228                 ch = &target->ch[i];
1229                 /*
1230                  * Whether or not creating a new CM ID succeeded, create a new
1231                  * QP. This guarantees that all completion callback function
1232                  * invocations have finished before request resetting starts.
1233                  */
1234                 ret += srp_create_ch_ib(ch);
1235
1236                 INIT_LIST_HEAD(&ch->free_tx);
1237                 for (j = 0; j < target->queue_size; ++j)
1238                         list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1239         }
1240
1241         target->qp_in_error = false;
1242
1243         for (i = 0; i < target->ch_count; i++) {
1244                 ch = &target->ch[i];
1245                 if (ret)
1246                         break;
1247                 ret = srp_connect_ch(ch, multich);
1248                 multich = true;
1249         }
1250
1251         if (ret == 0)
1252                 shost_printk(KERN_INFO, target->scsi_host,
1253                              PFX "reconnect succeeded\n");
1254
1255         return ret;
1256 }
1257
1258 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1259                          unsigned int dma_len, u32 rkey)
1260 {
1261         struct srp_direct_buf *desc = state->desc;
1262
1263         WARN_ON_ONCE(!dma_len);
1264
1265         desc->va = cpu_to_be64(dma_addr);
1266         desc->key = cpu_to_be32(rkey);
1267         desc->len = cpu_to_be32(dma_len);
1268
1269         state->total_len += dma_len;
1270         state->desc++;
1271         state->ndesc++;
1272 }
1273
1274 static int srp_map_finish_fmr(struct srp_map_state *state,
1275                               struct srp_rdma_ch *ch)
1276 {
1277         struct srp_target_port *target = ch->target;
1278         struct srp_device *dev = target->srp_host->srp_dev;
1279         struct ib_pd *pd = target->pd;
1280         struct ib_pool_fmr *fmr;
1281         u64 io_addr = 0;
1282
1283         if (state->fmr.next >= state->fmr.end) {
1284                 shost_printk(KERN_ERR, ch->target->scsi_host,
1285                              PFX "Out of MRs (mr_per_cmd = %d)\n",
1286                              ch->target->mr_per_cmd);
1287                 return -ENOMEM;
1288         }
1289
1290         WARN_ON_ONCE(!dev->use_fmr);
1291
1292         if (state->npages == 0)
1293                 return 0;
1294
1295         if (state->npages == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1296                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1297                              pd->unsafe_global_rkey);
1298                 goto reset_state;
1299         }
1300
1301         fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1302                                    state->npages, io_addr);
1303         if (IS_ERR(fmr))
1304                 return PTR_ERR(fmr);
1305
1306         *state->fmr.next++ = fmr;
1307         state->nmdesc++;
1308
1309         srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1310                      state->dma_len, fmr->fmr->rkey);
1311
1312 reset_state:
1313         state->npages = 0;
1314         state->dma_len = 0;
1315
1316         return 0;
1317 }
1318
1319 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1320 {
1321         srp_handle_qp_err(cq, wc, "FAST REG");
1322 }
1323
1324 /*
1325  * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
1326  * where to start in the first element. If sg_offset_p != NULL then
1327  * *sg_offset_p is updated to the offset in state->sg[retval] of the first
1328  * byte that has not yet been mapped.
1329  */
1330 static int srp_map_finish_fr(struct srp_map_state *state,
1331                              struct srp_request *req,
1332                              struct srp_rdma_ch *ch, int sg_nents,
1333                              unsigned int *sg_offset_p)
1334 {
1335         struct srp_target_port *target = ch->target;
1336         struct srp_device *dev = target->srp_host->srp_dev;
1337         struct ib_pd *pd = target->pd;
1338         struct ib_send_wr *bad_wr;
1339         struct ib_reg_wr wr;
1340         struct srp_fr_desc *desc;
1341         u32 rkey;
1342         int n, err;
1343
1344         if (state->fr.next >= state->fr.end) {
1345                 shost_printk(KERN_ERR, ch->target->scsi_host,
1346                              PFX "Out of MRs (mr_per_cmd = %d)\n",
1347                              ch->target->mr_per_cmd);
1348                 return -ENOMEM;
1349         }
1350
1351         WARN_ON_ONCE(!dev->use_fast_reg);
1352
1353         if (sg_nents == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1354                 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1355
1356                 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
1357                              sg_dma_len(state->sg) - sg_offset,
1358                              pd->unsafe_global_rkey);
1359                 if (sg_offset_p)
1360                         *sg_offset_p = 0;
1361                 return 1;
1362         }
1363
1364         desc = srp_fr_pool_get(ch->fr_pool);
1365         if (!desc)
1366                 return -ENOMEM;
1367
1368         rkey = ib_inc_rkey(desc->mr->rkey);
1369         ib_update_fast_reg_key(desc->mr, rkey);
1370
1371         n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
1372                          dev->mr_page_size);
1373         if (unlikely(n < 0)) {
1374                 srp_fr_pool_put(ch->fr_pool, &desc, 1);
1375                 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1376                          dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1377                          sg_offset_p ? *sg_offset_p : -1, n);
1378                 return n;
1379         }
1380
1381         WARN_ON_ONCE(desc->mr->length == 0);
1382
1383         req->reg_cqe.done = srp_reg_mr_err_done;
1384
1385         wr.wr.next = NULL;
1386         wr.wr.opcode = IB_WR_REG_MR;
1387         wr.wr.wr_cqe = &req->reg_cqe;
1388         wr.wr.num_sge = 0;
1389         wr.wr.send_flags = 0;
1390         wr.mr = desc->mr;
1391         wr.key = desc->mr->rkey;
1392         wr.access = (IB_ACCESS_LOCAL_WRITE |
1393                      IB_ACCESS_REMOTE_READ |
1394                      IB_ACCESS_REMOTE_WRITE);
1395
1396         *state->fr.next++ = desc;
1397         state->nmdesc++;
1398
1399         srp_map_desc(state, desc->mr->iova,
1400                      desc->mr->length, desc->mr->rkey);
1401
1402         err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1403         if (unlikely(err)) {
1404                 WARN_ON_ONCE(err == -ENOMEM);
1405                 return err;
1406         }
1407
1408         return n;
1409 }
1410
1411 static int srp_map_sg_entry(struct srp_map_state *state,
1412                             struct srp_rdma_ch *ch,
1413                             struct scatterlist *sg)
1414 {
1415         struct srp_target_port *target = ch->target;
1416         struct srp_device *dev = target->srp_host->srp_dev;
1417         struct ib_device *ibdev = dev->dev;
1418         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1419         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1420         unsigned int len = 0;
1421         int ret;
1422
1423         WARN_ON_ONCE(!dma_len);
1424
1425         while (dma_len) {
1426                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1427
1428                 if (state->npages == dev->max_pages_per_mr ||
1429                     (state->npages > 0 && offset != 0)) {
1430                         ret = srp_map_finish_fmr(state, ch);
1431                         if (ret)
1432                                 return ret;
1433                 }
1434
1435                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1436
1437                 if (!state->npages)
1438                         state->base_dma_addr = dma_addr;
1439                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1440                 state->dma_len += len;
1441                 dma_addr += len;
1442                 dma_len -= len;
1443         }
1444
1445         /*
1446          * If the end of the MR is not on a page boundary then we need to
1447          * close it out and start a new one -- we can only merge at page
1448          * boundaries.
1449          */
1450         ret = 0;
1451         if ((dma_addr & ~dev->mr_page_mask) != 0)
1452                 ret = srp_map_finish_fmr(state, ch);
1453         return ret;
1454 }
1455
1456 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1457                           struct srp_request *req, struct scatterlist *scat,
1458                           int count)
1459 {
1460         struct scatterlist *sg;
1461         int i, ret;
1462
1463         state->pages = req->map_page;
1464         state->fmr.next = req->fmr_list;
1465         state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
1466
1467         for_each_sg(scat, sg, count, i) {
1468                 ret = srp_map_sg_entry(state, ch, sg);
1469                 if (ret)
1470                         return ret;
1471         }
1472
1473         ret = srp_map_finish_fmr(state, ch);
1474         if (ret)
1475                 return ret;
1476
1477         return 0;
1478 }
1479
1480 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1481                          struct srp_request *req, struct scatterlist *scat,
1482                          int count)
1483 {
1484         unsigned int sg_offset = 0;
1485
1486         state->fr.next = req->fr_list;
1487         state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1488         state->sg = scat;
1489
1490         if (count == 0)
1491                 return 0;
1492
1493         while (count) {
1494                 int i, n;
1495
1496                 n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1497                 if (unlikely(n < 0))
1498                         return n;
1499
1500                 count -= n;
1501                 for (i = 0; i < n; i++)
1502                         state->sg = sg_next(state->sg);
1503         }
1504
1505         return 0;
1506 }
1507
1508 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1509                           struct srp_request *req, struct scatterlist *scat,
1510                           int count)
1511 {
1512         struct srp_target_port *target = ch->target;
1513         struct srp_device *dev = target->srp_host->srp_dev;
1514         struct scatterlist *sg;
1515         int i;
1516
1517         for_each_sg(scat, sg, count, i) {
1518                 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1519                              ib_sg_dma_len(dev->dev, sg),
1520                              target->pd->unsafe_global_rkey);
1521         }
1522
1523         return 0;
1524 }
1525
1526 /*
1527  * Register the indirect data buffer descriptor with the HCA.
1528  *
1529  * Note: since the indirect data buffer descriptor has been allocated with
1530  * kmalloc() it is guaranteed that this buffer is a physically contiguous
1531  * memory buffer.
1532  */
1533 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1534                        void **next_mr, void **end_mr, u32 idb_len,
1535                        __be32 *idb_rkey)
1536 {
1537         struct srp_target_port *target = ch->target;
1538         struct srp_device *dev = target->srp_host->srp_dev;
1539         struct srp_map_state state;
1540         struct srp_direct_buf idb_desc;
1541         u64 idb_pages[1];
1542         struct scatterlist idb_sg[1];
1543         int ret;
1544
1545         memset(&state, 0, sizeof(state));
1546         memset(&idb_desc, 0, sizeof(idb_desc));
1547         state.gen.next = next_mr;
1548         state.gen.end = end_mr;
1549         state.desc = &idb_desc;
1550         state.base_dma_addr = req->indirect_dma_addr;
1551         state.dma_len = idb_len;
1552
1553         if (dev->use_fast_reg) {
1554                 state.sg = idb_sg;
1555                 sg_init_one(idb_sg, req->indirect_desc, idb_len);
1556                 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1557 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1558                 idb_sg->dma_length = idb_sg->length;          /* hack^2 */
1559 #endif
1560                 ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1561                 if (ret < 0)
1562                         return ret;
1563                 WARN_ON_ONCE(ret < 1);
1564         } else if (dev->use_fmr) {
1565                 state.pages = idb_pages;
1566                 state.pages[0] = (req->indirect_dma_addr &
1567                                   dev->mr_page_mask);
1568                 state.npages = 1;
1569                 ret = srp_map_finish_fmr(&state, ch);
1570                 if (ret < 0)
1571                         return ret;
1572         } else {
1573                 return -EINVAL;
1574         }
1575
1576         *idb_rkey = idb_desc.key;
1577
1578         return 0;
1579 }
1580
1581 static void srp_check_mapping(struct srp_map_state *state,
1582                               struct srp_rdma_ch *ch, struct srp_request *req,
1583                               struct scatterlist *scat, int count)
1584 {
1585         struct srp_device *dev = ch->target->srp_host->srp_dev;
1586         struct srp_fr_desc **pfr;
1587         u64 desc_len = 0, mr_len = 0;
1588         int i;
1589
1590         for (i = 0; i < state->ndesc; i++)
1591                 desc_len += be32_to_cpu(req->indirect_desc[i].len);
1592         if (dev->use_fast_reg)
1593                 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
1594                         mr_len += (*pfr)->mr->length;
1595         else if (dev->use_fmr)
1596                 for (i = 0; i < state->nmdesc; i++)
1597                         mr_len += be32_to_cpu(req->indirect_desc[i].len);
1598         if (desc_len != scsi_bufflen(req->scmnd) ||
1599             mr_len > scsi_bufflen(req->scmnd))
1600                 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
1601                        scsi_bufflen(req->scmnd), desc_len, mr_len,
1602                        state->ndesc, state->nmdesc);
1603 }
1604
1605 /**
1606  * srp_map_data() - map SCSI data buffer onto an SRP request
1607  * @scmnd: SCSI command to map
1608  * @ch: SRP RDMA channel
1609  * @req: SRP request
1610  *
1611  * Returns the length in bytes of the SRP_CMD IU or a negative value if
1612  * mapping failed.
1613  */
1614 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1615                         struct srp_request *req)
1616 {
1617         struct srp_target_port *target = ch->target;
1618         struct ib_pd *pd = target->pd;
1619         struct scatterlist *scat;
1620         struct srp_cmd *cmd = req->cmd->buf;
1621         int len, nents, count, ret;
1622         struct srp_device *dev;
1623         struct ib_device *ibdev;
1624         struct srp_map_state state;
1625         struct srp_indirect_buf *indirect_hdr;
1626         u32 idb_len, table_len;
1627         __be32 idb_rkey;
1628         u8 fmt;
1629
1630         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1631                 return sizeof (struct srp_cmd);
1632
1633         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1634             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1635                 shost_printk(KERN_WARNING, target->scsi_host,
1636                              PFX "Unhandled data direction %d\n",
1637                              scmnd->sc_data_direction);
1638                 return -EINVAL;
1639         }
1640
1641         nents = scsi_sg_count(scmnd);
1642         scat  = scsi_sglist(scmnd);
1643
1644         dev = target->srp_host->srp_dev;
1645         ibdev = dev->dev;
1646
1647         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1648         if (unlikely(count == 0))
1649                 return -EIO;
1650
1651         fmt = SRP_DATA_DESC_DIRECT;
1652         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1653
1654         if (count == 1 && (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1655                 /*
1656                  * The midlayer only generated a single gather/scatter
1657                  * entry, or DMA mapping coalesced everything to a
1658                  * single entry.  So a direct descriptor along with
1659                  * the DMA MR suffices.
1660                  */
1661                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1662
1663                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1664                 buf->key = cpu_to_be32(pd->unsafe_global_rkey);
1665                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1666
1667                 req->nmdesc = 0;
1668                 goto map_complete;
1669         }
1670
1671         /*
1672          * We have more than one scatter/gather entry, so build our indirect
1673          * descriptor table, trying to merge as many entries as we can.
1674          */
1675         indirect_hdr = (void *) cmd->add_data;
1676
1677         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1678                                    target->indirect_size, DMA_TO_DEVICE);
1679
1680         memset(&state, 0, sizeof(state));
1681         state.desc = req->indirect_desc;
1682         if (dev->use_fast_reg)
1683                 ret = srp_map_sg_fr(&state, ch, req, scat, count);
1684         else if (dev->use_fmr)
1685                 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
1686         else
1687                 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1688         req->nmdesc = state.nmdesc;
1689         if (ret < 0)
1690                 goto unmap;
1691
1692         {
1693                 DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
1694                         "Memory mapping consistency check");
1695                 if (DYNAMIC_DEBUG_BRANCH(ddm))
1696                         srp_check_mapping(&state, ch, req, scat, count);
1697         }
1698
1699         /* We've mapped the request, now pull as much of the indirect
1700          * descriptor table as we can into the command buffer. If this
1701          * target is not using an external indirect table, we are
1702          * guaranteed to fit into the command, as the SCSI layer won't
1703          * give us more S/G entries than we allow.
1704          */
1705         if (state.ndesc == 1) {
1706                 /*
1707                  * Memory registration collapsed the sg-list into one entry,
1708                  * so use a direct descriptor.
1709                  */
1710                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1711
1712                 *buf = req->indirect_desc[0];
1713                 goto map_complete;
1714         }
1715
1716         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1717                                                 !target->allow_ext_sg)) {
1718                 shost_printk(KERN_ERR, target->scsi_host,
1719                              "Could not fit S/G list into SRP_CMD\n");
1720                 ret = -EIO;
1721                 goto unmap;
1722         }
1723
1724         count = min(state.ndesc, target->cmd_sg_cnt);
1725         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1726         idb_len = sizeof(struct srp_indirect_buf) + table_len;
1727
1728         fmt = SRP_DATA_DESC_INDIRECT;
1729         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1730         len += count * sizeof (struct srp_direct_buf);
1731
1732         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1733                count * sizeof (struct srp_direct_buf));
1734
1735         if (!(pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)) {
1736                 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1737                                   idb_len, &idb_rkey);
1738                 if (ret < 0)
1739                         goto unmap;
1740                 req->nmdesc++;
1741         } else {
1742                 idb_rkey = cpu_to_be32(pd->unsafe_global_rkey);
1743         }
1744
1745         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1746         indirect_hdr->table_desc.key = idb_rkey;
1747         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1748         indirect_hdr->len = cpu_to_be32(state.total_len);
1749
1750         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1751                 cmd->data_out_desc_cnt = count;
1752         else
1753                 cmd->data_in_desc_cnt = count;
1754
1755         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1756                                       DMA_TO_DEVICE);
1757
1758 map_complete:
1759         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1760                 cmd->buf_fmt = fmt << 4;
1761         else
1762                 cmd->buf_fmt = fmt;
1763
1764         return len;
1765
1766 unmap:
1767         srp_unmap_data(scmnd, ch, req);
1768         if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
1769                 ret = -E2BIG;
1770         return ret;
1771 }
1772
1773 /*
1774  * Return an IU and possible credit to the free pool
1775  */
1776 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1777                           enum srp_iu_type iu_type)
1778 {
1779         unsigned long flags;
1780
1781         spin_lock_irqsave(&ch->lock, flags);
1782         list_add(&iu->list, &ch->free_tx);
1783         if (iu_type != SRP_IU_RSP)
1784                 ++ch->req_lim;
1785         spin_unlock_irqrestore(&ch->lock, flags);
1786 }
1787
1788 /*
1789  * Must be called with ch->lock held to protect req_lim and free_tx.
1790  * If IU is not sent, it must be returned using srp_put_tx_iu().
1791  *
1792  * Note:
1793  * An upper limit for the number of allocated information units for each
1794  * request type is:
1795  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1796  *   more than Scsi_Host.can_queue requests.
1797  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1798  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1799  *   one unanswered SRP request to an initiator.
1800  */
1801 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1802                                       enum srp_iu_type iu_type)
1803 {
1804         struct srp_target_port *target = ch->target;
1805         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1806         struct srp_iu *iu;
1807
1808         lockdep_assert_held(&ch->lock);
1809
1810         ib_process_cq_direct(ch->send_cq, -1);
1811
1812         if (list_empty(&ch->free_tx))
1813                 return NULL;
1814
1815         /* Initiator responses to target requests do not consume credits */
1816         if (iu_type != SRP_IU_RSP) {
1817                 if (ch->req_lim <= rsv) {
1818                         ++target->zero_req_lim;
1819                         return NULL;
1820                 }
1821
1822                 --ch->req_lim;
1823         }
1824
1825         iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1826         list_del(&iu->list);
1827         return iu;
1828 }
1829
1830 /*
1831  * Note: if this function is called from inside ib_drain_sq() then it will
1832  * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE
1833  * with status IB_WC_SUCCESS then that's a bug.
1834  */
1835 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1836 {
1837         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1838         struct srp_rdma_ch *ch = cq->cq_context;
1839
1840         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1841                 srp_handle_qp_err(cq, wc, "SEND");
1842                 return;
1843         }
1844
1845         lockdep_assert_held(&ch->lock);
1846
1847         list_add(&iu->list, &ch->free_tx);
1848 }
1849
1850 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1851 {
1852         struct srp_target_port *target = ch->target;
1853         struct ib_sge list;
1854         struct ib_send_wr wr, *bad_wr;
1855
1856         list.addr   = iu->dma;
1857         list.length = len;
1858         list.lkey   = target->lkey;
1859
1860         iu->cqe.done = srp_send_done;
1861
1862         wr.next       = NULL;
1863         wr.wr_cqe     = &iu->cqe;
1864         wr.sg_list    = &list;
1865         wr.num_sge    = 1;
1866         wr.opcode     = IB_WR_SEND;
1867         wr.send_flags = IB_SEND_SIGNALED;
1868
1869         return ib_post_send(ch->qp, &wr, &bad_wr);
1870 }
1871
1872 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1873 {
1874         struct srp_target_port *target = ch->target;
1875         struct ib_recv_wr wr, *bad_wr;
1876         struct ib_sge list;
1877
1878         list.addr   = iu->dma;
1879         list.length = iu->size;
1880         list.lkey   = target->lkey;
1881
1882         iu->cqe.done = srp_recv_done;
1883
1884         wr.next     = NULL;
1885         wr.wr_cqe   = &iu->cqe;
1886         wr.sg_list  = &list;
1887         wr.num_sge  = 1;
1888
1889         return ib_post_recv(ch->qp, &wr, &bad_wr);
1890 }
1891
1892 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1893 {
1894         struct srp_target_port *target = ch->target;
1895         struct srp_request *req;
1896         struct scsi_cmnd *scmnd;
1897         unsigned long flags;
1898
1899         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1900                 spin_lock_irqsave(&ch->lock, flags);
1901                 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1902                 if (rsp->tag == ch->tsk_mgmt_tag) {
1903                         ch->tsk_mgmt_status = -1;
1904                         if (be32_to_cpu(rsp->resp_data_len) >= 4)
1905                                 ch->tsk_mgmt_status = rsp->data[3];
1906                         complete(&ch->tsk_mgmt_done);
1907                 } else {
1908                         shost_printk(KERN_ERR, target->scsi_host,
1909                                      "Received tsk mgmt response too late for tag %#llx\n",
1910                                      rsp->tag);
1911                 }
1912                 spin_unlock_irqrestore(&ch->lock, flags);
1913         } else {
1914                 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1915                 if (scmnd && scmnd->host_scribble) {
1916                         req = (void *)scmnd->host_scribble;
1917                         scmnd = srp_claim_req(ch, req, NULL, scmnd);
1918                 } else {
1919                         scmnd = NULL;
1920                 }
1921                 if (!scmnd) {
1922                         shost_printk(KERN_ERR, target->scsi_host,
1923                                      "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1924                                      rsp->tag, ch - target->ch, ch->qp->qp_num);
1925
1926                         spin_lock_irqsave(&ch->lock, flags);
1927                         ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1928                         spin_unlock_irqrestore(&ch->lock, flags);
1929
1930                         return;
1931                 }
1932                 scmnd->result = rsp->status;
1933
1934                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1935                         memcpy(scmnd->sense_buffer, rsp->data +
1936                                be32_to_cpu(rsp->resp_data_len),
1937                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1938                                      SCSI_SENSE_BUFFERSIZE));
1939                 }
1940
1941                 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1942                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1943                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1944                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1945                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1946                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1947                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1948                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1949
1950                 srp_free_req(ch, req, scmnd,
1951                              be32_to_cpu(rsp->req_lim_delta));
1952
1953                 scmnd->host_scribble = NULL;
1954                 scmnd->scsi_done(scmnd);
1955         }
1956 }
1957
1958 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1959                                void *rsp, int len)
1960 {
1961         struct srp_target_port *target = ch->target;
1962         struct ib_device *dev = target->srp_host->srp_dev->dev;
1963         unsigned long flags;
1964         struct srp_iu *iu;
1965         int err;
1966
1967         spin_lock_irqsave(&ch->lock, flags);
1968         ch->req_lim += req_delta;
1969         iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1970         spin_unlock_irqrestore(&ch->lock, flags);
1971
1972         if (!iu) {
1973                 shost_printk(KERN_ERR, target->scsi_host, PFX
1974                              "no IU available to send response\n");
1975                 return 1;
1976         }
1977
1978         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1979         memcpy(iu->buf, rsp, len);
1980         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1981
1982         err = srp_post_send(ch, iu, len);
1983         if (err) {
1984                 shost_printk(KERN_ERR, target->scsi_host, PFX
1985                              "unable to post response: %d\n", err);
1986                 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1987         }
1988
1989         return err;
1990 }
1991
1992 static void srp_process_cred_req(struct srp_rdma_ch *ch,
1993                                  struct srp_cred_req *req)
1994 {
1995         struct srp_cred_rsp rsp = {
1996                 .opcode = SRP_CRED_RSP,
1997                 .tag = req->tag,
1998         };
1999         s32 delta = be32_to_cpu(req->req_lim_delta);
2000
2001         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2002                 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
2003                              "problems processing SRP_CRED_REQ\n");
2004 }
2005
2006 static void srp_process_aer_req(struct srp_rdma_ch *ch,
2007                                 struct srp_aer_req *req)
2008 {
2009         struct srp_target_port *target = ch->target;
2010         struct srp_aer_rsp rsp = {
2011                 .opcode = SRP_AER_RSP,
2012                 .tag = req->tag,
2013         };
2014         s32 delta = be32_to_cpu(req->req_lim_delta);
2015
2016         shost_printk(KERN_ERR, target->scsi_host, PFX
2017                      "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
2018
2019         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2020                 shost_printk(KERN_ERR, target->scsi_host, PFX
2021                              "problems processing SRP_AER_REQ\n");
2022 }
2023
2024 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2025 {
2026         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
2027         struct srp_rdma_ch *ch = cq->cq_context;
2028         struct srp_target_port *target = ch->target;
2029         struct ib_device *dev = target->srp_host->srp_dev->dev;
2030         int res;
2031         u8 opcode;
2032
2033         if (unlikely(wc->status != IB_WC_SUCCESS)) {
2034                 srp_handle_qp_err(cq, wc, "RECV");
2035                 return;
2036         }
2037
2038         ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2039                                    DMA_FROM_DEVICE);
2040
2041         opcode = *(u8 *) iu->buf;
2042
2043         if (0) {
2044                 shost_printk(KERN_ERR, target->scsi_host,
2045                              PFX "recv completion, opcode 0x%02x\n", opcode);
2046                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
2047                                iu->buf, wc->byte_len, true);
2048         }
2049
2050         switch (opcode) {
2051         case SRP_RSP:
2052                 srp_process_rsp(ch, iu->buf);
2053                 break;
2054
2055         case SRP_CRED_REQ:
2056                 srp_process_cred_req(ch, iu->buf);
2057                 break;
2058
2059         case SRP_AER_REQ:
2060                 srp_process_aer_req(ch, iu->buf);
2061                 break;
2062
2063         case SRP_T_LOGOUT:
2064                 /* XXX Handle target logout */
2065                 shost_printk(KERN_WARNING, target->scsi_host,
2066                              PFX "Got target logout request\n");
2067                 break;
2068
2069         default:
2070                 shost_printk(KERN_WARNING, target->scsi_host,
2071                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2072                 break;
2073         }
2074
2075         ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2076                                       DMA_FROM_DEVICE);
2077
2078         res = srp_post_recv(ch, iu);
2079         if (res != 0)
2080                 shost_printk(KERN_ERR, target->scsi_host,
2081                              PFX "Recv failed with error code %d\n", res);
2082 }
2083
2084 /**
2085  * srp_tl_err_work() - handle a transport layer error
2086  * @work: Work structure embedded in an SRP target port.
2087  *
2088  * Note: This function may get invoked before the rport has been created,
2089  * hence the target->rport test.
2090  */
2091 static void srp_tl_err_work(struct work_struct *work)
2092 {
2093         struct srp_target_port *target;
2094
2095         target = container_of(work, struct srp_target_port, tl_err_work);
2096         if (target->rport)
2097                 srp_start_tl_fail_timers(target->rport);
2098 }
2099
2100 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2101                 const char *opname)
2102 {
2103         struct srp_rdma_ch *ch = cq->cq_context;
2104         struct srp_target_port *target = ch->target;
2105
2106         if (ch->connected && !target->qp_in_error) {
2107                 shost_printk(KERN_ERR, target->scsi_host,
2108                              PFX "failed %s status %s (%d) for CQE %p\n",
2109                              opname, ib_wc_status_msg(wc->status), wc->status,
2110                              wc->wr_cqe);
2111                 queue_work(system_long_wq, &target->tl_err_work);
2112         }
2113         target->qp_in_error = true;
2114 }
2115
2116 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2117 {
2118         struct srp_target_port *target = host_to_target(shost);
2119         struct srp_rport *rport = target->rport;
2120         struct srp_rdma_ch *ch;
2121         struct srp_request *req;
2122         struct srp_iu *iu;
2123         struct srp_cmd *cmd;
2124         struct ib_device *dev;
2125         unsigned long flags;
2126         u32 tag;
2127         u16 idx;
2128         int len, ret;
2129         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2130
2131         /*
2132          * The SCSI EH thread is the only context from which srp_queuecommand()
2133          * can get invoked for blocked devices (SDEV_BLOCK /
2134          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2135          * locking the rport mutex if invoked from inside the SCSI EH.
2136          */
2137         if (in_scsi_eh)
2138                 mutex_lock(&rport->mutex);
2139
2140         scmnd->result = srp_chkready(target->rport);
2141         if (unlikely(scmnd->result))
2142                 goto err;
2143
2144         WARN_ON_ONCE(scmnd->request->tag < 0);
2145         tag = blk_mq_unique_tag(scmnd->request);
2146         ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2147         idx = blk_mq_unique_tag_to_tag(tag);
2148         WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2149                   dev_name(&shost->shost_gendev), tag, idx,
2150                   target->req_ring_size);
2151
2152         spin_lock_irqsave(&ch->lock, flags);
2153         iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2154         spin_unlock_irqrestore(&ch->lock, flags);
2155
2156         if (!iu)
2157                 goto err;
2158
2159         req = &ch->req_ring[idx];
2160         dev = target->srp_host->srp_dev->dev;
2161         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2162                                    DMA_TO_DEVICE);
2163
2164         scmnd->host_scribble = (void *) req;
2165
2166         cmd = iu->buf;
2167         memset(cmd, 0, sizeof *cmd);
2168
2169         cmd->opcode = SRP_CMD;
2170         int_to_scsilun(scmnd->device->lun, &cmd->lun);
2171         cmd->tag    = tag;
2172         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2173
2174         req->scmnd    = scmnd;
2175         req->cmd      = iu;
2176
2177         len = srp_map_data(scmnd, ch, req);
2178         if (len < 0) {
2179                 shost_printk(KERN_ERR, target->scsi_host,
2180                              PFX "Failed to map data (%d)\n", len);
2181                 /*
2182                  * If we ran out of memory descriptors (-ENOMEM) because an
2183                  * application is queuing many requests with more than
2184                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2185                  * to reduce queue depth temporarily.
2186                  */
2187                 scmnd->result = len == -ENOMEM ?
2188                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2189                 goto err_iu;
2190         }
2191
2192         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2193                                       DMA_TO_DEVICE);
2194
2195         if (srp_post_send(ch, iu, len)) {
2196                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2197                 goto err_unmap;
2198         }
2199
2200         ret = 0;
2201
2202 unlock_rport:
2203         if (in_scsi_eh)
2204                 mutex_unlock(&rport->mutex);
2205
2206         return ret;
2207
2208 err_unmap:
2209         srp_unmap_data(scmnd, ch, req);
2210
2211 err_iu:
2212         srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2213
2214         /*
2215          * Avoid that the loops that iterate over the request ring can
2216          * encounter a dangling SCSI command pointer.
2217          */
2218         req->scmnd = NULL;
2219
2220 err:
2221         if (scmnd->result) {
2222                 scmnd->scsi_done(scmnd);
2223                 ret = 0;
2224         } else {
2225                 ret = SCSI_MLQUEUE_HOST_BUSY;
2226         }
2227
2228         goto unlock_rport;
2229 }
2230
2231 /*
2232  * Note: the resources allocated in this function are freed in
2233  * srp_free_ch_ib().
2234  */
2235 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2236 {
2237         struct srp_target_port *target = ch->target;
2238         int i;
2239
2240         ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2241                               GFP_KERNEL);
2242         if (!ch->rx_ring)
2243                 goto err_no_ring;
2244         ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2245                               GFP_KERNEL);
2246         if (!ch->tx_ring)
2247                 goto err_no_ring;
2248
2249         for (i = 0; i < target->queue_size; ++i) {
2250                 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2251                                               ch->max_ti_iu_len,
2252                                               GFP_KERNEL, DMA_FROM_DEVICE);
2253                 if (!ch->rx_ring[i])
2254                         goto err;
2255         }
2256
2257         for (i = 0; i < target->queue_size; ++i) {
2258                 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2259                                               target->max_iu_len,
2260                                               GFP_KERNEL, DMA_TO_DEVICE);
2261                 if (!ch->tx_ring[i])
2262                         goto err;
2263
2264                 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2265         }
2266
2267         return 0;
2268
2269 err:
2270         for (i = 0; i < target->queue_size; ++i) {
2271                 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2272                 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2273         }
2274
2275
2276 err_no_ring:
2277         kfree(ch->tx_ring);
2278         ch->tx_ring = NULL;
2279         kfree(ch->rx_ring);
2280         ch->rx_ring = NULL;
2281
2282         return -ENOMEM;
2283 }
2284
2285 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2286 {
2287         uint64_t T_tr_ns, max_compl_time_ms;
2288         uint32_t rq_tmo_jiffies;
2289
2290         /*
2291          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2292          * table 91), both the QP timeout and the retry count have to be set
2293          * for RC QP's during the RTR to RTS transition.
2294          */
2295         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2296                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2297
2298         /*
2299          * Set target->rq_tmo_jiffies to one second more than the largest time
2300          * it can take before an error completion is generated. See also
2301          * C9-140..142 in the IBTA spec for more information about how to
2302          * convert the QP Local ACK Timeout value to nanoseconds.
2303          */
2304         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2305         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2306         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2307         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2308
2309         return rq_tmo_jiffies;
2310 }
2311
2312 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2313                                const struct srp_login_rsp *lrsp,
2314                                struct srp_rdma_ch *ch)
2315 {
2316         struct srp_target_port *target = ch->target;
2317         struct ib_qp_attr *qp_attr = NULL;
2318         int attr_mask = 0;
2319         int ret;
2320         int i;
2321
2322         if (lrsp->opcode == SRP_LOGIN_RSP) {
2323                 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2324                 ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2325
2326                 /*
2327                  * Reserve credits for task management so we don't
2328                  * bounce requests back to the SCSI mid-layer.
2329                  */
2330                 target->scsi_host->can_queue
2331                         = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2332                               target->scsi_host->can_queue);
2333                 target->scsi_host->cmd_per_lun
2334                         = min_t(int, target->scsi_host->can_queue,
2335                                 target->scsi_host->cmd_per_lun);
2336         } else {
2337                 shost_printk(KERN_WARNING, target->scsi_host,
2338                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2339                 ret = -ECONNRESET;
2340                 goto error;
2341         }
2342
2343         if (!ch->rx_ring) {
2344                 ret = srp_alloc_iu_bufs(ch);
2345                 if (ret)
2346                         goto error;
2347         }
2348
2349         ret = -ENOMEM;
2350         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2351         if (!qp_attr)
2352                 goto error;
2353
2354         qp_attr->qp_state = IB_QPS_RTR;
2355         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2356         if (ret)
2357                 goto error_free;
2358
2359         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2360         if (ret)
2361                 goto error_free;
2362
2363         for (i = 0; i < target->queue_size; i++) {
2364                 struct srp_iu *iu = ch->rx_ring[i];
2365
2366                 ret = srp_post_recv(ch, iu);
2367                 if (ret)
2368                         goto error_free;
2369         }
2370
2371         qp_attr->qp_state = IB_QPS_RTS;
2372         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2373         if (ret)
2374                 goto error_free;
2375
2376         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2377
2378         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2379         if (ret)
2380                 goto error_free;
2381
2382         ret = ib_send_cm_rtu(cm_id, NULL, 0);
2383
2384 error_free:
2385         kfree(qp_attr);
2386
2387 error:
2388         ch->status = ret;
2389 }
2390
2391 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2392                                struct ib_cm_event *event,
2393                                struct srp_rdma_ch *ch)
2394 {
2395         struct srp_target_port *target = ch->target;
2396         struct Scsi_Host *shost = target->scsi_host;
2397         struct ib_class_port_info *cpi;
2398         int opcode;
2399
2400         switch (event->param.rej_rcvd.reason) {
2401         case IB_CM_REJ_PORT_CM_REDIRECT:
2402                 cpi = event->param.rej_rcvd.ari;
2403                 sa_path_set_dlid(&ch->path, htonl(ntohs(cpi->redirect_lid)));
2404                 ch->path.pkey = cpi->redirect_pkey;
2405                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2406                 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2407
2408                 ch->status = sa_path_get_dlid(&ch->path) ?
2409                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2410                 break;
2411
2412         case IB_CM_REJ_PORT_REDIRECT:
2413                 if (srp_target_is_topspin(target)) {
2414                         /*
2415                          * Topspin/Cisco SRP gateways incorrectly send
2416                          * reject reason code 25 when they mean 24
2417                          * (port redirect).
2418                          */
2419                         memcpy(ch->path.dgid.raw,
2420                                event->param.rej_rcvd.ari, 16);
2421
2422                         shost_printk(KERN_DEBUG, shost,
2423                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2424                                      be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2425                                      be64_to_cpu(ch->path.dgid.global.interface_id));
2426
2427                         ch->status = SRP_PORT_REDIRECT;
2428                 } else {
2429                         shost_printk(KERN_WARNING, shost,
2430                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2431                         ch->status = -ECONNRESET;
2432                 }
2433                 break;
2434
2435         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2436                 shost_printk(KERN_WARNING, shost,
2437                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2438                 ch->status = -ECONNRESET;
2439                 break;
2440
2441         case IB_CM_REJ_CONSUMER_DEFINED:
2442                 opcode = *(u8 *) event->private_data;
2443                 if (opcode == SRP_LOGIN_REJ) {
2444                         struct srp_login_rej *rej = event->private_data;
2445                         u32 reason = be32_to_cpu(rej->reason);
2446
2447                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2448                                 shost_printk(KERN_WARNING, shost,
2449                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2450                         else
2451                                 shost_printk(KERN_WARNING, shost, PFX
2452                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2453                                              target->sgid.raw,
2454                                              target->orig_dgid.raw, reason);
2455                 } else
2456                         shost_printk(KERN_WARNING, shost,
2457                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2458                                      " opcode 0x%02x\n", opcode);
2459                 ch->status = -ECONNRESET;
2460                 break;
2461
2462         case IB_CM_REJ_STALE_CONN:
2463                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2464                 ch->status = SRP_STALE_CONN;
2465                 break;
2466
2467         default:
2468                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2469                              event->param.rej_rcvd.reason);
2470                 ch->status = -ECONNRESET;
2471         }
2472 }
2473
2474 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2475 {
2476         struct srp_rdma_ch *ch = cm_id->context;
2477         struct srp_target_port *target = ch->target;
2478         int comp = 0;
2479
2480         switch (event->event) {
2481         case IB_CM_REQ_ERROR:
2482                 shost_printk(KERN_DEBUG, target->scsi_host,
2483                              PFX "Sending CM REQ failed\n");
2484                 comp = 1;
2485                 ch->status = -ECONNRESET;
2486                 break;
2487
2488         case IB_CM_REP_RECEIVED:
2489                 comp = 1;
2490                 srp_cm_rep_handler(cm_id, event->private_data, ch);
2491                 break;
2492
2493         case IB_CM_REJ_RECEIVED:
2494                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2495                 comp = 1;
2496
2497                 srp_cm_rej_handler(cm_id, event, ch);
2498                 break;
2499
2500         case IB_CM_DREQ_RECEIVED:
2501                 shost_printk(KERN_WARNING, target->scsi_host,
2502                              PFX "DREQ received - connection closed\n");
2503                 ch->connected = false;
2504                 if (ib_send_cm_drep(cm_id, NULL, 0))
2505                         shost_printk(KERN_ERR, target->scsi_host,
2506                                      PFX "Sending CM DREP failed\n");
2507                 queue_work(system_long_wq, &target->tl_err_work);
2508                 break;
2509
2510         case IB_CM_TIMEWAIT_EXIT:
2511                 shost_printk(KERN_ERR, target->scsi_host,
2512                              PFX "connection closed\n");
2513                 comp = 1;
2514
2515                 ch->status = 0;
2516                 break;
2517
2518         case IB_CM_MRA_RECEIVED:
2519         case IB_CM_DREQ_ERROR:
2520         case IB_CM_DREP_RECEIVED:
2521                 break;
2522
2523         default:
2524                 shost_printk(KERN_WARNING, target->scsi_host,
2525                              PFX "Unhandled CM event %d\n", event->event);
2526                 break;
2527         }
2528
2529         if (comp)
2530                 complete(&ch->done);
2531
2532         return 0;
2533 }
2534
2535 /**
2536  * srp_change_queue_depth - setting device queue depth
2537  * @sdev: scsi device struct
2538  * @qdepth: requested queue depth
2539  *
2540  * Returns queue depth.
2541  */
2542 static int
2543 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2544 {
2545         if (!sdev->tagged_supported)
2546                 qdepth = 1;
2547         return scsi_change_queue_depth(sdev, qdepth);
2548 }
2549
2550 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2551                              u8 func, u8 *status)
2552 {
2553         struct srp_target_port *target = ch->target;
2554         struct srp_rport *rport = target->rport;
2555         struct ib_device *dev = target->srp_host->srp_dev->dev;
2556         struct srp_iu *iu;
2557         struct srp_tsk_mgmt *tsk_mgmt;
2558         int res;
2559
2560         if (!ch->connected || target->qp_in_error)
2561                 return -1;
2562
2563         /*
2564          * Lock the rport mutex to avoid that srp_create_ch_ib() is
2565          * invoked while a task management function is being sent.
2566          */
2567         mutex_lock(&rport->mutex);
2568         spin_lock_irq(&ch->lock);
2569         iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2570         spin_unlock_irq(&ch->lock);
2571
2572         if (!iu) {
2573                 mutex_unlock(&rport->mutex);
2574
2575                 return -1;
2576         }
2577
2578         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2579                                    DMA_TO_DEVICE);
2580         tsk_mgmt = iu->buf;
2581         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2582
2583         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2584         int_to_scsilun(lun, &tsk_mgmt->lun);
2585         tsk_mgmt->tsk_mgmt_func = func;
2586         tsk_mgmt->task_tag      = req_tag;
2587
2588         spin_lock_irq(&ch->lock);
2589         ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT;
2590         tsk_mgmt->tag = ch->tsk_mgmt_tag;
2591         spin_unlock_irq(&ch->lock);
2592
2593         init_completion(&ch->tsk_mgmt_done);
2594
2595         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2596                                       DMA_TO_DEVICE);
2597         if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2598                 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2599                 mutex_unlock(&rport->mutex);
2600
2601                 return -1;
2602         }
2603         res = wait_for_completion_timeout(&ch->tsk_mgmt_done,
2604                                         msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS));
2605         if (res > 0 && status)
2606                 *status = ch->tsk_mgmt_status;
2607         mutex_unlock(&rport->mutex);
2608
2609         WARN_ON_ONCE(res < 0);
2610
2611         return res > 0 ? 0 : -1;
2612 }
2613
2614 static int srp_abort(struct scsi_cmnd *scmnd)
2615 {
2616         struct srp_target_port *target = host_to_target(scmnd->device->host);
2617         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2618         u32 tag;
2619         u16 ch_idx;
2620         struct srp_rdma_ch *ch;
2621         int ret;
2622
2623         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2624
2625         if (!req)
2626                 return SUCCESS;
2627         tag = blk_mq_unique_tag(scmnd->request);
2628         ch_idx = blk_mq_unique_tag_to_hwq(tag);
2629         if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2630                 return SUCCESS;
2631         ch = &target->ch[ch_idx];
2632         if (!srp_claim_req(ch, req, NULL, scmnd))
2633                 return SUCCESS;
2634         shost_printk(KERN_ERR, target->scsi_host,
2635                      "Sending SRP abort for tag %#x\n", tag);
2636         if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2637                               SRP_TSK_ABORT_TASK, NULL) == 0)
2638                 ret = SUCCESS;
2639         else if (target->rport->state == SRP_RPORT_LOST)
2640                 ret = FAST_IO_FAIL;
2641         else
2642                 ret = FAILED;
2643         srp_free_req(ch, req, scmnd, 0);
2644         scmnd->result = DID_ABORT << 16;
2645         scmnd->scsi_done(scmnd);
2646
2647         return ret;
2648 }
2649
2650 static int srp_reset_device(struct scsi_cmnd *scmnd)
2651 {
2652         struct srp_target_port *target = host_to_target(scmnd->device->host);
2653         struct srp_rdma_ch *ch;
2654         int i;
2655         u8 status;
2656
2657         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2658
2659         ch = &target->ch[0];
2660         if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2661                               SRP_TSK_LUN_RESET, &status))
2662                 return FAILED;
2663         if (status)
2664                 return FAILED;
2665
2666         for (i = 0; i < target->ch_count; i++) {
2667                 ch = &target->ch[i];
2668                 for (i = 0; i < target->req_ring_size; ++i) {
2669                         struct srp_request *req = &ch->req_ring[i];
2670
2671                         srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2672                 }
2673         }
2674
2675         return SUCCESS;
2676 }
2677
2678 static int srp_reset_host(struct scsi_cmnd *scmnd)
2679 {
2680         struct srp_target_port *target = host_to_target(scmnd->device->host);
2681
2682         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2683
2684         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2685 }
2686
2687 static int srp_slave_alloc(struct scsi_device *sdev)
2688 {
2689         struct Scsi_Host *shost = sdev->host;
2690         struct srp_target_port *target = host_to_target(shost);
2691         struct srp_device *srp_dev = target->srp_host->srp_dev;
2692
2693         if (true)
2694                 blk_queue_virt_boundary(sdev->request_queue,
2695                                         ~srp_dev->mr_page_mask);
2696
2697         return 0;
2698 }
2699
2700 static int srp_slave_configure(struct scsi_device *sdev)
2701 {
2702         struct Scsi_Host *shost = sdev->host;
2703         struct srp_target_port *target = host_to_target(shost);
2704         struct request_queue *q = sdev->request_queue;
2705         unsigned long timeout;
2706
2707         if (sdev->type == TYPE_DISK) {
2708                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2709                 blk_queue_rq_timeout(q, timeout);
2710         }
2711
2712         return 0;
2713 }
2714
2715 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2716                            char *buf)
2717 {
2718         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2719
2720         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2721 }
2722
2723 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2724                              char *buf)
2725 {
2726         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2727
2728         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2729 }
2730
2731 static ssize_t show_service_id(struct device *dev,
2732                                struct device_attribute *attr, char *buf)
2733 {
2734         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2735
2736         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2737 }
2738
2739 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2740                          char *buf)
2741 {
2742         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2743
2744         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2745 }
2746
2747 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2748                          char *buf)
2749 {
2750         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2751
2752         return sprintf(buf, "%pI6\n", target->sgid.raw);
2753 }
2754
2755 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2756                          char *buf)
2757 {
2758         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2759         struct srp_rdma_ch *ch = &target->ch[0];
2760
2761         return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2762 }
2763
2764 static ssize_t show_orig_dgid(struct device *dev,
2765                               struct device_attribute *attr, char *buf)
2766 {
2767         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2768
2769         return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2770 }
2771
2772 static ssize_t show_req_lim(struct device *dev,
2773                             struct device_attribute *attr, char *buf)
2774 {
2775         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2776         struct srp_rdma_ch *ch;
2777         int i, req_lim = INT_MAX;
2778
2779         for (i = 0; i < target->ch_count; i++) {
2780                 ch = &target->ch[i];
2781                 req_lim = min(req_lim, ch->req_lim);
2782         }
2783         return sprintf(buf, "%d\n", req_lim);
2784 }
2785
2786 static ssize_t show_zero_req_lim(struct device *dev,
2787                                  struct device_attribute *attr, char *buf)
2788 {
2789         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2790
2791         return sprintf(buf, "%d\n", target->zero_req_lim);
2792 }
2793
2794 static ssize_t show_local_ib_port(struct device *dev,
2795                                   struct device_attribute *attr, char *buf)
2796 {
2797         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2798
2799         return sprintf(buf, "%d\n", target->srp_host->port);
2800 }
2801
2802 static ssize_t show_local_ib_device(struct device *dev,
2803                                     struct device_attribute *attr, char *buf)
2804 {
2805         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2806
2807         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2808 }
2809
2810 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2811                              char *buf)
2812 {
2813         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2814
2815         return sprintf(buf, "%d\n", target->ch_count);
2816 }
2817
2818 static ssize_t show_comp_vector(struct device *dev,
2819                                 struct device_attribute *attr, char *buf)
2820 {
2821         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2822
2823         return sprintf(buf, "%d\n", target->comp_vector);
2824 }
2825
2826 static ssize_t show_tl_retry_count(struct device *dev,
2827                                    struct device_attribute *attr, char *buf)
2828 {
2829         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2830
2831         return sprintf(buf, "%d\n", target->tl_retry_count);
2832 }
2833
2834 static ssize_t show_cmd_sg_entries(struct device *dev,
2835                                    struct device_attribute *attr, char *buf)
2836 {
2837         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2838
2839         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2840 }
2841
2842 static ssize_t show_allow_ext_sg(struct device *dev,
2843                                  struct device_attribute *attr, char *buf)
2844 {
2845         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2846
2847         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2848 }
2849
2850 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2851 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2852 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2853 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2854 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2855 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2856 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2857 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2858 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2859 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2860 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2861 static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
2862 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2863 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2864 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2865 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2866
2867 static struct device_attribute *srp_host_attrs[] = {
2868         &dev_attr_id_ext,
2869         &dev_attr_ioc_guid,
2870         &dev_attr_service_id,
2871         &dev_attr_pkey,
2872         &dev_attr_sgid,
2873         &dev_attr_dgid,
2874         &dev_attr_orig_dgid,
2875         &dev_attr_req_lim,
2876         &dev_attr_zero_req_lim,
2877         &dev_attr_local_ib_port,
2878         &dev_attr_local_ib_device,
2879         &dev_attr_ch_count,
2880         &dev_attr_comp_vector,
2881         &dev_attr_tl_retry_count,
2882         &dev_attr_cmd_sg_entries,
2883         &dev_attr_allow_ext_sg,
2884         NULL
2885 };
2886
2887 static struct scsi_host_template srp_template = {
2888         .module                         = THIS_MODULE,
2889         .name                           = "InfiniBand SRP initiator",
2890         .proc_name                      = DRV_NAME,
2891         .slave_alloc                    = srp_slave_alloc,
2892         .slave_configure                = srp_slave_configure,
2893         .info                           = srp_target_info,
2894         .queuecommand                   = srp_queuecommand,
2895         .change_queue_depth             = srp_change_queue_depth,
2896         .eh_timed_out                   = srp_timed_out,
2897         .eh_abort_handler               = srp_abort,
2898         .eh_device_reset_handler        = srp_reset_device,
2899         .eh_host_reset_handler          = srp_reset_host,
2900         .skip_settle_delay              = true,
2901         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2902         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2903         .this_id                        = -1,
2904         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2905         .use_clustering                 = ENABLE_CLUSTERING,
2906         .shost_attrs                    = srp_host_attrs,
2907         .track_queue_depth              = 1,
2908 };
2909
2910 static int srp_sdev_count(struct Scsi_Host *host)
2911 {
2912         struct scsi_device *sdev;
2913         int c = 0;
2914
2915         shost_for_each_device(sdev, host)
2916                 c++;
2917
2918         return c;
2919 }
2920
2921 /*
2922  * Return values:
2923  * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2924  * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2925  *    removal has been scheduled.
2926  * 0 and target->state != SRP_TARGET_REMOVED upon success.
2927  */
2928 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2929 {
2930         struct srp_rport_identifiers ids;
2931         struct srp_rport *rport;
2932
2933         target->state = SRP_TARGET_SCANNING;
2934         sprintf(target->target_name, "SRP.T10:%016llX",
2935                 be64_to_cpu(target->id_ext));
2936
2937         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent))
2938                 return -ENODEV;
2939
2940         memcpy(ids.port_id, &target->id_ext, 8);
2941         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2942         ids.roles = SRP_RPORT_ROLE_TARGET;
2943         rport = srp_rport_add(target->scsi_host, &ids);
2944         if (IS_ERR(rport)) {
2945                 scsi_remove_host(target->scsi_host);
2946                 return PTR_ERR(rport);
2947         }
2948
2949         rport->lld_data = target;
2950         target->rport = rport;
2951
2952         spin_lock(&host->target_lock);
2953         list_add_tail(&target->list, &host->target_list);
2954         spin_unlock(&host->target_lock);
2955
2956         scsi_scan_target(&target->scsi_host->shost_gendev,
2957                          0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
2958
2959         if (srp_connected_ch(target) < target->ch_count ||
2960             target->qp_in_error) {
2961                 shost_printk(KERN_INFO, target->scsi_host,
2962                              PFX "SCSI scan failed - removing SCSI host\n");
2963                 srp_queue_remove_work(target);
2964                 goto out;
2965         }
2966
2967         pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
2968                  dev_name(&target->scsi_host->shost_gendev),
2969                  srp_sdev_count(target->scsi_host));
2970
2971         spin_lock_irq(&target->lock);
2972         if (target->state == SRP_TARGET_SCANNING)
2973                 target->state = SRP_TARGET_LIVE;
2974         spin_unlock_irq(&target->lock);
2975
2976 out:
2977         return 0;
2978 }
2979
2980 static void srp_release_dev(struct device *dev)
2981 {
2982         struct srp_host *host =
2983                 container_of(dev, struct srp_host, dev);
2984
2985         complete(&host->released);
2986 }
2987
2988 static struct class srp_class = {
2989         .name    = "infiniband_srp",
2990         .dev_release = srp_release_dev
2991 };
2992
2993 /**
2994  * srp_conn_unique() - check whether the connection to a target is unique
2995  * @host:   SRP host.
2996  * @target: SRP target port.
2997  */
2998 static bool srp_conn_unique(struct srp_host *host,
2999                             struct srp_target_port *target)
3000 {
3001         struct srp_target_port *t;
3002         bool ret = false;
3003
3004         if (target->state == SRP_TARGET_REMOVED)
3005                 goto out;
3006
3007         ret = true;
3008
3009         spin_lock(&host->target_lock);
3010         list_for_each_entry(t, &host->target_list, list) {
3011                 if (t != target &&
3012                     target->id_ext == t->id_ext &&
3013                     target->ioc_guid == t->ioc_guid &&
3014                     target->initiator_ext == t->initiator_ext) {
3015                         ret = false;
3016                         break;
3017                 }
3018         }
3019         spin_unlock(&host->target_lock);
3020
3021 out:
3022         return ret;
3023 }
3024
3025 /*
3026  * Target ports are added by writing
3027  *
3028  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
3029  *     pkey=<P_Key>,service_id=<service ID>
3030  *
3031  * to the add_target sysfs attribute.
3032  */
3033 enum {
3034         SRP_OPT_ERR             = 0,
3035         SRP_OPT_ID_EXT          = 1 << 0,
3036         SRP_OPT_IOC_GUID        = 1 << 1,
3037         SRP_OPT_DGID            = 1 << 2,
3038         SRP_OPT_PKEY            = 1 << 3,
3039         SRP_OPT_SERVICE_ID      = 1 << 4,
3040         SRP_OPT_MAX_SECT        = 1 << 5,
3041         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
3042         SRP_OPT_IO_CLASS        = 1 << 7,
3043         SRP_OPT_INITIATOR_EXT   = 1 << 8,
3044         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
3045         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
3046         SRP_OPT_SG_TABLESIZE    = 1 << 11,
3047         SRP_OPT_COMP_VECTOR     = 1 << 12,
3048         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
3049         SRP_OPT_QUEUE_SIZE      = 1 << 14,
3050         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
3051                                    SRP_OPT_IOC_GUID     |
3052                                    SRP_OPT_DGID         |
3053                                    SRP_OPT_PKEY         |
3054                                    SRP_OPT_SERVICE_ID),
3055 };
3056
3057 static const match_table_t srp_opt_tokens = {
3058         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
3059         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
3060         { SRP_OPT_DGID,                 "dgid=%s"               },
3061         { SRP_OPT_PKEY,                 "pkey=%x"               },
3062         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
3063         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
3064         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
3065         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
3066         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
3067         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
3068         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
3069         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
3070         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
3071         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
3072         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
3073         { SRP_OPT_ERR,                  NULL                    }
3074 };
3075
3076 static int srp_parse_options(const char *buf, struct srp_target_port *target)
3077 {
3078         char *options, *sep_opt;
3079         char *p;
3080         char dgid[3];
3081         substring_t args[MAX_OPT_ARGS];
3082         int opt_mask = 0;
3083         int token;
3084         int ret = -EINVAL;
3085         int i;
3086
3087         options = kstrdup(buf, GFP_KERNEL);
3088         if (!options)
3089                 return -ENOMEM;
3090
3091         sep_opt = options;
3092         while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3093                 if (!*p)
3094                         continue;
3095
3096                 token = match_token(p, srp_opt_tokens, args);
3097                 opt_mask |= token;
3098
3099                 switch (token) {
3100                 case SRP_OPT_ID_EXT:
3101                         p = match_strdup(args);
3102                         if (!p) {
3103                                 ret = -ENOMEM;
3104                                 goto out;
3105                         }
3106                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3107                         kfree(p);
3108                         break;
3109
3110                 case SRP_OPT_IOC_GUID:
3111                         p = match_strdup(args);
3112                         if (!p) {
3113                                 ret = -ENOMEM;
3114                                 goto out;
3115                         }
3116                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
3117                         kfree(p);
3118                         break;
3119
3120                 case SRP_OPT_DGID:
3121                         p = match_strdup(args);
3122                         if (!p) {
3123                                 ret = -ENOMEM;
3124                                 goto out;
3125                         }
3126                         if (strlen(p) != 32) {
3127                                 pr_warn("bad dest GID parameter '%s'\n", p);
3128                                 kfree(p);
3129                                 goto out;
3130                         }
3131
3132                         for (i = 0; i < 16; ++i) {
3133                                 strlcpy(dgid, p + i * 2, sizeof(dgid));
3134                                 if (sscanf(dgid, "%hhx",
3135                                            &target->orig_dgid.raw[i]) < 1) {
3136                                         ret = -EINVAL;
3137                                         kfree(p);
3138                                         goto out;
3139                                 }
3140                         }
3141                         kfree(p);
3142                         break;
3143
3144                 case SRP_OPT_PKEY:
3145                         if (match_hex(args, &token)) {
3146                                 pr_warn("bad P_Key parameter '%s'\n", p);
3147                                 goto out;
3148                         }
3149                         target->pkey = cpu_to_be16(token);
3150                         break;
3151
3152                 case SRP_OPT_SERVICE_ID:
3153                         p = match_strdup(args);
3154                         if (!p) {
3155                                 ret = -ENOMEM;
3156                                 goto out;
3157                         }
3158                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3159                         kfree(p);
3160                         break;
3161
3162                 case SRP_OPT_MAX_SECT:
3163                         if (match_int(args, &token)) {
3164                                 pr_warn("bad max sect parameter '%s'\n", p);
3165                                 goto out;
3166                         }
3167                         target->scsi_host->max_sectors = token;
3168                         break;
3169
3170                 case SRP_OPT_QUEUE_SIZE:
3171                         if (match_int(args, &token) || token < 1) {
3172                                 pr_warn("bad queue_size parameter '%s'\n", p);
3173                                 goto out;
3174                         }
3175                         target->scsi_host->can_queue = token;
3176                         target->queue_size = token + SRP_RSP_SQ_SIZE +
3177                                              SRP_TSK_MGMT_SQ_SIZE;
3178                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3179                                 target->scsi_host->cmd_per_lun = token;
3180                         break;
3181
3182                 case SRP_OPT_MAX_CMD_PER_LUN:
3183                         if (match_int(args, &token) || token < 1) {
3184                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3185                                         p);
3186                                 goto out;
3187                         }
3188                         target->scsi_host->cmd_per_lun = token;
3189                         break;
3190
3191                 case SRP_OPT_IO_CLASS:
3192                         if (match_hex(args, &token)) {
3193                                 pr_warn("bad IO class parameter '%s'\n", p);
3194                                 goto out;
3195                         }
3196                         if (token != SRP_REV10_IB_IO_CLASS &&
3197                             token != SRP_REV16A_IB_IO_CLASS) {
3198                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3199                                         token, SRP_REV10_IB_IO_CLASS,
3200                                         SRP_REV16A_IB_IO_CLASS);
3201                                 goto out;
3202                         }
3203                         target->io_class = token;
3204                         break;
3205
3206                 case SRP_OPT_INITIATOR_EXT:
3207                         p = match_strdup(args);
3208                         if (!p) {
3209                                 ret = -ENOMEM;
3210                                 goto out;
3211                         }
3212                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3213                         kfree(p);
3214                         break;
3215
3216                 case SRP_OPT_CMD_SG_ENTRIES:
3217                         if (match_int(args, &token) || token < 1 || token > 255) {
3218                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3219                                         p);
3220                                 goto out;
3221                         }
3222                         target->cmd_sg_cnt = token;
3223                         break;
3224
3225                 case SRP_OPT_ALLOW_EXT_SG:
3226                         if (match_int(args, &token)) {
3227                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3228                                 goto out;
3229                         }
3230                         target->allow_ext_sg = !!token;
3231                         break;
3232
3233                 case SRP_OPT_SG_TABLESIZE:
3234                         if (match_int(args, &token) || token < 1 ||
3235                                         token > SG_MAX_SEGMENTS) {
3236                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
3237                                         p);
3238                                 goto out;
3239                         }
3240                         target->sg_tablesize = token;
3241                         break;
3242
3243                 case SRP_OPT_COMP_VECTOR:
3244                         if (match_int(args, &token) || token < 0) {
3245                                 pr_warn("bad comp_vector parameter '%s'\n", p);
3246                                 goto out;
3247                         }
3248                         target->comp_vector = token;
3249                         break;
3250
3251                 case SRP_OPT_TL_RETRY_COUNT:
3252                         if (match_int(args, &token) || token < 2 || token > 7) {
3253                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3254                                         p);
3255                                 goto out;
3256                         }
3257                         target->tl_retry_count = token;
3258                         break;
3259
3260                 default:
3261                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3262                                 p);
3263                         goto out;
3264                 }
3265         }
3266
3267         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3268                 ret = 0;
3269         else
3270                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3271                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3272                             !(srp_opt_tokens[i].token & opt_mask))
3273                                 pr_warn("target creation request is missing parameter '%s'\n",
3274                                         srp_opt_tokens[i].pattern);
3275
3276         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3277             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3278                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3279                         target->scsi_host->cmd_per_lun,
3280                         target->scsi_host->can_queue);
3281
3282 out:
3283         kfree(options);
3284         return ret;
3285 }
3286
3287 static ssize_t srp_create_target(struct device *dev,
3288                                  struct device_attribute *attr,
3289                                  const char *buf, size_t count)
3290 {
3291         struct srp_host *host =
3292                 container_of(dev, struct srp_host, dev);
3293         struct Scsi_Host *target_host;
3294         struct srp_target_port *target;
3295         struct srp_rdma_ch *ch;
3296         struct srp_device *srp_dev = host->srp_dev;
3297         struct ib_device *ibdev = srp_dev->dev;
3298         int ret, node_idx, node, cpu, i;
3299         unsigned int max_sectors_per_mr, mr_per_cmd = 0;
3300         bool multich = false;
3301
3302         target_host = scsi_host_alloc(&srp_template,
3303                                       sizeof (struct srp_target_port));
3304         if (!target_host)
3305                 return -ENOMEM;
3306
3307         target_host->transportt  = ib_srp_transport_template;
3308         target_host->max_channel = 0;
3309         target_host->max_id      = 1;
3310         target_host->max_lun     = -1LL;
3311         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3312
3313         target = host_to_target(target_host);
3314
3315         target->io_class        = SRP_REV16A_IB_IO_CLASS;
3316         target->scsi_host       = target_host;
3317         target->srp_host        = host;
3318         target->pd              = host->srp_dev->pd;
3319         target->lkey            = host->srp_dev->pd->local_dma_lkey;
3320         target->cmd_sg_cnt      = cmd_sg_entries;
3321         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
3322         target->allow_ext_sg    = allow_ext_sg;
3323         target->tl_retry_count  = 7;
3324         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
3325
3326         /*
3327          * Avoid that the SCSI host can be removed by srp_remove_target()
3328          * before this function returns.
3329          */
3330         scsi_host_get(target->scsi_host);
3331
3332         ret = mutex_lock_interruptible(&host->add_target_mutex);
3333         if (ret < 0)
3334                 goto put;
3335
3336         ret = srp_parse_options(buf, target);
3337         if (ret)
3338                 goto out;
3339
3340         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3341
3342         if (!srp_conn_unique(target->srp_host, target)) {
3343                 shost_printk(KERN_INFO, target->scsi_host,
3344                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3345                              be64_to_cpu(target->id_ext),
3346                              be64_to_cpu(target->ioc_guid),
3347                              be64_to_cpu(target->initiator_ext));
3348                 ret = -EEXIST;
3349                 goto out;
3350         }
3351
3352         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3353             target->cmd_sg_cnt < target->sg_tablesize) {
3354                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3355                 target->sg_tablesize = target->cmd_sg_cnt;
3356         }
3357
3358         if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
3359                 /*
3360                  * FR and FMR can only map one HCA page per entry. If the
3361                  * start address is not aligned on a HCA page boundary two
3362                  * entries will be used for the head and the tail although
3363                  * these two entries combined contain at most one HCA page of
3364                  * data. Hence the "+ 1" in the calculation below.
3365                  *
3366                  * The indirect data buffer descriptor is contiguous so the
3367                  * memory for that buffer will only be registered if
3368                  * register_always is true. Hence add one to mr_per_cmd if
3369                  * register_always has been set.
3370                  */
3371                 max_sectors_per_mr = srp_dev->max_pages_per_mr <<
3372                                   (ilog2(srp_dev->mr_page_size) - 9);
3373                 mr_per_cmd = register_always +
3374                         (target->scsi_host->max_sectors + 1 +
3375                          max_sectors_per_mr - 1) / max_sectors_per_mr;
3376                 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3377                          target->scsi_host->max_sectors,
3378                          srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3379                          max_sectors_per_mr, mr_per_cmd);
3380         }
3381
3382         target_host->sg_tablesize = target->sg_tablesize;
3383         target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
3384         target->mr_per_cmd = mr_per_cmd;
3385         target->indirect_size = target->sg_tablesize *
3386                                 sizeof (struct srp_direct_buf);
3387         target->max_iu_len = sizeof (struct srp_cmd) +
3388                              sizeof (struct srp_indirect_buf) +
3389                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3390
3391         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3392         INIT_WORK(&target->remove_work, srp_remove_work);
3393         spin_lock_init(&target->lock);
3394         ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3395         if (ret)
3396                 goto out;
3397
3398         ret = -ENOMEM;
3399         target->ch_count = max_t(unsigned, num_online_nodes(),
3400                                  min(ch_count ? :
3401                                      min(4 * num_online_nodes(),
3402                                          ibdev->num_comp_vectors),
3403                                      num_online_cpus()));
3404         target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3405                              GFP_KERNEL);
3406         if (!target->ch)
3407                 goto out;
3408
3409         node_idx = 0;
3410         for_each_online_node(node) {
3411                 const int ch_start = (node_idx * target->ch_count /
3412                                       num_online_nodes());
3413                 const int ch_end = ((node_idx + 1) * target->ch_count /
3414                                     num_online_nodes());
3415                 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3416                                       num_online_nodes() + target->comp_vector)
3417                                      % ibdev->num_comp_vectors;
3418                 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3419                                     num_online_nodes() + target->comp_vector)
3420                                    % ibdev->num_comp_vectors;
3421                 int cpu_idx = 0;
3422
3423                 for_each_online_cpu(cpu) {
3424                         if (cpu_to_node(cpu) != node)
3425                                 continue;
3426                         if (ch_start + cpu_idx >= ch_end)
3427                                 continue;
3428                         ch = &target->ch[ch_start + cpu_idx];
3429                         ch->target = target;
3430                         ch->comp_vector = cv_start == cv_end ? cv_start :
3431                                 cv_start + cpu_idx % (cv_end - cv_start);
3432                         spin_lock_init(&ch->lock);
3433                         INIT_LIST_HEAD(&ch->free_tx);
3434                         ret = srp_new_cm_id(ch);
3435                         if (ret)
3436                                 goto err_disconnect;
3437
3438                         ret = srp_create_ch_ib(ch);
3439                         if (ret)
3440                                 goto err_disconnect;
3441
3442                         ret = srp_alloc_req_data(ch);
3443                         if (ret)
3444                                 goto err_disconnect;
3445
3446                         ret = srp_connect_ch(ch, multich);
3447                         if (ret) {
3448                                 shost_printk(KERN_ERR, target->scsi_host,
3449                                              PFX "Connection %d/%d to %pI6 failed\n",
3450                                              ch_start + cpu_idx,
3451                                              target->ch_count,
3452                                              ch->target->orig_dgid.raw);
3453                                 if (node_idx == 0 && cpu_idx == 0) {
3454                                         goto free_ch;
3455                                 } else {
3456                                         srp_free_ch_ib(target, ch);
3457                                         srp_free_req_data(target, ch);
3458                                         target->ch_count = ch - target->ch;
3459                                         goto connected;
3460                                 }
3461                         }
3462
3463                         multich = true;
3464                         cpu_idx++;
3465                 }
3466                 node_idx++;
3467         }
3468
3469 connected:
3470         target->scsi_host->nr_hw_queues = target->ch_count;
3471
3472         ret = srp_add_target(host, target);
3473         if (ret)
3474                 goto err_disconnect;
3475
3476         if (target->state != SRP_TARGET_REMOVED) {
3477                 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3478                              "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3479                              be64_to_cpu(target->id_ext),
3480                              be64_to_cpu(target->ioc_guid),
3481                              be16_to_cpu(target->pkey),
3482                              be64_to_cpu(target->service_id),
3483                              target->sgid.raw, target->orig_dgid.raw);
3484         }
3485
3486         ret = count;
3487
3488 out:
3489         mutex_unlock(&host->add_target_mutex);
3490
3491 put:
3492         scsi_host_put(target->scsi_host);
3493         if (ret < 0)
3494                 scsi_host_put(target->scsi_host);
3495
3496         return ret;
3497
3498 err_disconnect:
3499         srp_disconnect_target(target);
3500
3501 free_ch:
3502         for (i = 0; i < target->ch_count; i++) {
3503                 ch = &target->ch[i];
3504                 srp_free_ch_ib(target, ch);
3505                 srp_free_req_data(target, ch);
3506         }
3507
3508         kfree(target->ch);
3509         goto out;
3510 }
3511
3512 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3513
3514 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3515                           char *buf)
3516 {
3517         struct srp_host *host = container_of(dev, struct srp_host, dev);
3518
3519         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3520 }
3521
3522 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3523
3524 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3525                          char *buf)
3526 {
3527         struct srp_host *host = container_of(dev, struct srp_host, dev);
3528
3529         return sprintf(buf, "%d\n", host->port);
3530 }
3531
3532 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3533
3534 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3535 {
3536         struct srp_host *host;
3537
3538         host = kzalloc(sizeof *host, GFP_KERNEL);
3539         if (!host)
3540                 return NULL;
3541
3542         INIT_LIST_HEAD(&host->target_list);
3543         spin_lock_init(&host->target_lock);
3544         init_completion(&host->released);
3545         mutex_init(&host->add_target_mutex);
3546         host->srp_dev = device;
3547         host->port = port;
3548
3549         host->dev.class = &srp_class;
3550         host->dev.parent = device->dev->dev.parent;
3551         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3552
3553         if (device_register(&host->dev))
3554                 goto free_host;
3555         if (device_create_file(&host->dev, &dev_attr_add_target))
3556                 goto err_class;
3557         if (device_create_file(&host->dev, &dev_attr_ibdev))
3558                 goto err_class;
3559         if (device_create_file(&host->dev, &dev_attr_port))
3560                 goto err_class;
3561
3562         return host;
3563
3564 err_class:
3565         device_unregister(&host->dev);
3566
3567 free_host:
3568         kfree(host);
3569
3570         return NULL;
3571 }
3572
3573 static void srp_add_one(struct ib_device *device)
3574 {
3575         struct srp_device *srp_dev;
3576         struct ib_device_attr *attr = &device->attrs;
3577         struct srp_host *host;
3578         int mr_page_shift, p;
3579         u64 max_pages_per_mr;
3580         unsigned int flags = 0;
3581
3582         srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
3583         if (!srp_dev)
3584                 return;
3585
3586         /*
3587          * Use the smallest page size supported by the HCA, down to a
3588          * minimum of 4096 bytes. We're unlikely to build large sglists
3589          * out of smaller entries.
3590          */
3591         mr_page_shift           = max(12, ffs(attr->page_size_cap) - 1);
3592         srp_dev->mr_page_size   = 1 << mr_page_shift;
3593         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
3594         max_pages_per_mr        = attr->max_mr_size;
3595         do_div(max_pages_per_mr, srp_dev->mr_page_size);
3596         pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
3597                  attr->max_mr_size, srp_dev->mr_page_size,
3598                  max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
3599         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3600                                           max_pages_per_mr);
3601
3602         srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3603                             device->map_phys_fmr && device->unmap_fmr);
3604         srp_dev->has_fr = (attr->device_cap_flags &
3605                            IB_DEVICE_MEM_MGT_EXTENSIONS);
3606         if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
3607                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3608         } else if (!never_register &&
3609                    attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
3610                 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3611                                          (!srp_dev->has_fmr || prefer_fr));
3612                 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3613         }
3614
3615         if (never_register || !register_always ||
3616             (!srp_dev->has_fmr && !srp_dev->has_fr))
3617                 flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
3618
3619         if (srp_dev->use_fast_reg) {
3620                 srp_dev->max_pages_per_mr =
3621                         min_t(u32, srp_dev->max_pages_per_mr,
3622                               attr->max_fast_reg_page_list_len);
3623         }
3624         srp_dev->mr_max_size    = srp_dev->mr_page_size *
3625                                    srp_dev->max_pages_per_mr;
3626         pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3627                  device->name, mr_page_shift, attr->max_mr_size,
3628                  attr->max_fast_reg_page_list_len,
3629                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3630
3631         INIT_LIST_HEAD(&srp_dev->dev_list);
3632
3633         srp_dev->dev = device;
3634         srp_dev->pd  = ib_alloc_pd(device, flags);
3635         if (IS_ERR(srp_dev->pd))
3636                 goto free_dev;
3637
3638
3639         for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3640                 host = srp_add_port(srp_dev, p);
3641                 if (host)
3642                         list_add_tail(&host->list, &srp_dev->dev_list);
3643         }
3644
3645         ib_set_client_data(device, &srp_client, srp_dev);
3646         return;
3647
3648 free_dev:
3649         kfree(srp_dev);
3650 }
3651
3652 static void srp_remove_one(struct ib_device *device, void *client_data)
3653 {
3654         struct srp_device *srp_dev;
3655         struct srp_host *host, *tmp_host;
3656         struct srp_target_port *target;
3657
3658         srp_dev = client_data;
3659         if (!srp_dev)
3660                 return;
3661
3662         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3663                 device_unregister(&host->dev);
3664                 /*
3665                  * Wait for the sysfs entry to go away, so that no new
3666                  * target ports can be created.
3667                  */
3668                 wait_for_completion(&host->released);
3669
3670                 /*
3671                  * Remove all target ports.
3672                  */
3673                 spin_lock(&host->target_lock);
3674                 list_for_each_entry(target, &host->target_list, list)
3675                         srp_queue_remove_work(target);
3676                 spin_unlock(&host->target_lock);
3677
3678                 /*
3679                  * Wait for tl_err and target port removal tasks.
3680                  */
3681                 flush_workqueue(system_long_wq);
3682                 flush_workqueue(srp_remove_wq);
3683
3684                 kfree(host);
3685         }
3686
3687         ib_dealloc_pd(srp_dev->pd);
3688
3689         kfree(srp_dev);
3690 }
3691
3692 static struct srp_function_template ib_srp_transport_functions = {
3693         .has_rport_state         = true,
3694         .reset_timer_if_blocked  = true,
3695         .reconnect_delay         = &srp_reconnect_delay,
3696         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
3697         .dev_loss_tmo            = &srp_dev_loss_tmo,
3698         .reconnect               = srp_rport_reconnect,
3699         .rport_delete            = srp_rport_delete,
3700         .terminate_rport_io      = srp_terminate_io,
3701 };
3702
3703 static int __init srp_init_module(void)
3704 {
3705         int ret;
3706
3707         if (srp_sg_tablesize) {
3708                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3709                 if (!cmd_sg_entries)
3710                         cmd_sg_entries = srp_sg_tablesize;
3711         }
3712
3713         if (!cmd_sg_entries)
3714                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3715
3716         if (cmd_sg_entries > 255) {
3717                 pr_warn("Clamping cmd_sg_entries to 255\n");
3718                 cmd_sg_entries = 255;
3719         }
3720
3721         if (!indirect_sg_entries)
3722                 indirect_sg_entries = cmd_sg_entries;
3723         else if (indirect_sg_entries < cmd_sg_entries) {
3724                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3725                         cmd_sg_entries);
3726                 indirect_sg_entries = cmd_sg_entries;
3727         }
3728
3729         if (indirect_sg_entries > SG_MAX_SEGMENTS) {
3730                 pr_warn("Clamping indirect_sg_entries to %u\n",
3731                         SG_MAX_SEGMENTS);
3732                 indirect_sg_entries = SG_MAX_SEGMENTS;
3733         }
3734
3735         srp_remove_wq = create_workqueue("srp_remove");
3736         if (!srp_remove_wq) {
3737                 ret = -ENOMEM;
3738                 goto out;
3739         }
3740
3741         ret = -ENOMEM;
3742         ib_srp_transport_template =
3743                 srp_attach_transport(&ib_srp_transport_functions);
3744         if (!ib_srp_transport_template)
3745                 goto destroy_wq;
3746
3747         ret = class_register(&srp_class);
3748         if (ret) {
3749                 pr_err("couldn't register class infiniband_srp\n");
3750                 goto release_tr;
3751         }
3752
3753         ib_sa_register_client(&srp_sa_client);
3754
3755         ret = ib_register_client(&srp_client);
3756         if (ret) {
3757                 pr_err("couldn't register IB client\n");
3758                 goto unreg_sa;
3759         }
3760
3761 out:
3762         return ret;
3763
3764 unreg_sa:
3765         ib_sa_unregister_client(&srp_sa_client);
3766         class_unregister(&srp_class);
3767
3768 release_tr:
3769         srp_release_transport(ib_srp_transport_template);
3770
3771 destroy_wq:
3772         destroy_workqueue(srp_remove_wq);
3773         goto out;
3774 }
3775
3776 static void __exit srp_cleanup_module(void)
3777 {
3778         ib_unregister_client(&srp_client);
3779         ib_sa_unregister_client(&srp_sa_client);
3780         class_unregister(&srp_class);
3781         srp_release_transport(ib_srp_transport_template);
3782         destroy_workqueue(srp_remove_wq);
3783 }
3784
3785 module_init(srp_init_module);
3786 module_exit(srp_cleanup_module);