drivers/target/target_core_transport.c

   1 /*******************************************************************************
   2  * Filename:  target_core_transport.c
   3  *
   4  * This file contains the Generic Target Engine Core.
   5  *
   6  * (c) Copyright 2002-2013 Datera, Inc.
   7  *
   8  * Nicholas A. Bellinger <nab@kernel.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  23  *
  24  ******************************************************************************/
  25
  26 #include <linux/net.h>
  27 #include <linux/delay.h>
  28 #include <linux/string.h>
  29 #include <linux/timer.h>
  30 #include <linux/slab.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/kthread.h>
  33 #include <linux/in.h>
  34 #include <linux/cdrom.h>
  35 #include <linux/module.h>
  36 #include <linux/ratelimit.h>
  37 #include <linux/vmalloc.h>
  38 #include <asm/unaligned.h>
  39 #include <net/sock.h>
  40 #include <net/tcp.h>
  41 #include <scsi/scsi_proto.h>
  42 #include <scsi/scsi_common.h>
  43
  44 #include <target/target_core_base.h>
  45 #include <target/target_core_backend.h>
  46 #include <target/target_core_fabric.h>
  47
  48 #include "target_core_internal.h"
  49 #include "target_core_alua.h"
  50 #include "target_core_pr.h"
  51 #include "target_core_ua.h"
  52
  53 #define CREATE_TRACE_POINTS
  54 #include <trace/events/target.h>
  55
  56 static struct workqueue_struct *target_completion_wq;
  57 static struct kmem_cache *se_sess_cache;
  58 struct kmem_cache *se_ua_cache;
  59 struct kmem_cache *t10_pr_reg_cache;
  60 struct kmem_cache *t10_alua_lu_gp_cache;
  61 struct kmem_cache *t10_alua_lu_gp_mem_cache;
  62 struct kmem_cache *t10_alua_tg_pt_gp_cache;
  63 struct kmem_cache *t10_alua_lba_map_cache;
  64 struct kmem_cache *t10_alua_lba_map_mem_cache;
  65
  66 static void transport_complete_task_attr(struct se_cmd *cmd);
  67 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason);
  68 static void transport_handle_queue_full(struct se_cmd *cmd,
  69                 struct se_device *dev, int err, bool write_pending);
  70 static int transport_put_cmd(struct se_cmd *cmd);
  71 static void target_complete_ok_work(struct work_struct *work);
  72
  73 int init_se_kmem_caches(void)
  74 {
  75         se_sess_cache = kmem_cache_create("se_sess_cache",
  76                         sizeof(struct se_session), __alignof__(struct se_session),
  77                         0, NULL);
  78         if (!se_sess_cache) {
  79                 pr_err("kmem_cache_create() for struct se_session"
  80                                 " failed\n");
  81                 goto out;
  82         }
  83         se_ua_cache = kmem_cache_create("se_ua_cache",
  84                         sizeof(struct se_ua), __alignof__(struct se_ua),
  85                         0, NULL);
  86         if (!se_ua_cache) {
  87                 pr_err("kmem_cache_create() for struct se_ua failed\n");
  88                 goto out_free_sess_cache;
  89         }
  90         t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
  91                         sizeof(struct t10_pr_registration),
  92                         __alignof__(struct t10_pr_registration), 0, NULL);
  93         if (!t10_pr_reg_cache) {
  94                 pr_err("kmem_cache_create() for struct t10_pr_registration"
  95                                 " failed\n");
  96                 goto out_free_ua_cache;
  97         }
  98         t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
  99                         sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
 100                         0, NULL);
 101         if (!t10_alua_lu_gp_cache) {
 102                 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
 103                                 " failed\n");
 104                 goto out_free_pr_reg_cache;
 105         }
 106         t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
 107                         sizeof(struct t10_alua_lu_gp_member),
 108                         __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
 109         if (!t10_alua_lu_gp_mem_cache) {
 110                 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
 111                                 "cache failed\n");
 112                 goto out_free_lu_gp_cache;
 113         }
 114         t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
 115                         sizeof(struct t10_alua_tg_pt_gp),
 116                         __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
 117         if (!t10_alua_tg_pt_gp_cache) {
 118                 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
 119                                 "cache failed\n");
 120                 goto out_free_lu_gp_mem_cache;
 121         }
 122         t10_alua_lba_map_cache = kmem_cache_create(
 123                         "t10_alua_lba_map_cache",
 124                         sizeof(struct t10_alua_lba_map),
 125                         __alignof__(struct t10_alua_lba_map), 0, NULL);
 126         if (!t10_alua_lba_map_cache) {
 127                 pr_err("kmem_cache_create() for t10_alua_lba_map_"
 128                                 "cache failed\n");
 129                 goto out_free_tg_pt_gp_cache;
 130         }
 131         t10_alua_lba_map_mem_cache = kmem_cache_create(
 132                         "t10_alua_lba_map_mem_cache",
 133                         sizeof(struct t10_alua_lba_map_member),
 134                         __alignof__(struct t10_alua_lba_map_member), 0, NULL);
 135         if (!t10_alua_lba_map_mem_cache) {
 136                 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_"
 137                                 "cache failed\n");
 138                 goto out_free_lba_map_cache;
 139         }
 140
 141         target_completion_wq = alloc_workqueue("target_completion",
 142                                                WQ_MEM_RECLAIM, 0);
 143         if (!target_completion_wq)
 144                 goto out_free_lba_map_mem_cache;
 145
 146         return 0;
 147
 148 out_free_lba_map_mem_cache:
 149         kmem_cache_destroy(t10_alua_lba_map_mem_cache);
 150 out_free_lba_map_cache:
 151         kmem_cache_destroy(t10_alua_lba_map_cache);
 152 out_free_tg_pt_gp_cache:
 153         kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
 154 out_free_lu_gp_mem_cache:
 155         kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
 156 out_free_lu_gp_cache:
 157         kmem_cache_destroy(t10_alua_lu_gp_cache);
 158 out_free_pr_reg_cache:
 159         kmem_cache_destroy(t10_pr_reg_cache);
 160 out_free_ua_cache:
 161         kmem_cache_destroy(se_ua_cache);
 162 out_free_sess_cache:
 163         kmem_cache_destroy(se_sess_cache);
 164 out:
 165         return -ENOMEM;
 166 }
 167
 168 void release_se_kmem_caches(void)
 169 {
 170         destroy_workqueue(target_completion_wq);
 171         kmem_cache_destroy(se_sess_cache);
 172         kmem_cache_destroy(se_ua_cache);
 173         kmem_cache_destroy(t10_pr_reg_cache);
 174         kmem_cache_destroy(t10_alua_lu_gp_cache);
 175         kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
 176         kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
 177         kmem_cache_destroy(t10_alua_lba_map_cache);
 178         kmem_cache_destroy(t10_alua_lba_map_mem_cache);
 179 }
 180
 181 /* This code ensures unique mib indexes are handed out. */
 182 static DEFINE_SPINLOCK(scsi_mib_index_lock);
 183 static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
 184
 185 /*
 186  * Allocate a new row index for the entry type specified
 187  */
 188 u32 scsi_get_new_index(scsi_index_t type)
 189 {
 190         u32 new_index;
 191
 192         BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
 193
 194         spin_lock(&scsi_mib_index_lock);
 195         new_index = ++scsi_mib_index[type];
 196         spin_unlock(&scsi_mib_index_lock);
 197
 198         return new_index;
 199 }
 200
 201 void transport_subsystem_check_init(void)
 202 {
 203         int ret;
 204         static int sub_api_initialized;
 205
 206         if (sub_api_initialized)
 207                 return;
 208
 209         ret = request_module("target_core_iblock");
 210         if (ret != 0)
 211                 pr_err("Unable to load target_core_iblock\n");
 212
 213         ret = request_module("target_core_file");
 214         if (ret != 0)
 215                 pr_err("Unable to load target_core_file\n");
 216
 217         ret = request_module("target_core_pscsi");
 218         if (ret != 0)
 219                 pr_err("Unable to load target_core_pscsi\n");
 220
 221         ret = request_module("target_core_user");
 222         if (ret != 0)
 223                 pr_err("Unable to load target_core_user\n");
 224
 225         sub_api_initialized = 1;
 226 }
 227
 228 struct se_session *transport_init_session(enum target_prot_op sup_prot_ops)
 229 {
 230         struct se_session *se_sess;
 231
 232         se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
 233         if (!se_sess) {
 234                 pr_err("Unable to allocate struct se_session from"
 235                                 " se_sess_cache\n");
 236                 return ERR_PTR(-ENOMEM);
 237         }
 238         INIT_LIST_HEAD(&se_sess->sess_list);
 239         INIT_LIST_HEAD(&se_sess->sess_acl_list);
 240         INIT_LIST_HEAD(&se_sess->sess_cmd_list);
 241         INIT_LIST_HEAD(&se_sess->sess_wait_list);
 242         spin_lock_init(&se_sess->sess_cmd_lock);
 243         se_sess->sup_prot_ops = sup_prot_ops;
 244
 245         return se_sess;
 246 }
 247 EXPORT_SYMBOL(transport_init_session);
 248
 249 int transport_alloc_session_tags(struct se_session *se_sess,
 250                                  unsigned int tag_num, unsigned int tag_size)
 251 {
 252         int rc;
 253
 254         se_sess->sess_cmd_map = kzalloc(tag_num * tag_size,
 255                                         GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
 256         if (!se_sess->sess_cmd_map) {
 257                 se_sess->sess_cmd_map = vzalloc(tag_num * tag_size);
 258                 if (!se_sess->sess_cmd_map) {
 259                         pr_err("Unable to allocate se_sess->sess_cmd_map\n");
 260                         return -ENOMEM;
 261                 }
 262         }
 263
 264         rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num);
 265         if (rc < 0) {
 266                 pr_err("Unable to init se_sess->sess_tag_pool,"
 267                         " tag_num: %u\n", tag_num);
 268                 kvfree(se_sess->sess_cmd_map);
 269                 se_sess->sess_cmd_map = NULL;
 270                 return -ENOMEM;
 271         }
 272
 273         return 0;
 274 }
 275 EXPORT_SYMBOL(transport_alloc_session_tags);
 276
 277 struct se_session *transport_init_session_tags(unsigned int tag_num,
 278                                                unsigned int tag_size,
 279                                                enum target_prot_op sup_prot_ops)
 280 {
 281         struct se_session *se_sess;
 282         int rc;
 283
 284         if (tag_num != 0 && !tag_size) {
 285                 pr_err("init_session_tags called with percpu-ida tag_num:"
 286                        " %u, but zero tag_size\n", tag_num);
 287                 return ERR_PTR(-EINVAL);
 288         }
 289         if (!tag_num && tag_size) {
 290                 pr_err("init_session_tags called with percpu-ida tag_size:"
 291                        " %u, but zero tag_num\n", tag_size);
 292                 return ERR_PTR(-EINVAL);
 293         }
 294
 295         se_sess = transport_init_session(sup_prot_ops);
 296         if (IS_ERR(se_sess))
 297                 return se_sess;
 298
 299         rc = transport_alloc_session_tags(se_sess, tag_num, tag_size);
 300         if (rc < 0) {
 301                 transport_free_session(se_sess);
 302                 return ERR_PTR(-ENOMEM);
 303         }
 304
 305         return se_sess;
 306 }
 307 EXPORT_SYMBOL(transport_init_session_tags);
 308
 309 /*
 310  * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called.
 311  */
 312 void __transport_register_session(
 313         struct se_portal_group *se_tpg,
 314         struct se_node_acl *se_nacl,
 315         struct se_session *se_sess,
 316         void *fabric_sess_ptr)
 317 {
 318         const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo;
 319         unsigned char buf[PR_REG_ISID_LEN];
 320
 321         se_sess->se_tpg = se_tpg;
 322         se_sess->fabric_sess_ptr = fabric_sess_ptr;
 323         /*
 324          * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
 325          *
 326          * Only set for struct se_session's that will actually be moving I/O.
 327          * eg: *NOT* discovery sessions.
 328          */
 329         if (se_nacl) {
 330                 /*
 331                  *
 332                  * Determine if fabric allows for T10-PI feature bits exposed to
 333                  * initiators for device backends with !dev->dev_attrib.pi_prot_type.
 334                  *
 335                  * If so, then always save prot_type on a per se_node_acl node
 336                  * basis and re-instate the previous sess_prot_type to avoid
 337                  * disabling PI from below any previously initiator side
 338                  * registered LUNs.
 339                  */
 340                 if (se_nacl->saved_prot_type)
 341                         se_sess->sess_prot_type = se_nacl->saved_prot_type;
 342                 else if (tfo->tpg_check_prot_fabric_only)
 343                         se_sess->sess_prot_type = se_nacl->saved_prot_type =
 344                                         tfo->tpg_check_prot_fabric_only(se_tpg);
 345                 /*
 346                  * If the fabric module supports an ISID based TransportID,
 347                  * save this value in binary from the fabric I_T Nexus now.
 348                  */
 349                 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
 350                         memset(&buf[0], 0, PR_REG_ISID_LEN);
 351                         se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
 352                                         &buf[0], PR_REG_ISID_LEN);
 353                         se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
 354                 }
 355
 356                 spin_lock_irq(&se_nacl->nacl_sess_lock);
 357                 /*
 358                  * The se_nacl->nacl_sess pointer will be set to the
 359                  * last active I_T Nexus for each struct se_node_acl.
 360                  */
 361                 se_nacl->nacl_sess = se_sess;
 362
 363                 list_add_tail(&se_sess->sess_acl_list,
 364                               &se_nacl->acl_sess_list);
 365                 spin_unlock_irq(&se_nacl->nacl_sess_lock);
 366         }
 367         list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
 368
 369         pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
 370                 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
 371 }
 372 EXPORT_SYMBOL(__transport_register_session);
 373
 374 void transport_register_session(
 375         struct se_portal_group *se_tpg,
 376         struct se_node_acl *se_nacl,
 377         struct se_session *se_sess,
 378         void *fabric_sess_ptr)
 379 {
 380         unsigned long flags;
 381
 382         spin_lock_irqsave(&se_tpg->session_lock, flags);
 383         __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
 384         spin_unlock_irqrestore(&se_tpg->session_lock, flags);
 385 }
 386 EXPORT_SYMBOL(transport_register_session);
 387
 388 struct se_session *
 389 target_alloc_session(struct se_portal_group *tpg,
 390                      unsigned int tag_num, unsigned int tag_size,
 391                      enum target_prot_op prot_op,
 392                      const char *initiatorname, void *private,
 393                      int (*callback)(struct se_portal_group *,
 394                                      struct se_session *, void *))
 395 {
 396         struct se_session *sess;
 397
 398         /*
 399          * If the fabric driver is using percpu-ida based pre allocation
 400          * of I/O descriptor tags, go ahead and perform that setup now..
 401          */
 402         if (tag_num != 0)
 403                 sess = transport_init_session_tags(tag_num, tag_size, prot_op);
 404         else
 405                 sess = transport_init_session(prot_op);
 406
 407         if (IS_ERR(sess))
 408                 return sess;
 409
 410         sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg,
 411                                         (unsigned char *)initiatorname);
 412         if (!sess->se_node_acl) {
 413                 transport_free_session(sess);
 414                 return ERR_PTR(-EACCES);
 415         }
 416         /*
 417          * Go ahead and perform any remaining fabric setup that is
 418          * required before transport_register_session().
 419          */
 420         if (callback != NULL) {
 421                 int rc = callback(tpg, sess, private);
 422                 if (rc) {
 423                         transport_free_session(sess);
 424                         return ERR_PTR(rc);
 425                 }
 426         }
 427
 428         transport_register_session(tpg, sess->se_node_acl, sess, private);
 429         return sess;
 430 }
 431 EXPORT_SYMBOL(target_alloc_session);
 432
 433 ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page)
 434 {
 435         struct se_session *se_sess;
 436         ssize_t len = 0;
 437
 438         spin_lock_bh(&se_tpg->session_lock);
 439         list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) {
 440                 if (!se_sess->se_node_acl)
 441                         continue;
 442                 if (!se_sess->se_node_acl->dynamic_node_acl)
 443                         continue;
 444                 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE)
 445                         break;
 446
 447                 len += snprintf(page + len, PAGE_SIZE - len, "%s\n",
 448                                 se_sess->se_node_acl->initiatorname);
 449                 len += 1; /* Include NULL terminator */
 450         }
 451         spin_unlock_bh(&se_tpg->session_lock);
 452
 453         return len;
 454 }
 455 EXPORT_SYMBOL(target_show_dynamic_sessions);
 456
 457 static void target_complete_nacl(struct kref *kref)
 458 {
 459         struct se_node_acl *nacl = container_of(kref,
 460                                 struct se_node_acl, acl_kref);
 461         struct se_portal_group *se_tpg = nacl->se_tpg;
 462
 463         if (!nacl->dynamic_stop) {
 464                 complete(&nacl->acl_free_comp);
 465                 return;
 466         }
 467
 468         mutex_lock(&se_tpg->acl_node_mutex);
 469         list_del(&nacl->acl_list);
 470         mutex_unlock(&se_tpg->acl_node_mutex);
 471
 472         core_tpg_wait_for_nacl_pr_ref(nacl);
 473         core_free_device_list_for_node(nacl, se_tpg);
 474         kfree(nacl);
 475 }
 476
 477 void target_put_nacl(struct se_node_acl *nacl)
 478 {
 479         kref_put(&nacl->acl_kref, target_complete_nacl);
 480 }
 481 EXPORT_SYMBOL(target_put_nacl);
 482
 483 void transport_deregister_session_configfs(struct se_session *se_sess)
 484 {
 485         struct se_node_acl *se_nacl;
 486         unsigned long flags;
 487         /*
 488          * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
 489          */
 490         se_nacl = se_sess->se_node_acl;
 491         if (se_nacl) {
 492                 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
 493                 if (!list_empty(&se_sess->sess_acl_list))
 494                         list_del_init(&se_sess->sess_acl_list);
 495                 /*
 496                  * If the session list is empty, then clear the pointer.
 497                  * Otherwise, set the struct se_session pointer from the tail
 498                  * element of the per struct se_node_acl active session list.
 499                  */
 500                 if (list_empty(&se_nacl->acl_sess_list))
 501                         se_nacl->nacl_sess = NULL;
 502                 else {
 503                         se_nacl->nacl_sess = container_of(
 504                                         se_nacl->acl_sess_list.prev,
 505                                         struct se_session, sess_acl_list);
 506                 }
 507                 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
 508         }
 509 }
 510 EXPORT_SYMBOL(transport_deregister_session_configfs);
 511
 512 void transport_free_session(struct se_session *se_sess)
 513 {
 514         struct se_node_acl *se_nacl = se_sess->se_node_acl;
 515
 516         /*
 517          * Drop the se_node_acl->nacl_kref obtained from within
 518          * core_tpg_get_initiator_node_acl().
 519          */
 520         if (se_nacl) {
 521                 struct se_portal_group *se_tpg = se_nacl->se_tpg;
 522                 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo;
 523                 unsigned long flags;
 524
 525                 se_sess->se_node_acl = NULL;
 526
 527                 /*
 528                  * Also determine if we need to drop the extra ->cmd_kref if
 529                  * it had been previously dynamically generated, and
 530                  * the endpoint is not caching dynamic ACLs.
 531                  */
 532                 mutex_lock(&se_tpg->acl_node_mutex);
 533                 if (se_nacl->dynamic_node_acl &&
 534                     !se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
 535                         spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
 536                         if (list_empty(&se_nacl->acl_sess_list))
 537                                 se_nacl->dynamic_stop = true;
 538                         spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
 539
 540                         if (se_nacl->dynamic_stop)
 541                                 list_del(&se_nacl->acl_list);
 542                 }
 543                 mutex_unlock(&se_tpg->acl_node_mutex);
 544
 545                 if (se_nacl->dynamic_stop)
 546                         target_put_nacl(se_nacl);
 547
 548                 target_put_nacl(se_nacl);
 549         }
 550         if (se_sess->sess_cmd_map) {
 551                 percpu_ida_destroy(&se_sess->sess_tag_pool);
 552                 kvfree(se_sess->sess_cmd_map);
 553         }
 554         kmem_cache_free(se_sess_cache, se_sess);
 555 }
 556 EXPORT_SYMBOL(transport_free_session);
 557
 558 void transport_deregister_session(struct se_session *se_sess)
 559 {
 560         struct se_portal_group *se_tpg = se_sess->se_tpg;
 561         unsigned long flags;
 562
 563         if (!se_tpg) {
 564                 transport_free_session(se_sess);
 565                 return;
 566         }
 567
 568         spin_lock_irqsave(&se_tpg->session_lock, flags);
 569         list_del(&se_sess->sess_list);
 570         se_sess->se_tpg = NULL;
 571         se_sess->fabric_sess_ptr = NULL;
 572         spin_unlock_irqrestore(&se_tpg->session_lock, flags);
 573
 574         pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
 575                 se_tpg->se_tpg_tfo->get_fabric_name());
 576         /*
 577          * If last kref is dropping now for an explicit NodeACL, awake sleeping
 578          * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
 579          * removal context from within transport_free_session() code.
 580          *
 581          * For dynamic ACL, target_put_nacl() uses target_complete_nacl()
 582          * to release all remaining generate_node_acl=1 created ACL resources.
 583          */
 584
 585         transport_free_session(se_sess);
 586 }
 587 EXPORT_SYMBOL(transport_deregister_session);
 588
 589 static void target_remove_from_state_list(struct se_cmd *cmd)
 590 {
 591         struct se_device *dev = cmd->se_dev;
 592         unsigned long flags;
 593
 594         if (!dev)
 595                 return;
 596
 597         spin_lock_irqsave(&dev->execute_task_lock, flags);
 598         if (cmd->state_active) {
 599                 list_del(&cmd->state_list);
 600                 cmd->state_active = false;
 601         }
 602         spin_unlock_irqrestore(&dev->execute_task_lock, flags);
 603 }
 604
 605 static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
 606 {
 607         unsigned long flags;
 608
 609         target_remove_from_state_list(cmd);
 610
 611         /*
 612          * Clear struct se_cmd->se_lun before the handoff to FE.
 613          */
 614         cmd->se_lun = NULL;
 615
 616         spin_lock_irqsave(&cmd->t_state_lock, flags);
 617         /*
 618          * Determine if frontend context caller is requesting the stopping of
 619          * this command for frontend exceptions.
 620          */
 621         if (cmd->transport_state & CMD_T_STOP) {
 622                 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
 623                         __func__, __LINE__, cmd->tag);
 624
 625                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 626
 627                 complete_all(&cmd->t_transport_stop_comp);
 628                 return 1;
 629         }
 630         cmd->transport_state &= ~CMD_T_ACTIVE;
 631         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 632
 633         /*
 634          * Some fabric modules like tcm_loop can release their internally
 635          * allocated I/O reference and struct se_cmd now.
 636          *
 637          * Fabric modules are expected to return '1' here if the se_cmd being
 638          * passed is released at this point, or zero if not being released.
 639          */
 640         return cmd->se_tfo->check_stop_free(cmd);
 641 }
 642
 643 static void transport_lun_remove_cmd(struct se_cmd *cmd)
 644 {
 645         struct se_lun *lun = cmd->se_lun;
 646
 647         if (!lun)
 648                 return;
 649
 650         if (cmpxchg(&cmd->lun_ref_active, true, false))
 651                 percpu_ref_put(&lun->lun_ref);
 652 }
 653
 654 void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
 655 {
 656         bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF);
 657
 658         if (cmd->se_cmd_flags & SCF_SE_LUN_CMD)
 659                 transport_lun_remove_cmd(cmd);
 660         /*
 661          * Allow the fabric driver to unmap any resources before
 662          * releasing the descriptor via TFO->release_cmd()
 663          */
 664         if (remove)
 665                 cmd->se_tfo->aborted_task(cmd);
 666
 667         if (transport_cmd_check_stop_to_fabric(cmd))
 668                 return;
 669         if (remove && ack_kref)
 670                 transport_put_cmd(cmd);
 671 }
 672
 673 static void target_complete_failure_work(struct work_struct *work)
 674 {
 675         struct se_cmd *cmd = container_of(work, struct se_cmd, work);
 676
 677         transport_generic_request_failure(cmd,
 678                         TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
 679 }
 680
 681 /*
 682  * Used when asking transport to copy Sense Data from the underlying
 683  * Linux/SCSI struct scsi_cmnd
 684  */
 685 static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
 686 {
 687         struct se_device *dev = cmd->se_dev;
 688
 689         WARN_ON(!cmd->se_lun);
 690
 691         if (!dev)
 692                 return NULL;
 693
 694         if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
 695                 return NULL;
 696
 697         cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
 698
 699         pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
 700                 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
 701         return cmd->sense_buffer;
 702 }
 703
 704 void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
 705 {
 706         struct se_device *dev = cmd->se_dev;
 707         int success = scsi_status == GOOD;
 708         unsigned long flags;
 709
 710         cmd->scsi_status = scsi_status;
 711
 712
 713         spin_lock_irqsave(&cmd->t_state_lock, flags);
 714
 715         if (dev && dev->transport->transport_complete) {
 716                 dev->transport->transport_complete(cmd,
 717                                 cmd->t_data_sg,
 718                                 transport_get_sense_buffer(cmd));
 719                 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
 720                         success = 1;
 721         }
 722
 723         /*
 724          * Check for case where an explicit ABORT_TASK has been received
 725          * and transport_wait_for_tasks() will be waiting for completion..
 726          */
 727         if (cmd->transport_state & CMD_T_ABORTED ||
 728             cmd->transport_state & CMD_T_STOP) {
 729                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 730                 complete_all(&cmd->t_transport_stop_comp);
 731                 return;
 732         } else if (!success) {
 733                 INIT_WORK(&cmd->work, target_complete_failure_work);
 734         } else {
 735                 INIT_WORK(&cmd->work, target_complete_ok_work);
 736         }
 737
 738         cmd->t_state = TRANSPORT_COMPLETE;
 739         cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
 740         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
 741
 742         if (cmd->se_cmd_flags & SCF_USE_CPUID)
 743                 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
 744         else
 745                 queue_work(target_completion_wq, &cmd->work);
 746 }
 747 EXPORT_SYMBOL(target_complete_cmd);
 748
 749 void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length)
 750 {
 751         if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) {
 752                 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
 753                         cmd->residual_count += cmd->data_length - length;
 754                 } else {
 755                         cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
 756                         cmd->residual_count = cmd->data_length - length;
 757                 }
 758
 759                 cmd->data_length = length;
 760         }
 761
 762         target_complete_cmd(cmd, scsi_status);
 763 }
 764 EXPORT_SYMBOL(target_complete_cmd_with_length);
 765
 766 static void target_add_to_state_list(struct se_cmd *cmd)
 767 {
 768         struct se_device *dev = cmd->se_dev;
 769         unsigned long flags;
 770
 771         spin_lock_irqsave(&dev->execute_task_lock, flags);
 772         if (!cmd->state_active) {
 773                 list_add_tail(&cmd->state_list, &dev->state_list);
 774                 cmd->state_active = true;
 775         }
 776         spin_unlock_irqrestore(&dev->execute_task_lock, flags);
 777 }
 778
 779 /*
 780  * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
 781  */
 782 static void transport_write_pending_qf(struct se_cmd *cmd);
 783 static void transport_complete_qf(struct se_cmd *cmd);
 784
 785 void target_qf_do_work(struct work_struct *work)
 786 {
 787         struct se_device *dev = container_of(work, struct se_device,
 788                                         qf_work_queue);
 789         LIST_HEAD(qf_cmd_list);
 790         struct se_cmd *cmd, *cmd_tmp;
 791
 792         spin_lock_irq(&dev->qf_cmd_lock);
 793         list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
 794         spin_unlock_irq(&dev->qf_cmd_lock);
 795
 796         list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
 797                 list_del(&cmd->se_qf_node);
 798                 atomic_dec_mb(&dev->dev_qf_count);
 799
 800                 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
 801                         " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
 802                         (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
 803                         (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
 804                         : "UNKNOWN");
 805
 806                 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
 807                         transport_write_pending_qf(cmd);
 808                 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK ||
 809                          cmd->t_state == TRANSPORT_COMPLETE_QF_ERR)
 810                         transport_complete_qf(cmd);
 811         }
 812 }
 813
 814 unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
 815 {
 816         switch (cmd->data_direction) {
 817         case DMA_NONE:
 818                 return "NONE";
 819         case DMA_FROM_DEVICE:
 820                 return "READ";
 821         case DMA_TO_DEVICE:
 822                 return "WRITE";
 823         case DMA_BIDIRECTIONAL:
 824                 return "BIDI";
 825         default:
 826                 break;
 827         }
 828
 829         return "UNKNOWN";
 830 }
 831
 832 void transport_dump_dev_state(
 833         struct se_device *dev,
 834         char *b,
 835         int *bl)
 836 {
 837         *bl += sprintf(b + *bl, "Status: ");
 838         if (dev->export_count)
 839                 *bl += sprintf(b + *bl, "ACTIVATED");
 840         else
 841                 *bl += sprintf(b + *bl, "DEACTIVATED");
 842
 843         *bl += sprintf(b + *bl, "  Max Queue Depth: %d", dev->queue_depth);
 844         *bl += sprintf(b + *bl, "  SectorSize: %u  HwMaxSectors: %u\n",
 845                 dev->dev_attrib.block_size,
 846                 dev->dev_attrib.hw_max_sectors);
 847         *bl += sprintf(b + *bl, "        ");
 848 }
 849
 850 void transport_dump_vpd_proto_id(
 851         struct t10_vpd *vpd,
 852         unsigned char *p_buf,
 853         int p_buf_len)
 854 {
 855         unsigned char buf[VPD_TMP_BUF_SIZE];
 856         int len;
 857
 858         memset(buf, 0, VPD_TMP_BUF_SIZE);
 859         len = sprintf(buf, "T10 VPD Protocol Identifier: ");
 860
 861         switch (vpd->protocol_identifier) {
 862         case 0x00:
 863                 sprintf(buf+len, "Fibre Channel\n");
 864                 break;
 865         case 0x10:
 866                 sprintf(buf+len, "Parallel SCSI\n");
 867                 break;
 868         case 0x20:
 869                 sprintf(buf+len, "SSA\n");
 870                 break;
 871         case 0x30:
 872                 sprintf(buf+len, "IEEE 1394\n");
 873                 break;
 874         case 0x40:
 875                 sprintf(buf+len, "SCSI Remote Direct Memory Access"
 876                                 " Protocol\n");
 877                 break;
 878         case 0x50:
 879                 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
 880                 break;
 881         case 0x60:
 882                 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
 883                 break;
 884         case 0x70:
 885                 sprintf(buf+len, "Automation/Drive Interface Transport"
 886                                 " Protocol\n");
 887                 break;
 888         case 0x80:
 889                 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
 890                 break;
 891         default:
 892                 sprintf(buf+len, "Unknown 0x%02x\n",
 893                                 vpd->protocol_identifier);
 894                 break;
 895         }
 896
 897         if (p_buf)
 898                 strncpy(p_buf, buf, p_buf_len);
 899         else
 900                 pr_debug("%s", buf);
 901 }
 902
 903 void
 904 transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
 905 {
 906         /*
 907          * Check if the Protocol Identifier Valid (PIV) bit is set..
 908          *
 909          * from spc3r23.pdf section 7.5.1
 910          */
 911          if (page_83[1] & 0x80) {
 912                 vpd->protocol_identifier = (page_83[0] & 0xf0);
 913                 vpd->protocol_identifier_set = 1;
 914                 transport_dump_vpd_proto_id(vpd, NULL, 0);
 915         }
 916 }
 917 EXPORT_SYMBOL(transport_set_vpd_proto_id);
 918
 919 int transport_dump_vpd_assoc(
 920         struct t10_vpd *vpd,
 921         unsigned char *p_buf,
 922         int p_buf_len)
 923 {
 924         unsigned char buf[VPD_TMP_BUF_SIZE];
 925         int ret = 0;
 926         int len;
 927
 928         memset(buf, 0, VPD_TMP_BUF_SIZE);
 929         len = sprintf(buf, "T10 VPD Identifier Association: ");
 930
 931         switch (vpd->association) {
 932         case 0x00:
 933                 sprintf(buf+len, "addressed logical unit\n");
 934                 break;
 935         case 0x10:
 936                 sprintf(buf+len, "target port\n");
 937                 break;
 938         case 0x20:
 939                 sprintf(buf+len, "SCSI target device\n");
 940                 break;
 941         default:
 942                 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
 943                 ret = -EINVAL;
 944                 break;
 945         }
 946
 947         if (p_buf)
 948                 strncpy(p_buf, buf, p_buf_len);
 949         else
 950                 pr_debug("%s", buf);
 951
 952         return ret;
 953 }
 954
 955 int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
 956 {
 957         /*
 958          * The VPD identification association..
 959          *
 960          * from spc3r23.pdf Section 7.6.3.1 Table 297
 961          */
 962         vpd->association = (page_83[1] & 0x30);
 963         return transport_dump_vpd_assoc(vpd, NULL, 0);
 964 }
 965 EXPORT_SYMBOL(transport_set_vpd_assoc);
 966
 967 int transport_dump_vpd_ident_type(
 968         struct t10_vpd *vpd,
 969         unsigned char *p_buf,
 970         int p_buf_len)
 971 {
 972         unsigned char buf[VPD_TMP_BUF_SIZE];
 973         int ret = 0;
 974         int len;
 975
 976         memset(buf, 0, VPD_TMP_BUF_SIZE);
 977         len = sprintf(buf, "T10 VPD Identifier Type: ");
 978
 979         switch (vpd->device_identifier_type) {
 980         case 0x00:
 981                 sprintf(buf+len, "Vendor specific\n");
 982                 break;
 983         case 0x01:
 984                 sprintf(buf+len, "T10 Vendor ID based\n");
 985                 break;
 986         case 0x02:
 987                 sprintf(buf+len, "EUI-64 based\n");
 988                 break;
 989         case 0x03:
 990                 sprintf(buf+len, "NAA\n");
 991                 break;
 992         case 0x04:
 993                 sprintf(buf+len, "Relative target port identifier\n");
 994                 break;
 995         case 0x08:
 996                 sprintf(buf+len, "SCSI name string\n");
 997                 break;
 998         default:
 999                 sprintf(buf+len, "Unsupported: 0x%02x\n",
1000                                 vpd->device_identifier_type);
1001                 ret = -EINVAL;
1002                 break;
1003         }
1004
1005         if (p_buf) {
1006                 if (p_buf_len < strlen(buf)+1)
1007                         return -EINVAL;
1008                 strncpy(p_buf, buf, p_buf_len);
1009         } else {
1010                 pr_debug("%s", buf);
1011         }
1012
1013         return ret;
1014 }
1015
1016 int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1017 {
1018         /*
1019          * The VPD identifier type..
1020          *
1021          * from spc3r23.pdf Section 7.6.3.1 Table 298
1022          */
1023         vpd->device_identifier_type = (page_83[1] & 0x0f);
1024         return transport_dump_vpd_ident_type(vpd, NULL, 0);
1025 }
1026 EXPORT_SYMBOL(transport_set_vpd_ident_type);
1027
1028 int transport_dump_vpd_ident(
1029         struct t10_vpd *vpd,
1030         unsigned char *p_buf,
1031         int p_buf_len)
1032 {
1033         unsigned char buf[VPD_TMP_BUF_SIZE];
1034         int ret = 0;
1035
1036         memset(buf, 0, VPD_TMP_BUF_SIZE);
1037
1038         switch (vpd->device_identifier_code_set) {
1039         case 0x01: /* Binary */
1040                 snprintf(buf, sizeof(buf),
1041                         "T10 VPD Binary Device Identifier: %s\n",
1042                         &vpd->device_identifier[0]);
1043                 break;
1044         case 0x02: /* ASCII */
1045                 snprintf(buf, sizeof(buf),
1046                         "T10 VPD ASCII Device Identifier: %s\n",
1047                         &vpd->device_identifier[0]);
1048                 break;
1049         case 0x03: /* UTF-8 */
1050                 snprintf(buf, sizeof(buf),
1051                         "T10 VPD UTF-8 Device Identifier: %s\n",
1052                         &vpd->device_identifier[0]);
1053                 break;
1054         default:
1055                 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1056                         " 0x%02x", vpd->device_identifier_code_set);
1057                 ret = -EINVAL;
1058                 break;
1059         }
1060
1061         if (p_buf)
1062                 strncpy(p_buf, buf, p_buf_len);
1063         else
1064                 pr_debug("%s", buf);
1065
1066         return ret;
1067 }
1068
1069 int
1070 transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1071 {
1072         static const char hex_str[] = "0123456789abcdef";
1073         int j = 0, i = 4; /* offset to start of the identifier */
1074
1075         /*
1076          * The VPD Code Set (encoding)
1077          *
1078          * from spc3r23.pdf Section 7.6.3.1 Table 296
1079          */
1080         vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1081         switch (vpd->device_identifier_code_set) {
1082         case 0x01: /* Binary */
1083                 vpd->device_identifier[j++] =
1084                                 hex_str[vpd->device_identifier_type];
1085                 while (i < (4 + page_83[3])) {
1086                         vpd->device_identifier[j++] =
1087                                 hex_str[(page_83[i] & 0xf0) >> 4];
1088                         vpd->device_identifier[j++] =
1089                                 hex_str[page_83[i] & 0x0f];
1090                         i++;
1091                 }
1092                 break;
1093         case 0x02: /* ASCII */
1094         case 0x03: /* UTF-8 */
1095                 while (i < (4 + page_83[3]))
1096                         vpd->device_identifier[j++] = page_83[i++];
1097                 break;
1098         default:
1099                 break;
1100         }
1101
1102         return transport_dump_vpd_ident(vpd, NULL, 0);
1103 }
1104 EXPORT_SYMBOL(transport_set_vpd_ident);
1105
1106 static sense_reason_t
1107 target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev,
1108                                unsigned int size)
1109 {
1110         u32 mtl;
1111
1112         if (!cmd->se_tfo->max_data_sg_nents)
1113                 return TCM_NO_SENSE;
1114         /*
1115          * Check if fabric enforced maximum SGL entries per I/O descriptor
1116          * exceeds se_cmd->data_length.  If true, set SCF_UNDERFLOW_BIT +
1117          * residual_count and reduce original cmd->data_length to maximum
1118          * length based on single PAGE_SIZE entry scatter-lists.
1119          */
1120         mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE);
1121         if (cmd->data_length > mtl) {
1122                 /*
1123                  * If an existing CDB overflow is present, calculate new residual
1124                  * based on CDB size minus fabric maximum transfer length.
1125                  *
1126                  * If an existing CDB underflow is present, calculate new residual
1127                  * based on original cmd->data_length minus fabric maximum transfer
1128                  * length.
1129                  *
1130                  * Otherwise, set the underflow residual based on cmd->data_length
1131                  * minus fabric maximum transfer length.
1132                  */
1133                 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1134                         cmd->residual_count = (size - mtl);
1135                 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
1136                         u32 orig_dl = size + cmd->residual_count;
1137                         cmd->residual_count = (orig_dl - mtl);
1138                 } else {
1139                         cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
1140                         cmd->residual_count = (cmd->data_length - mtl);
1141                 }
1142                 cmd->data_length = mtl;
1143                 /*
1144                  * Reset sbc_check_prot() calculated protection payload
1145                  * length based upon the new smaller MTL.
1146                  */
1147                 if (cmd->prot_length) {
1148                         u32 sectors = (mtl / dev->dev_attrib.block_size);
1149                         cmd->prot_length = dev->prot_length * sectors;
1150                 }
1151         }
1152         return TCM_NO_SENSE;
1153 }
1154
1155 sense_reason_t
1156 target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
1157 {
1158         struct se_device *dev = cmd->se_dev;
1159
1160         if (cmd->unknown_data_length) {
1161                 cmd->data_length = size;
1162         } else if (size != cmd->data_length) {
1163                 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:"
1164                         " %u does not match SCSI CDB Length: %u for SAM Opcode:"
1165                         " 0x%02x\n", cmd->se_tfo->get_fabric_name(),
1166                                 cmd->data_length, size, cmd->t_task_cdb[0]);
1167
1168                 if (cmd->data_direction == DMA_TO_DEVICE) {
1169                         if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) {
1170                                 pr_err_ratelimited("Rejecting underflow/overflow"
1171                                                    " for WRITE data CDB\n");
1172                                 return TCM_INVALID_CDB_FIELD;
1173                         }
1174                         /*
1175                          * Some fabric drivers like iscsi-target still expect to
1176                          * always reject overflow writes.  Reject this case until
1177                          * full fabric driver level support for overflow writes
1178                          * is introduced tree-wide.
1179                          */
1180                         if (size > cmd->data_length) {
1181                                 pr_err_ratelimited("Rejecting overflow for"
1182                                                    " WRITE control CDB\n");
1183                                 return TCM_INVALID_CDB_FIELD;
1184                         }
1185                 }
1186                 /*
1187                  * Reject READ_* or WRITE_* with overflow/underflow for
1188                  * type SCF_SCSI_DATA_CDB.
1189                  */
1190                 if (dev->dev_attrib.block_size != 512)  {
1191                         pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
1192                                 " CDB on non 512-byte sector setup subsystem"
1193                                 " plugin: %s\n", dev->transport->name);
1194                         /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
1195                         return TCM_INVALID_CDB_FIELD;
1196                 }
1197                 /*
1198                  * For the overflow case keep the existing fabric provided
1199                  * ->data_length.  Otherwise for the underflow case, reset
1200                  * ->data_length to the smaller SCSI expected data transfer
1201                  * length.
1202                  */
1203                 if (size > cmd->data_length) {
1204                         cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
1205                         cmd->residual_count = (size - cmd->data_length);
1206                 } else {
1207                         cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
1208                         cmd->residual_count = (cmd->data_length - size);
1209                         cmd->data_length = size;
1210                 }
1211         }
1212
1213         return target_check_max_data_sg_nents(cmd, dev, size);
1214
1215 }
1216
1217 /*
1218  * Used by fabric modules containing a local struct se_cmd within their
1219  * fabric dependent per I/O descriptor.
1220  *
1221  * Preserves the value of @cmd->tag.
1222  */
1223 void transport_init_se_cmd(
1224         struct se_cmd *cmd,
1225         const struct target_core_fabric_ops *tfo,
1226         struct se_session *se_sess,
1227         u32 data_length,
1228         int data_direction,
1229         int task_attr,
1230         unsigned char *sense_buffer)
1231 {
1232         INIT_LIST_HEAD(&cmd->se_delayed_node);
1233         INIT_LIST_HEAD(&cmd->se_qf_node);
1234         INIT_LIST_HEAD(&cmd->se_cmd_list);
1235         INIT_LIST_HEAD(&cmd->state_list);
1236         init_completion(&cmd->t_transport_stop_comp);
1237         init_completion(&cmd->cmd_wait_comp);
1238         spin_lock_init(&cmd->t_state_lock);
1239         kref_init(&cmd->cmd_kref);
1240
1241         cmd->se_tfo = tfo;
1242         cmd->se_sess = se_sess;
1243         cmd->data_length = data_length;
1244         cmd->data_direction = data_direction;
1245         cmd->sam_task_attr = task_attr;
1246         cmd->sense_buffer = sense_buffer;
1247
1248         cmd->state_active = false;
1249 }
1250 EXPORT_SYMBOL(transport_init_se_cmd);
1251
1252 static sense_reason_t
1253 transport_check_alloc_task_attr(struct se_cmd *cmd)
1254 {
1255         struct se_device *dev = cmd->se_dev;
1256
1257         /*
1258          * Check if SAM Task Attribute emulation is enabled for this
1259          * struct se_device storage object
1260          */
1261         if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1262                 return 0;
1263
1264         if (cmd->sam_task_attr == TCM_ACA_TAG) {
1265                 pr_debug("SAM Task Attribute ACA"
1266                         " emulation is not supported\n");
1267                 return TCM_INVALID_CDB_FIELD;
1268         }
1269
1270         return 0;
1271 }
1272
1273 sense_reason_t
1274 target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
1275 {
1276         struct se_device *dev = cmd->se_dev;
1277         sense_reason_t ret;
1278
1279         /*
1280          * Ensure that the received CDB is less than the max (252 + 8) bytes
1281          * for VARIABLE_LENGTH_CMD
1282          */
1283         if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1284                 pr_err("Received SCSI CDB with command_size: %d that"
1285                         " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1286                         scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1287                 return TCM_INVALID_CDB_FIELD;
1288         }
1289         /*
1290          * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
1291          * allocate the additional extended CDB buffer now..  Otherwise
1292          * setup the pointer from __t_task_cdb to t_task_cdb.
1293          */
1294         if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
1295                 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1296                                                 GFP_KERNEL);
1297                 if (!cmd->t_task_cdb) {
1298                         pr_err("Unable to allocate cmd->t_task_cdb"
1299                                 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1300                                 scsi_command_size(cdb),
1301                                 (unsigned long)sizeof(cmd->__t_task_cdb));
1302                         return TCM_OUT_OF_RESOURCES;
1303                 }
1304         } else
1305                 cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1306         /*
1307          * Copy the original CDB into cmd->
1308          */
1309         memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1310
1311         trace_target_sequencer_start(cmd);
1312
1313         ret = dev->transport->parse_cdb(cmd);
1314         if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
1315                 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n",
1316                                     cmd->se_tfo->get_fabric_name(),
1317                                     cmd->se_sess->se_node_acl->initiatorname,
1318                                     cmd->t_task_cdb[0]);
1319         if (ret)
1320                 return ret;
1321
1322         ret = transport_check_alloc_task_attr(cmd);
1323         if (ret)
1324                 return ret;
1325
1326         cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
1327         atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
1328         return 0;
1329 }
1330 EXPORT_SYMBOL(target_setup_cmd_from_cdb);
1331
1332 /*
1333  * Used by fabric module frontends to queue tasks directly.
1334  * May only be used from process context.
1335  */
1336 int transport_handle_cdb_direct(
1337         struct se_cmd *cmd)
1338 {
1339         sense_reason_t ret;
1340
1341         if (!cmd->se_lun) {
1342                 dump_stack();
1343                 pr_err("cmd->se_lun is NULL\n");
1344                 return -EINVAL;
1345         }
1346         if (in_interrupt()) {
1347                 dump_stack();
1348                 pr_err("transport_generic_handle_cdb cannot be called"
1349                                 " from interrupt context\n");
1350                 return -EINVAL;
1351         }
1352         /*
1353          * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that
1354          * outstanding descriptors are handled correctly during shutdown via
1355          * transport_wait_for_tasks()
1356          *
1357          * Also, we don't take cmd->t_state_lock here as we only expect
1358          * this to be called for initial descriptor submission.
1359          */
1360         cmd->t_state = TRANSPORT_NEW_CMD;
1361         cmd->transport_state |= CMD_T_ACTIVE;
1362
1363         /*
1364          * transport_generic_new_cmd() is already handling QUEUE_FULL,
1365          * so follow TRANSPORT_NEW_CMD processing thread context usage
1366          * and call transport_generic_request_failure() if necessary..
1367          */
1368         ret = transport_generic_new_cmd(cmd);
1369         if (ret)
1370                 transport_generic_request_failure(cmd, ret);
1371         return 0;
1372 }
1373 EXPORT_SYMBOL(transport_handle_cdb_direct);
1374
1375 sense_reason_t
1376 transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
1377                 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
1378 {
1379         if (!sgl || !sgl_count)
1380                 return 0;
1381
1382         /*
1383          * Reject SCSI data overflow with map_mem_to_cmd() as incoming
1384          * scatterlists already have been set to follow what the fabric
1385          * passes for the original expected data transfer length.
1386          */
1387         if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
1388                 pr_warn("Rejecting SCSI DATA overflow for fabric using"
1389                         " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
1390                 return TCM_INVALID_CDB_FIELD;
1391         }
1392
1393         cmd->t_data_sg = sgl;
1394         cmd->t_data_nents = sgl_count;
1395         cmd->t_bidi_data_sg = sgl_bidi;
1396         cmd->t_bidi_data_nents = sgl_bidi_count;
1397
1398         cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1399         return 0;
1400 }
1401
1402 /*
1403  * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized
1404  *                       se_cmd + use pre-allocated SGL memory.
1405  *
1406  * @se_cmd: command descriptor to submit
1407  * @se_sess: associated se_sess for endpoint
1408  * @cdb: pointer to SCSI CDB
1409  * @sense: pointer to SCSI sense buffer
1410  * @unpacked_lun: unpacked LUN to reference for struct se_lun
1411  * @data_length: fabric expected data transfer length
1412  * @task_addr: SAM task attribute
1413  * @data_dir: DMA data direction
1414  * @flags: flags for command submission from target_sc_flags_tables
1415  * @sgl: struct scatterlist memory for unidirectional mapping
1416  * @sgl_count: scatterlist count for unidirectional mapping
1417  * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping
1418  * @sgl_bidi_count: scatterlist count for bidirectional READ mapping
1419  * @sgl_prot: struct scatterlist memory protection information
1420  * @sgl_prot_count: scatterlist count for protection information
1421  *
1422  * Task tags are supported if the caller has set @se_cmd->tag.
1423  *
1424  * Returns non zero to signal active I/O shutdown failure.  All other
1425  * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
1426  * but still return zero here.
1427  *
1428  * This may only be called from process context, and also currently
1429  * assumes internal allocation of fabric payload buffer by target-core.
1430  */
1431 int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
1432                 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1433                 u32 data_length, int task_attr, int data_dir, int flags,
1434                 struct scatterlist *sgl, u32 sgl_count,
1435                 struct scatterlist *sgl_bidi, u32 sgl_bidi_count,
1436                 struct scatterlist *sgl_prot, u32 sgl_prot_count)
1437 {
1438         struct se_portal_group *se_tpg;
1439         sense_reason_t rc;
1440         int ret;
1441
1442         se_tpg = se_sess->se_tpg;
1443         BUG_ON(!se_tpg);
1444         BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
1445         BUG_ON(in_interrupt());
1446         /*
1447          * Initialize se_cmd for target operation.  From this point
1448          * exceptions are handled by sending exception status via
1449          * target_core_fabric_ops->queue_status() callback
1450          */
1451         transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1452                                 data_length, data_dir, task_attr, sense);
1453
1454         if (flags & TARGET_SCF_USE_CPUID)
1455                 se_cmd->se_cmd_flags |= SCF_USE_CPUID;
1456         else
1457                 se_cmd->cpuid = WORK_CPU_UNBOUND;
1458
1459         if (flags & TARGET_SCF_UNKNOWN_SIZE)
1460                 se_cmd->unknown_data_length = 1;
1461         /*
1462          * Obtain struct se_cmd->cmd_kref reference and add new cmd to
1463          * se_sess->sess_cmd_list.  A second kref_get here is necessary
1464          * for fabrics using TARGET_SCF_ACK_KREF that expect a second
1465          * kref_put() to happen during fabric packet acknowledgement.
1466          */
1467         ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1468         if (ret)
1469                 return ret;
1470         /*
1471          * Signal bidirectional data payloads to target-core
1472          */
1473         if (flags & TARGET_SCF_BIDI_OP)
1474                 se_cmd->se_cmd_flags |= SCF_BIDI;
1475         /*
1476          * Locate se_lun pointer and attach it to struct se_cmd
1477          */
1478         rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
1479         if (rc) {
1480                 transport_send_check_condition_and_sense(se_cmd, rc, 0);
1481                 target_put_sess_cmd(se_cmd);
1482                 return 0;
1483         }
1484
1485         rc = target_setup_cmd_from_cdb(se_cmd, cdb);
1486         if (rc != 0) {
1487                 transport_generic_request_failure(se_cmd, rc);
1488                 return 0;
1489         }
1490
1491         /*
1492          * Save pointers for SGLs containing protection information,
1493          * if present.
1494          */
1495         if (sgl_prot_count) {
1496                 se_cmd->t_prot_sg = sgl_prot;
1497                 se_cmd->t_prot_nents = sgl_prot_count;
1498                 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC;
1499         }
1500
1501         /*
1502          * When a non zero sgl_count has been passed perform SGL passthrough
1503          * mapping for pre-allocated fabric memory instead of having target
1504          * core perform an internal SGL allocation..
1505          */
1506         if (sgl_count != 0) {
1507                 BUG_ON(!sgl);
1508
1509                 /*
1510                  * A work-around for tcm_loop as some userspace code via
1511                  * scsi-generic do not memset their associated read buffers,
1512                  * so go ahead and do that here for type non-data CDBs.  Also
1513                  * note that this is currently guaranteed to be a single SGL
1514                  * for this case by target core in target_setup_cmd_from_cdb()
1515                  * -> transport_generic_cmd_sequencer().
1516                  */
1517                 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
1518                      se_cmd->data_direction == DMA_FROM_DEVICE) {
1519                         unsigned char *buf = NULL;
1520
1521                         if (sgl)
1522                                 buf = kmap(sg_page(sgl)) + sgl->offset;
1523
1524                         if (buf) {
1525                                 memset(buf, 0, sgl->length);
1526                                 kunmap(sg_page(sgl));
1527                         }
1528                 }
1529
1530                 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
1531                                 sgl_bidi, sgl_bidi_count);
1532                 if (rc != 0) {
1533                         transport_generic_request_failure(se_cmd, rc);
1534                         return 0;
1535                 }
1536         }
1537
1538         /*
1539          * Check if we need to delay processing because of ALUA
1540          * Active/NonOptimized primary access state..
1541          */
1542         core_alua_check_nonop_delay(se_cmd);
1543
1544         transport_handle_cdb_direct(se_cmd);
1545         return 0;
1546 }
1547 EXPORT_SYMBOL(target_submit_cmd_map_sgls);
1548
1549 /*
1550  * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
1551  *
1552  * @se_cmd: command descriptor to submit
1553  * @se_sess: associated se_sess for endpoint
1554  * @cdb: pointer to SCSI CDB
1555  * @sense: pointer to SCSI sense buffer
1556  * @unpacked_lun: unpacked LUN to reference for struct se_lun
1557  * @data_length: fabric expected data transfer length
1558  * @task_addr: SAM task attribute
1559  * @data_dir: DMA data direction
1560  * @flags: flags for command submission from target_sc_flags_tables
1561  *
1562  * Task tags are supported if the caller has set @se_cmd->tag.
1563  *
1564  * Returns non zero to signal active I/O shutdown failure.  All other
1565  * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
1566  * but still return zero here.
1567  *
1568  * This may only be called from process context, and also currently
1569  * assumes internal allocation of fabric payload buffer by target-core.
1570  *
1571  * It also assumes interal target core SGL memory allocation.
1572  */
1573 int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
1574                 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1575                 u32 data_length, int task_attr, int data_dir, int flags)
1576 {
1577         return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
1578                         unpacked_lun, data_length, task_attr, data_dir,
1579                         flags, NULL, 0, NULL, 0, NULL, 0);
1580 }
1581 EXPORT_SYMBOL(target_submit_cmd);
1582
1583 static void target_complete_tmr_failure(struct work_struct *work)
1584 {
1585         struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);
1586
1587         se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
1588         se_cmd->se_tfo->queue_tm_rsp(se_cmd);
1589
1590         transport_cmd_check_stop_to_fabric(se_cmd);
1591 }
1592
1593 /**
1594  * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd
1595  *                     for TMR CDBs
1596  *
1597  * @se_cmd: command descriptor to submit
1598  * @se_sess: associated se_sess for endpoint
1599  * @sense: pointer to SCSI sense buffer
1600  * @unpacked_lun: unpacked LUN to reference for struct se_lun
1601  * @fabric_context: fabric context for TMR req
1602  * @tm_type: Type of TM request
1603  * @gfp: gfp type for caller
1604  * @tag: referenced task tag for TMR_ABORT_TASK
1605  * @flags: submit cmd flags
1606  *
1607  * Callable from all contexts.
1608  **/
1609
1610 int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
1611                 unsigned char *sense, u64 unpacked_lun,
1612                 void *fabric_tmr_ptr, unsigned char tm_type,
1613                 gfp_t gfp, u64 tag, int flags)
1614 {
1615         struct se_portal_group *se_tpg;
1616         int ret;
1617
1618         se_tpg = se_sess->se_tpg;
1619         BUG_ON(!se_tpg);
1620
1621         transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1622                               0, DMA_NONE, TCM_SIMPLE_TAG, sense);
1623         /*
1624          * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
1625          * allocation failure.
1626          */
1627         ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
1628         if (ret < 0)
1629                 return -ENOMEM;
1630
1631         if (tm_type == TMR_ABORT_TASK)
1632                 se_cmd->se_tmr_req->ref_task_tag = tag;
1633
1634         /* See target_submit_cmd for commentary */
1635         ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1636         if (ret) {
1637                 core_tmr_release_req(se_cmd->se_tmr_req);
1638                 return ret;
1639         }
1640
1641         ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
1642         if (ret) {
1643                 /*
1644                  * For callback during failure handling, push this work off
1645                  * to process context with TMR_LUN_DOES_NOT_EXIST status.
1646                  */
1647                 INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
1648                 schedule_work(&se_cmd->work);
1649                 return 0;
1650         }
1651         transport_generic_handle_tmr(se_cmd);
1652         return 0;
1653 }
1654 EXPORT_SYMBOL(target_submit_tmr);
1655
1656 /*
1657  * Handle SAM-esque emulation for generic transport request failures.
1658  */
1659 void transport_generic_request_failure(struct se_cmd *cmd,
1660                 sense_reason_t sense_reason)
1661 {
1662         int ret = 0, post_ret = 0;
1663
1664         if (transport_check_aborted_status(cmd, 1))
1665                 return;
1666
1667         pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08llx"
1668                 " CDB: 0x%02x\n", cmd, cmd->tag, cmd->t_task_cdb[0]);
1669         pr_debug("-----[ i_state: %d t_state: %d sense_reason: %d\n",
1670                 cmd->se_tfo->get_cmd_state(cmd),
1671                 cmd->t_state, sense_reason);
1672         pr_debug("-----[ CMD_T_ACTIVE: %d CMD_T_STOP: %d CMD_T_SENT: %d\n",
1673                 (cmd->transport_state & CMD_T_ACTIVE) != 0,
1674                 (cmd->transport_state & CMD_T_STOP) != 0,
1675                 (cmd->transport_state & CMD_T_SENT) != 0);
1676
1677         /*
1678          * For SAM Task Attribute emulation for failed struct se_cmd
1679          */
1680         transport_complete_task_attr(cmd);
1681         /*
1682          * Handle special case for COMPARE_AND_WRITE failure, where the
1683          * callback is expected to drop the per device ->caw_sem.
1684          */
1685         if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
1686              cmd->transport_complete_callback)
1687                 cmd->transport_complete_callback(cmd, false, &post_ret);
1688
1689         switch (sense_reason) {
1690         case TCM_NON_EXISTENT_LUN:
1691         case TCM_UNSUPPORTED_SCSI_OPCODE:
1692         case TCM_INVALID_CDB_FIELD:
1693         case TCM_INVALID_PARAMETER_LIST:
1694         case TCM_PARAMETER_LIST_LENGTH_ERROR:
1695         case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
1696         case TCM_UNKNOWN_MODE_PAGE:
1697         case TCM_WRITE_PROTECTED:
1698         case TCM_ADDRESS_OUT_OF_RANGE:
1699         case TCM_CHECK_CONDITION_ABORT_CMD:
1700         case TCM_CHECK_CONDITION_UNIT_ATTENTION:
1701         case TCM_CHECK_CONDITION_NOT_READY:
1702         case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED:
1703         case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED:
1704         case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED:
1705         case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE:
1706         case TCM_TOO_MANY_TARGET_DESCS:
1707         case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE:
1708         case TCM_TOO_MANY_SEGMENT_DESCS:
1709         case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE:
1710                 break;
1711         case TCM_OUT_OF_RESOURCES:
1712                 sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1713                 break;
1714         case TCM_RESERVATION_CONFLICT:
1715                 /*
1716                  * No SENSE Data payload for this case, set SCSI Status
1717                  * and queue the response to $FABRIC_MOD.
1718                  *
1719                  * Uses linux/include/scsi/scsi.h SAM status codes defs
1720                  */
1721                 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1722                 /*
1723                  * For UA Interlock Code 11b, a RESERVATION CONFLICT will
1724                  * establish a UNIT ATTENTION with PREVIOUS RESERVATION
1725                  * CONFLICT STATUS.
1726                  *
1727                  * See spc4r17, section 7.4.6 Control Mode Page, Table 349
1728                  */
1729                 if (cmd->se_sess &&
1730                     cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) {
1731                         target_ua_allocate_lun(cmd->se_sess->se_node_acl,
1732                                                cmd->orig_fe_lun, 0x2C,
1733                                         ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
1734                 }
1735                 trace_target_cmd_complete(cmd);
1736                 ret = cmd->se_tfo->queue_status(cmd);
1737                 if (ret)
1738                         goto queue_full;
1739                 goto check_stop;
1740         default:
1741                 pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1742                         cmd->t_task_cdb[0], sense_reason);
1743                 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1744                 break;
1745         }
1746
1747         ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
1748         if (ret)
1749                 goto queue_full;
1750
1751 check_stop:
1752         transport_lun_remove_cmd(cmd);
1753         transport_cmd_check_stop_to_fabric(cmd);
1754         return;
1755
1756 queue_full:
1757         transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
1758 }
1759 EXPORT_SYMBOL(transport_generic_request_failure);
1760
1761 void __target_execute_cmd(struct se_cmd *cmd, bool do_checks)
1762 {
1763         sense_reason_t ret;
1764
1765         if (!cmd->execute_cmd) {
1766                 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1767                 goto err;
1768         }
1769         if (do_checks) {
1770                 /*
1771                  * Check for an existing UNIT ATTENTION condition after
1772                  * target_handle_task_attr() has done SAM task attr
1773                  * checking, and possibly have already defered execution
1774                  * out to target_restart_delayed_cmds() context.
1775                  */
1776                 ret = target_scsi3_ua_check(cmd);
1777                 if (ret)
1778                         goto err;
1779
1780                 ret = target_alua_state_check(cmd);
1781                 if (ret)
1782                         goto err;
1783
1784                 ret = target_check_reservation(cmd);
1785                 if (ret) {
1786                         cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1787                         goto err;
1788                 }
1789         }
1790
1791         ret = cmd->execute_cmd(cmd);
1792         if (!ret)
1793                 return;
1794 err:
1795         spin_lock_irq(&cmd->t_state_lock);
1796         cmd->transport_state &= ~CMD_T_SENT;
1797         spin_unlock_irq(&cmd->t_state_lock);
1798
1799         transport_generic_request_failure(cmd, ret);
1800 }
1801
1802 static int target_write_prot_action(struct se_cmd *cmd)
1803 {
1804         u32 sectors;
1805         /*
1806          * Perform WRITE_INSERT of PI using software emulation when backend
1807          * device has PI enabled, if the transport has not already generated
1808          * PI using hardware WRITE_INSERT offload.
1809          */
1810         switch (cmd->prot_op) {
1811         case TARGET_PROT_DOUT_INSERT:
1812                 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT))
1813                         sbc_dif_generate(cmd);
1814                 break;
1815         case TARGET_PROT_DOUT_STRIP:
1816                 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP)
1817                         break;
1818
1819                 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size);
1820                 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
1821                                              sectors, 0, cmd->t_prot_sg, 0);
1822                 if (unlikely(cmd->pi_err)) {
1823                         spin_lock_irq(&cmd->t_state_lock);
1824                         cmd->transport_state &= ~CMD_T_SENT;
1825                         spin_unlock_irq(&cmd->t_state_lock);
1826                         transport_generic_request_failure(cmd, cmd->pi_err);
1827                         return -1;
1828                 }
1829                 break;
1830         default:
1831                 break;
1832         }
1833
1834         return 0;
1835 }
1836
1837 static bool target_handle_task_attr(struct se_cmd *cmd)
1838 {
1839         struct se_device *dev = cmd->se_dev;
1840
1841         if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1842                 return false;
1843
1844         cmd->se_cmd_flags |= SCF_TASK_ATTR_SET;
1845
1846         /*
1847          * Check for the existence of HEAD_OF_QUEUE, and if true return 1
1848          * to allow the passed struct se_cmd list of tasks to the front of the list.
1849          */
1850         switch (cmd->sam_task_attr) {
1851         case TCM_HEAD_TAG:
1852                 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n",
1853                          cmd->t_task_cdb[0]);
1854                 return false;
1855         case TCM_ORDERED_TAG:
1856                 atomic_inc_mb(&dev->dev_ordered_sync);
1857
1858                 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n",
1859                          cmd->t_task_cdb[0]);
1860
1861                 /*
1862                  * Execute an ORDERED command if no other older commands
1863                  * exist that need to be completed first.
1864                  */
1865                 if (!atomic_read(&dev->simple_cmds))
1866                         return false;
1867                 break;
1868         default:
1869                 /*
1870                  * For SIMPLE and UNTAGGED Task Attribute commands
1871                  */
1872                 atomic_inc_mb(&dev->simple_cmds);
1873                 break;
1874         }
1875
1876         if (atomic_read(&dev->dev_ordered_sync) == 0)
1877                 return false;
1878
1879         spin_lock(&dev->delayed_cmd_lock);
1880         list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
1881         spin_unlock(&dev->delayed_cmd_lock);
1882
1883         pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn",
1884                 cmd->t_task_cdb[0], cmd->sam_task_attr);
1885         return true;
1886 }
1887
1888 static int __transport_check_aborted_status(struct se_cmd *, int);
1889
1890 void target_execute_cmd(struct se_cmd *cmd)
1891 {
1892         /*
1893          * Determine if frontend context caller is requesting the stopping of
1894          * this command for frontend exceptions.
1895          *
1896          * If the received CDB has aleady been aborted stop processing it here.
1897          */
1898         spin_lock_irq(&cmd->t_state_lock);
1899         if (__transport_check_aborted_status(cmd, 1)) {
1900                 spin_unlock_irq(&cmd->t_state_lock);
1901                 return;
1902         }
1903         if (cmd->transport_state & CMD_T_STOP) {
1904                 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
1905                         __func__, __LINE__, cmd->tag);
1906
1907                 spin_unlock_irq(&cmd->t_state_lock);
1908                 complete_all(&cmd->t_transport_stop_comp);
1909                 return;
1910         }
1911
1912         cmd->t_state = TRANSPORT_PROCESSING;
1913         cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
1914         spin_unlock_irq(&cmd->t_state_lock);
1915
1916         if (target_write_prot_action(cmd))
1917                 return;
1918
1919         if (target_handle_task_attr(cmd)) {
1920                 spin_lock_irq(&cmd->t_state_lock);
1921                 cmd->transport_state &= ~CMD_T_SENT;
1922                 spin_unlock_irq(&cmd->t_state_lock);
1923                 return;
1924         }
1925
1926         __target_execute_cmd(cmd, true);
1927 }
1928 EXPORT_SYMBOL(target_execute_cmd);
1929
1930 /*
1931  * Process all commands up to the last received ORDERED task attribute which
1932  * requires another blocking boundary
1933  */
1934 static void target_restart_delayed_cmds(struct se_device *dev)
1935 {
1936         for (;;) {
1937                 struct se_cmd *cmd;
1938
1939                 spin_lock(&dev->delayed_cmd_lock);
1940                 if (list_empty(&dev->delayed_cmd_list)) {
1941                         spin_unlock(&dev->delayed_cmd_lock);
1942                         break;
1943                 }
1944
1945                 cmd = list_entry(dev->delayed_cmd_list.next,
1946                                  struct se_cmd, se_delayed_node);
1947                 list_del(&cmd->se_delayed_node);
1948                 spin_unlock(&dev->delayed_cmd_lock);
1949
1950                 __target_execute_cmd(cmd, true);
1951
1952                 if (cmd->sam_task_attr == TCM_ORDERED_TAG)
1953                         break;
1954         }
1955 }
1956
1957 /*
1958  * Called from I/O completion to determine which dormant/delayed
1959  * and ordered cmds need to have their tasks added to the execution queue.
1960  */
1961 static void transport_complete_task_attr(struct se_cmd *cmd)
1962 {
1963         struct se_device *dev = cmd->se_dev;
1964
1965         if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1966                 return;
1967
1968         if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
1969                 goto restart;
1970
1971         if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
1972                 atomic_dec_mb(&dev->simple_cmds);
1973                 dev->dev_cur_ordered_id++;
1974         } else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
1975                 dev->dev_cur_ordered_id++;
1976                 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
1977                          dev->dev_cur_ordered_id);
1978         } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
1979                 atomic_dec_mb(&dev->dev_ordered_sync);
1980
1981                 dev->dev_cur_ordered_id++;
1982                 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n",
1983                          dev->dev_cur_ordered_id);
1984         }
1985 restart:
1986         target_restart_delayed_cmds(dev);
1987 }
1988
1989 static void transport_complete_qf(struct se_cmd *cmd)
1990 {
1991         int ret = 0;
1992
1993         transport_complete_task_attr(cmd);
1994         /*
1995          * If a fabric driver ->write_pending() or ->queue_data_in() callback
1996          * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and
1997          * the same callbacks should not be retried.  Return CHECK_CONDITION
1998          * if a scsi_status is not already set.
1999          *
2000          * If a fabric driver ->queue_status() has returned non zero, always
2001          * keep retrying no matter what..
2002          */
2003         if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) {
2004                 if (cmd->scsi_status)
2005                         goto queue_status;
2006
2007                 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
2008                 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
2009                 cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER;
2010                 translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
2011                 goto queue_status;
2012         }
2013
2014         if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
2015                 goto queue_status;
2016
2017         switch (cmd->data_direction) {
2018         case DMA_FROM_DEVICE:
2019                 if (cmd->scsi_status)
2020                         goto queue_status;
2021
2022                 trace_target_cmd_complete(cmd);
2023                 ret = cmd->se_tfo->queue_data_in(cmd);
2024                 break;
2025         case DMA_TO_DEVICE:
2026                 if (cmd->se_cmd_flags & SCF_BIDI) {
2027                         ret = cmd->se_tfo->queue_data_in(cmd);
2028                         break;
2029                 }
2030                 /* Fall through for DMA_TO_DEVICE */
2031         case DMA_NONE:
2032 queue_status:
2033                 trace_target_cmd_complete(cmd);
2034                 ret = cmd->se_tfo->queue_status(cmd);
2035                 break;
2036         default:
2037                 break;
2038         }
2039
2040         if (ret < 0) {
2041                 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2042                 return;
2043         }
2044         transport_lun_remove_cmd(cmd);
2045         transport_cmd_check_stop_to_fabric(cmd);
2046 }
2047
2048 static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev,
2049                                         int err, bool write_pending)
2050 {
2051         /*
2052          * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or
2053          * ->queue_data_in() callbacks from new process context.
2054          *
2055          * Otherwise for other errors, transport_complete_qf() will send
2056          * CHECK_CONDITION via ->queue_status() instead of attempting to
2057          * retry associated fabric driver data-transfer callbacks.
2058          */
2059         if (err == -EAGAIN || err == -ENOMEM) {
2060                 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP :
2061                                                  TRANSPORT_COMPLETE_QF_OK;
2062         } else {
2063                 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err);
2064                 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR;
2065         }
2066
2067         spin_lock_irq(&dev->qf_cmd_lock);
2068         list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
2069         atomic_inc_mb(&dev->dev_qf_count);
2070         spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
2071
2072         schedule_work(&cmd->se_dev->qf_work_queue);
2073 }
2074
2075 static bool target_read_prot_action(struct se_cmd *cmd)
2076 {
2077         switch (cmd->prot_op) {
2078         case TARGET_PROT_DIN_STRIP:
2079                 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) {
2080                         u32 sectors = cmd->data_length >>
2081                                   ilog2(cmd->se_dev->dev_attrib.block_size);
2082
2083                         cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
2084                                                      sectors, 0, cmd->t_prot_sg,
2085                                                      0);
2086                         if (cmd->pi_err)
2087                                 return true;
2088                 }
2089                 break;
2090         case TARGET_PROT_DIN_INSERT:
2091                 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT)
2092                         break;
2093
2094                 sbc_dif_generate(cmd);
2095                 break;
2096         default:
2097                 break;
2098         }
2099
2100         return false;
2101 }
2102
2103 static void target_complete_ok_work(struct work_struct *work)
2104 {
2105         struct se_cmd *cmd = container_of(work, struct se_cmd, work);
2106         int ret;
2107
2108         /*
2109          * Check if we need to move delayed/dormant tasks from cmds on the
2110          * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
2111          * Attribute.
2112          */
2113         transport_complete_task_attr(cmd);
2114
2115         /*
2116          * Check to schedule QUEUE_FULL work, or execute an existing
2117          * cmd->transport_qf_callback()
2118          */
2119         if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
2120                 schedule_work(&cmd->se_dev->qf_work_queue);
2121
2122         /*
2123          * Check if we need to send a sense buffer from
2124          * the struct se_cmd in question.
2125          */
2126         if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
2127                 WARN_ON(!cmd->scsi_status);
2128                 ret = transport_send_check_condition_and_sense(
2129                                         cmd, 0, 1);
2130                 if (ret)
2131                         goto queue_full;
2132
2133                 transport_lun_remove_cmd(cmd);
2134                 transport_cmd_check_stop_to_fabric(cmd);
2135                 return;
2136         }
2137         /*
2138          * Check for a callback, used by amongst other things
2139          * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation.
2140          */
2141         if (cmd->transport_complete_callback) {
2142                 sense_reason_t rc;
2143                 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE);
2144                 bool zero_dl = !(cmd->data_length);
2145                 int post_ret = 0;
2146
2147                 rc = cmd->transport_complete_callback(cmd, true, &post_ret);
2148                 if (!rc && !post_ret) {
2149                         if (caw && zero_dl)
2150                                 goto queue_rsp;
2151
2152                         return;
2153                 } else if (rc) {
2154                         ret = transport_send_check_condition_and_sense(cmd,
2155                                                 rc, 0);
2156                         if (ret)
2157                                 goto queue_full;
2158
2159                         transport_lun_remove_cmd(cmd);
2160                         transport_cmd_check_stop_to_fabric(cmd);
2161                         return;
2162                 }
2163         }
2164
2165 queue_rsp:
2166         switch (cmd->data_direction) {
2167         case DMA_FROM_DEVICE:
2168                 if (cmd->scsi_status)
2169                         goto queue_status;
2170
2171                 atomic_long_add(cmd->data_length,
2172                                 &cmd->se_lun->lun_stats.tx_data_octets);
2173                 /*
2174                  * Perform READ_STRIP of PI using software emulation when
2175                  * backend had PI enabled, if the transport will not be
2176                  * performing hardware READ_STRIP offload.
2177                  */
2178                 if (target_read_prot_action(cmd)) {
2179                         ret = transport_send_check_condition_and_sense(cmd,
2180                                                 cmd->pi_err, 0);
2181                         if (ret)
2182                                 goto queue_full;
2183
2184                         transport_lun_remove_cmd(cmd);
2185                         transport_cmd_check_stop_to_fabric(cmd);
2186                         return;
2187                 }
2188
2189                 trace_target_cmd_complete(cmd);
2190                 ret = cmd->se_tfo->queue_data_in(cmd);
2191                 if (ret)
2192                         goto queue_full;
2193                 break;
2194         case DMA_TO_DEVICE:
2195                 atomic_long_add(cmd->data_length,
2196                                 &cmd->se_lun->lun_stats.rx_data_octets);
2197                 /*
2198                  * Check if we need to send READ payload for BIDI-COMMAND
2199                  */
2200                 if (cmd->se_cmd_flags & SCF_BIDI) {
2201                         atomic_long_add(cmd->data_length,
2202                                         &cmd->se_lun->lun_stats.tx_data_octets);
2203                         ret = cmd->se_tfo->queue_data_in(cmd);
2204                         if (ret)
2205                                 goto queue_full;
2206                         break;
2207                 }
2208                 /* Fall through for DMA_TO_DEVICE */
2209         case DMA_NONE:
2210 queue_status:
2211                 trace_target_cmd_complete(cmd);
2212                 ret = cmd->se_tfo->queue_status(cmd);
2213                 if (ret)
2214                         goto queue_full;
2215                 break;
2216         default:
2217                 break;
2218         }
2219
2220         transport_lun_remove_cmd(cmd);
2221         transport_cmd_check_stop_to_fabric(cmd);
2222         return;
2223
2224 queue_full:
2225         pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
2226                 " data_direction: %d\n", cmd, cmd->data_direction);
2227
2228         transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2229 }
2230
2231 void target_free_sgl(struct scatterlist *sgl, int nents)
2232 {
2233         struct scatterlist *sg;
2234         int count;
2235
2236         for_each_sg(sgl, sg, nents, count)
2237                 __free_page(sg_page(sg));
2238
2239         kfree(sgl);
2240 }
2241 EXPORT_SYMBOL(target_free_sgl);
2242
2243 static inline void transport_reset_sgl_orig(struct se_cmd *cmd)
2244 {
2245         /*
2246          * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE
2247          * emulation, and free + reset pointers if necessary..
2248          */
2249         if (!cmd->t_data_sg_orig)
2250                 return;
2251
2252         kfree(cmd->t_data_sg);
2253         cmd->t_data_sg = cmd->t_data_sg_orig;
2254         cmd->t_data_sg_orig = NULL;
2255         cmd->t_data_nents = cmd->t_data_nents_orig;
2256         cmd->t_data_nents_orig = 0;
2257 }
2258
2259 static inline void transport_free_pages(struct se_cmd *cmd)
2260 {
2261         if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2262                 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents);
2263                 cmd->t_prot_sg = NULL;
2264                 cmd->t_prot_nents = 0;
2265         }
2266
2267         if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
2268                 /*
2269                  * Release special case READ buffer payload required for
2270                  * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE
2271                  */
2272                 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
2273                         target_free_sgl(cmd->t_bidi_data_sg,
2274                                            cmd->t_bidi_data_nents);
2275                         cmd->t_bidi_data_sg = NULL;
2276                         cmd->t_bidi_data_nents = 0;
2277                 }
2278                 transport_reset_sgl_orig(cmd);
2279                 return;
2280         }
2281         transport_reset_sgl_orig(cmd);
2282
2283         target_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
2284         cmd->t_data_sg = NULL;
2285         cmd->t_data_nents = 0;
2286
2287         target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
2288         cmd->t_bidi_data_sg = NULL;
2289         cmd->t_bidi_data_nents = 0;
2290 }
2291
2292 /**
2293  * transport_put_cmd - release a reference to a command
2294  * @cmd:       command to release
2295  *
2296  * This routine releases our reference to the command and frees it if possible.
2297  */
2298 static int transport_put_cmd(struct se_cmd *cmd)
2299 {
2300         BUG_ON(!cmd->se_tfo);
2301         /*
2302          * If this cmd has been setup with target_get_sess_cmd(), drop
2303          * the kref and call ->release_cmd() in kref callback.
2304          */
2305         return target_put_sess_cmd(cmd);
2306 }
2307
2308 void *transport_kmap_data_sg(struct se_cmd *cmd)
2309 {
2310         struct scatterlist *sg = cmd->t_data_sg;
2311         struct page **pages;
2312         int i;
2313
2314         /*
2315          * We need to take into account a possible offset here for fabrics like
2316          * tcm_loop who may be using a contig buffer from the SCSI midlayer for
2317          * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
2318          */
2319         if (!cmd->t_data_nents)
2320                 return NULL;
2321
2322         BUG_ON(!sg);
2323         if (cmd->t_data_nents == 1)
2324                 return kmap(sg_page(sg)) + sg->offset;
2325
2326         /* >1 page. use vmap */
2327         pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL);
2328         if (!pages)
2329                 return NULL;
2330
2331         /* convert sg[] to pages[] */
2332         for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
2333                 pages[i] = sg_page(sg);
2334         }
2335
2336         cmd->t_data_vmap = vmap(pages, cmd->t_data_nents,  VM_MAP, PAGE_KERNEL);
2337         kfree(pages);
2338         if (!cmd->t_data_vmap)
2339                 return NULL;
2340
2341         return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
2342 }
2343 EXPORT_SYMBOL(transport_kmap_data_sg);
2344
2345 void transport_kunmap_data_sg(struct se_cmd *cmd)
2346 {
2347         if (!cmd->t_data_nents) {
2348                 return;
2349         } else if (cmd->t_data_nents == 1) {
2350                 kunmap(sg_page(cmd->t_data_sg));
2351                 return;
2352         }
2353
2354         vunmap(cmd->t_data_vmap);
2355         cmd->t_data_vmap = NULL;
2356 }
2357 EXPORT_SYMBOL(transport_kunmap_data_sg);
2358
2359 int
2360 target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length,
2361                  bool zero_page, bool chainable)
2362 {
2363         struct scatterlist *sg;
2364         struct page *page;
2365         gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0;
2366         unsigned int nalloc, nent;
2367         int i = 0;
2368
2369         nalloc = nent = DIV_ROUND_UP(length, PAGE_SIZE);
2370         if (chainable)
2371                 nalloc++;
2372         sg = kmalloc_array(nalloc, sizeof(struct scatterlist), GFP_KERNEL);
2373         if (!sg)
2374                 return -ENOMEM;
2375
2376         sg_init_table(sg, nalloc);
2377
2378         while (length) {
2379                 u32 page_len = min_t(u32, length, PAGE_SIZE);
2380                 page = alloc_page(GFP_KERNEL | zero_flag);
2381                 if (!page)
2382                         goto out;
2383
2384                 sg_set_page(&sg[i], page, page_len, 0);
2385                 length -= page_len;
2386                 i++;
2387         }
2388         *sgl = sg;
2389         *nents = nent;
2390         return 0;
2391
2392 out:
2393         while (i > 0) {
2394                 i--;
2395                 __free_page(sg_page(&sg[i]));
2396         }
2397         kfree(sg);
2398         return -ENOMEM;
2399 }
2400 EXPORT_SYMBOL(target_alloc_sgl);
2401
2402 /*
2403  * Allocate any required resources to execute the command.  For writes we
2404  * might not have the payload yet, so notify the fabric via a call to
2405  * ->write_pending instead. Otherwise place it on the execution queue.
2406  */
2407 sense_reason_t
2408 transport_generic_new_cmd(struct se_cmd *cmd)
2409 {
2410         unsigned long flags;
2411         int ret = 0;
2412         bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);
2413
2414         if (cmd->prot_op != TARGET_PROT_NORMAL &&
2415             !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2416                 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents,
2417                                        cmd->prot_length, true, false);
2418                 if (ret < 0)
2419                         return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2420         }
2421
2422         /*
2423          * Determine is the TCM fabric module has already allocated physical
2424          * memory, and is directly calling transport_generic_map_mem_to_cmd()
2425          * beforehand.
2426          */
2427         if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
2428             cmd->data_length) {
2429
2430                 if ((cmd->se_cmd_flags & SCF_BIDI) ||
2431                     (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) {
2432                         u32 bidi_length;
2433
2434                         if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)
2435                                 bidi_length = cmd->t_task_nolb *
2436                                               cmd->se_dev->dev_attrib.block_size;
2437                         else
2438                                 bidi_length = cmd->data_length;
2439
2440                         ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
2441                                                &cmd->t_bidi_data_nents,
2442                                                bidi_length, zero_flag, false);
2443                         if (ret < 0)
2444                                 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2445                 }
2446
2447                 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents,
2448                                        cmd->data_length, zero_flag, false);
2449                 if (ret < 0)
2450                         return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2451         } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
2452                     cmd->data_length) {
2453                 /*
2454                  * Special case for COMPARE_AND_WRITE with fabrics
2455                  * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC.
2456                  */
2457                 u32 caw_length = cmd->t_task_nolb *
2458                                  cmd->se_dev->dev_attrib.block_size;
2459
2460                 ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
2461                                        &cmd->t_bidi_data_nents,
2462                                        caw_length, zero_flag, false);
2463                 if (ret < 0)
2464                         return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2465         }
2466         /*
2467          * If this command is not a write we can execute it right here,
2468          * for write buffers we need to notify the fabric driver first
2469          * and let it call back once the write buffers are ready.
2470          */
2471         target_add_to_state_list(cmd);
2472         if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) {
2473                 target_execute_cmd(cmd);
2474                 return 0;
2475         }
2476
2477         spin_lock_irqsave(&cmd->t_state_lock, flags);
2478         cmd->t_state = TRANSPORT_WRITE_PENDING;
2479         /*
2480          * Determine if frontend context caller is requesting the stopping of
2481          * this command for frontend exceptions.
2482          */
2483         if (cmd->transport_state & CMD_T_STOP) {
2484                 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2485                          __func__, __LINE__, cmd->tag);
2486
2487                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2488
2489                 complete_all(&cmd->t_transport_stop_comp);
2490                 return 0;
2491         }
2492         cmd->transport_state &= ~CMD_T_ACTIVE;
2493         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2494
2495         ret = cmd->se_tfo->write_pending(cmd);
2496         if (ret)
2497                 goto queue_full;
2498
2499         return 0;
2500
2501 queue_full:
2502         pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
2503         transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2504         return 0;
2505 }
2506 EXPORT_SYMBOL(transport_generic_new_cmd);
2507
2508 static void transport_write_pending_qf(struct se_cmd *cmd)
2509 {
2510         int ret;
2511
2512         ret = cmd->se_tfo->write_pending(cmd);
2513         if (ret) {
2514                 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
2515                          cmd);
2516                 transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2517         }
2518 }
2519
2520 static bool
2521 __transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *,
2522                            unsigned long *flags);
2523
2524 static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas)
2525 {
2526         unsigned long flags;
2527
2528         spin_lock_irqsave(&cmd->t_state_lock, flags);
2529         __transport_wait_for_tasks(cmd, true, aborted, tas, &flags);
2530         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2531 }
2532
2533 int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2534 {
2535         int ret = 0;
2536         bool aborted = false, tas = false;
2537
2538         if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
2539                 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2540                         target_wait_free_cmd(cmd, &aborted, &tas);
2541
2542                 if (!aborted || tas)
2543                         ret = transport_put_cmd(cmd);
2544         } else {
2545                 if (wait_for_tasks)
2546                         target_wait_free_cmd(cmd, &aborted, &tas);
2547                 /*
2548                  * Handle WRITE failure case where transport_generic_new_cmd()
2549                  * has already added se_cmd to state_list, but fabric has
2550                  * failed command before I/O submission.
2551                  */
2552                 if (cmd->state_active)
2553                         target_remove_from_state_list(cmd);
2554
2555                 if (cmd->se_lun)
2556                         transport_lun_remove_cmd(cmd);
2557
2558                 if (!aborted || tas)
2559                         ret = transport_put_cmd(cmd);
2560         }
2561         /*
2562          * If the task has been internally aborted due to TMR ABORT_TASK
2563          * or LUN_RESET, target_core_tmr.c is responsible for performing
2564          * the remaining calls to target_put_sess_cmd(), and not the
2565          * callers of this function.
2566          */
2567         if (aborted) {
2568                 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag);
2569                 wait_for_completion(&cmd->cmd_wait_comp);
2570                 cmd->se_tfo->release_cmd(cmd);
2571                 ret = 1;
2572         }
2573         return ret;
2574 }
2575 EXPORT_SYMBOL(transport_generic_free_cmd);
2576
2577 /* target_get_sess_cmd - Add command to active ->sess_cmd_list
2578  * @se_cmd:     command descriptor to add
2579  * @ack_kref:   Signal that fabric will perform an ack target_put_sess_cmd()
2580  */
2581 int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
2582 {
2583         struct se_session *se_sess = se_cmd->se_sess;
2584         unsigned long flags;
2585         int ret = 0;
2586
2587         /*
2588          * Add a second kref if the fabric caller is expecting to handle
2589          * fabric acknowledgement that requires two target_put_sess_cmd()
2590          * invocations before se_cmd descriptor release.
2591          */
2592         if (ack_kref) {
2593                 if (!kref_get_unless_zero(&se_cmd->cmd_kref))
2594                         return -EINVAL;
2595
2596                 se_cmd->se_cmd_flags |= SCF_ACK_KREF;
2597         }
2598
2599         spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2600         if (se_sess->sess_tearing_down) {
2601                 ret = -ESHUTDOWN;
2602                 goto out;
2603         }
2604         list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
2605 out:
2606         spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2607
2608         if (ret && ack_kref)
2609                 target_put_sess_cmd(se_cmd);
2610
2611         return ret;
2612 }
2613 EXPORT_SYMBOL(target_get_sess_cmd);
2614
2615 static void target_free_cmd_mem(struct se_cmd *cmd)
2616 {
2617         transport_free_pages(cmd);
2618
2619         if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
2620                 core_tmr_release_req(cmd->se_tmr_req);
2621         if (cmd->t_task_cdb != cmd->__t_task_cdb)
2622                 kfree(cmd->t_task_cdb);
2623 }
2624
2625 static void target_release_cmd_kref(struct kref *kref)
2626 {
2627         struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
2628         struct se_session *se_sess = se_cmd->se_sess;
2629         unsigned long flags;
2630         bool fabric_stop;
2631
2632         if (se_sess) {
2633                 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2634
2635                 spin_lock(&se_cmd->t_state_lock);
2636                 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) &&
2637                               (se_cmd->transport_state & CMD_T_ABORTED);
2638                 spin_unlock(&se_cmd->t_state_lock);
2639
2640                 if (se_cmd->cmd_wait_set || fabric_stop) {
2641                         list_del_init(&se_cmd->se_cmd_list);
2642                         spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2643                         target_free_cmd_mem(se_cmd);
2644                         complete(&se_cmd->cmd_wait_comp);
2645                         return;
2646                 }
2647                 list_del_init(&se_cmd->se_cmd_list);
2648                 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2649         }
2650
2651         target_free_cmd_mem(se_cmd);
2652         se_cmd->se_tfo->release_cmd(se_cmd);
2653 }
2654
2655 /**
2656  * target_put_sess_cmd - decrease the command reference count
2657  * @se_cmd:     command to drop a reference from
2658  *
2659  * Returns 1 if and only if this target_put_sess_cmd() call caused the
2660  * refcount to drop to zero. Returns zero otherwise.
2661  */
2662 int target_put_sess_cmd(struct se_cmd *se_cmd)
2663 {
2664         return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
2665 }
2666 EXPORT_SYMBOL(target_put_sess_cmd);
2667
2668 /* target_sess_cmd_list_set_waiting - Flag all commands in
2669  *         sess_cmd_list to complete cmd_wait_comp.  Set
2670  *         sess_tearing_down so no more commands are queued.
2671  * @se_sess:    session to flag
2672  */
2673 void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
2674 {
2675         struct se_cmd *se_cmd, *tmp_cmd;
2676         unsigned long flags;
2677         int rc;
2678
2679         spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2680         if (se_sess->sess_tearing_down) {
2681                 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2682                 return;
2683         }
2684         se_sess->sess_tearing_down = 1;
2685         list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
2686
2687         list_for_each_entry_safe(se_cmd, tmp_cmd,
2688                                  &se_sess->sess_wait_list, se_cmd_list) {
2689                 rc = kref_get_unless_zero(&se_cmd->cmd_kref);
2690                 if (rc) {
2691                         se_cmd->cmd_wait_set = 1;
2692                         spin_lock(&se_cmd->t_state_lock);
2693                         se_cmd->transport_state |= CMD_T_FABRIC_STOP;
2694                         spin_unlock(&se_cmd->t_state_lock);
2695                 } else
2696                         list_del_init(&se_cmd->se_cmd_list);
2697         }
2698
2699         spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2700 }
2701 EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
2702
2703 /* target_wait_for_sess_cmds - Wait for outstanding descriptors
2704  * @se_sess:    session to wait for active I/O
2705  */
2706 void target_wait_for_sess_cmds(struct se_session *se_sess)
2707 {
2708         struct se_cmd *se_cmd, *tmp_cmd;
2709         unsigned long flags;
2710         bool tas;
2711
2712         list_for_each_entry_safe(se_cmd, tmp_cmd,
2713                                 &se_sess->sess_wait_list, se_cmd_list) {
2714                 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
2715                         " %d\n", se_cmd, se_cmd->t_state,
2716                         se_cmd->se_tfo->get_cmd_state(se_cmd));
2717
2718                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
2719                 tas = (se_cmd->transport_state & CMD_T_TAS);
2720                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
2721
2722                 if (!target_put_sess_cmd(se_cmd)) {
2723                         if (tas)
2724                                 target_put_sess_cmd(se_cmd);
2725                 }
2726
2727                 wait_for_completion(&se_cmd->cmd_wait_comp);
2728                 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
2729                         " fabric state: %d\n", se_cmd, se_cmd->t_state,
2730                         se_cmd->se_tfo->get_cmd_state(se_cmd));
2731
2732                 se_cmd->se_tfo->release_cmd(se_cmd);
2733         }
2734
2735         spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2736         WARN_ON(!list_empty(&se_sess->sess_cmd_list));
2737         spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2738
2739 }
2740 EXPORT_SYMBOL(target_wait_for_sess_cmds);
2741
2742 static void target_lun_confirm(struct percpu_ref *ref)
2743 {
2744         struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
2745
2746         complete(&lun->lun_ref_comp);
2747 }
2748
2749 void transport_clear_lun_ref(struct se_lun *lun)
2750 {
2751         /*
2752          * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop
2753          * the initial reference and schedule confirm kill to be
2754          * executed after one full RCU grace period has completed.
2755          */
2756         percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm);
2757         /*
2758          * The first completion waits for percpu_ref_switch_to_atomic_rcu()
2759          * to call target_lun_confirm after lun->lun_ref has been marked
2760          * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t
2761          * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref
2762          * fails for all new incoming I/O.
2763          */
2764         wait_for_completion(&lun->lun_ref_comp);
2765         /*
2766          * The second completion waits for percpu_ref_put_many() to
2767          * invoke ->release() after lun->lun_ref has switched to
2768          * atomic_t mode, and lun->lun_ref.count has reached zero.
2769          *
2770          * At this point all target-core lun->lun_ref references have
2771          * been dropped via transport_lun_remove_cmd(), and it's safe
2772          * to proceed with the remaining LUN shutdown.
2773          */
2774         wait_for_completion(&lun->lun_shutdown_comp);
2775 }
2776
2777 static bool
2778 __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop,
2779                            bool *aborted, bool *tas, unsigned long *flags)
2780         __releases(&cmd->t_state_lock)
2781         __acquires(&cmd->t_state_lock)
2782 {
2783
2784         assert_spin_locked(&cmd->t_state_lock);
2785         WARN_ON_ONCE(!irqs_disabled());
2786
2787         if (fabric_stop)
2788                 cmd->transport_state |= CMD_T_FABRIC_STOP;
2789
2790         if (cmd->transport_state & CMD_T_ABORTED)
2791                 *aborted = true;
2792
2793         if (cmd->transport_state & CMD_T_TAS)
2794                 *tas = true;
2795
2796         if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
2797             !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2798                 return false;
2799
2800         if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
2801             !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2802                 return false;
2803
2804         if (!(cmd->transport_state & CMD_T_ACTIVE))
2805                 return false;
2806
2807         if (fabric_stop && *aborted)
2808                 return false;
2809
2810         cmd->transport_state |= CMD_T_STOP;
2811
2812         pr_debug("wait_for_tasks: Stopping %p ITT: 0x%08llx i_state: %d,"
2813                  " t_state: %d, CMD_T_STOP\n", cmd, cmd->tag,
2814                  cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
2815
2816         spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
2817
2818         wait_for_completion(&cmd->t_transport_stop_comp);
2819
2820         spin_lock_irqsave(&cmd->t_state_lock, *flags);
2821         cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
2822
2823         pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->"
2824                  "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag);
2825
2826         return true;
2827 }
2828
2829 /**
2830  * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp
2831  * @cmd: command to wait on
2832  */
2833 bool transport_wait_for_tasks(struct se_cmd *cmd)
2834 {
2835         unsigned long flags;
2836         bool ret, aborted = false, tas = false;
2837
2838         spin_lock_irqsave(&cmd->t_state_lock, flags);
2839         ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags);
2840         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2841
2842         return ret;
2843 }
2844 EXPORT_SYMBOL(transport_wait_for_tasks);
2845
2846 struct sense_info {
2847         u8 key;
2848         u8 asc;
2849         u8 ascq;
2850         bool add_sector_info;
2851 };
2852
2853 static const struct sense_info sense_info_table[] = {
2854         [TCM_NO_SENSE] = {
2855                 .key = NOT_READY
2856         },
2857         [TCM_NON_EXISTENT_LUN] = {
2858                 .key = ILLEGAL_REQUEST,
2859                 .asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */
2860         },
2861         [TCM_UNSUPPORTED_SCSI_OPCODE] = {
2862                 .key = ILLEGAL_REQUEST,
2863                 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */
2864         },
2865         [TCM_SECTOR_COUNT_TOO_MANY] = {
2866                 .key = ILLEGAL_REQUEST,
2867                 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */
2868         },
2869         [TCM_UNKNOWN_MODE_PAGE] = {
2870                 .key = ILLEGAL_REQUEST,
2871                 .asc = 0x24, /* INVALID FIELD IN CDB */
2872         },
2873         [TCM_CHECK_CONDITION_ABORT_CMD] = {
2874                 .key = ABORTED_COMMAND,
2875                 .asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */
2876                 .ascq = 0x03,
2877         },
2878         [TCM_INCORRECT_AMOUNT_OF_DATA] = {
2879                 .key = ABORTED_COMMAND,
2880                 .asc = 0x0c, /* WRITE ERROR */
2881                 .ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */
2882         },
2883         [TCM_INVALID_CDB_FIELD] = {
2884                 .key = ILLEGAL_REQUEST,
2885                 .asc = 0x24, /* INVALID FIELD IN CDB */
2886         },
2887         [TCM_INVALID_PARAMETER_LIST] = {
2888                 .key = ILLEGAL_REQUEST,
2889                 .asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */
2890         },
2891         [TCM_TOO_MANY_TARGET_DESCS] = {
2892                 .key = ILLEGAL_REQUEST,
2893                 .asc = 0x26,
2894                 .ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */
2895         },
2896         [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = {
2897                 .key = ILLEGAL_REQUEST,
2898                 .asc = 0x26,
2899                 .ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */
2900         },
2901         [TCM_TOO_MANY_SEGMENT_DESCS] = {
2902                 .key = ILLEGAL_REQUEST,
2903                 .asc = 0x26,
2904                 .ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */
2905         },
2906         [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = {
2907                 .key = ILLEGAL_REQUEST,
2908                 .asc = 0x26,
2909                 .ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */
2910         },
2911         [TCM_PARAMETER_LIST_LENGTH_ERROR] = {
2912                 .key = ILLEGAL_REQUEST,
2913                 .asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */
2914         },
2915         [TCM_UNEXPECTED_UNSOLICITED_DATA] = {
2916                 .key = ILLEGAL_REQUEST,
2917                 .asc = 0x0c, /* WRITE ERROR */
2918                 .ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */
2919         },
2920         [TCM_SERVICE_CRC_ERROR] = {
2921                 .key = ABORTED_COMMAND,
2922                 .asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */
2923                 .ascq = 0x05, /* N/A */
2924         },
2925         [TCM_SNACK_REJECTED] = {
2926                 .key = ABORTED_COMMAND,
2927                 .asc = 0x11, /* READ ERROR */
2928                 .ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */
2929         },
2930         [TCM_WRITE_PROTECTED] = {
2931                 .key = DATA_PROTECT,
2932                 .asc = 0x27, /* WRITE PROTECTED */
2933         },
2934         [TCM_ADDRESS_OUT_OF_RANGE] = {
2935                 .key = ILLEGAL_REQUEST,
2936                 .asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */
2937         },
2938         [TCM_CHECK_CONDITION_UNIT_ATTENTION] = {
2939                 .key = UNIT_ATTENTION,
2940         },
2941         [TCM_CHECK_CONDITION_NOT_READY] = {
2942                 .key = NOT_READY,
2943         },
2944         [TCM_MISCOMPARE_VERIFY] = {
2945                 .key = MISCOMPARE,
2946                 .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */
2947                 .ascq = 0x00,
2948         },
2949         [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = {
2950                 .key = ABORTED_COMMAND,
2951                 .asc = 0x10,
2952                 .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */
2953                 .add_sector_info = true,
2954         },
2955         [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = {
2956                 .key = ABORTED_COMMAND,
2957                 .asc = 0x10,
2958                 .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */
2959                 .add_sector_info = true,
2960         },
2961         [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = {
2962                 .key = ABORTED_COMMAND,
2963                 .asc = 0x10,
2964                 .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */
2965                 .add_sector_info = true,
2966         },
2967         [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = {
2968                 .key = COPY_ABORTED,
2969                 .asc = 0x0d,
2970                 .ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */
2971
2972         },
2973         [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = {
2974                 /*
2975                  * Returning ILLEGAL REQUEST would cause immediate IO errors on
2976                  * Solaris initiators.  Returning NOT READY instead means the
2977                  * operations will be retried a finite number of times and we
2978                  * can survive intermittent errors.
2979                  */
2980                 .key = NOT_READY,
2981                 .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */
2982         },
2983 };
2984
2985 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
2986 {
2987         const struct sense_info *si;
2988         u8 *buffer = cmd->sense_buffer;
2989         int r = (__force int)reason;
2990         u8 asc, ascq;
2991         bool desc_format = target_sense_desc_format(cmd->se_dev);
2992
2993         if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key)
2994                 si = &sense_info_table[r];
2995         else
2996                 si = &sense_info_table[(__force int)
2997                                        TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE];
2998
2999         if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) {
3000                 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
3001                 WARN_ON_ONCE(asc == 0);
3002         } else if (si->asc == 0) {
3003                 WARN_ON_ONCE(cmd->scsi_asc == 0);
3004                 asc = cmd->scsi_asc;
3005                 ascq = cmd->scsi_ascq;
3006         } else {
3007                 asc = si->asc;
3008                 ascq = si->ascq;
3009         }
3010
3011         scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq);
3012         if (si->add_sector_info)
3013                 return scsi_set_sense_information(buffer,
3014                                                   cmd->scsi_sense_length,
3015                                                   cmd->bad_sector);
3016
3017         return 0;
3018 }
3019
3020 int
3021 transport_send_check_condition_and_sense(struct se_cmd *cmd,
3022                 sense_reason_t reason, int from_transport)
3023 {
3024         unsigned long flags;
3025
3026         spin_lock_irqsave(&cmd->t_state_lock, flags);
3027         if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
3028                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3029                 return 0;
3030         }
3031         cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
3032         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3033
3034         if (!from_transport) {
3035                 int rc;
3036
3037                 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
3038                 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
3039                 cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER;
3040                 rc = translate_sense_reason(cmd, reason);
3041                 if (rc)
3042                         return rc;
3043         }
3044
3045         trace_target_cmd_complete(cmd);
3046         return cmd->se_tfo->queue_status(cmd);
3047 }
3048 EXPORT_SYMBOL(transport_send_check_condition_and_sense);
3049
3050 static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3051         __releases(&cmd->t_state_lock)
3052         __acquires(&cmd->t_state_lock)
3053 {
3054         int ret;
3055
3056         assert_spin_locked(&cmd->t_state_lock);
3057         WARN_ON_ONCE(!irqs_disabled());
3058
3059         if (!(cmd->transport_state & CMD_T_ABORTED))
3060                 return 0;
3061         /*
3062          * If cmd has been aborted but either no status is to be sent or it has
3063          * already been sent, just return
3064          */
3065         if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) {
3066                 if (send_status)
3067                         cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS;
3068                 return 1;
3069         }
3070
3071         pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:"
3072                 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag);
3073
3074         cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS;
3075         cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3076         trace_target_cmd_complete(cmd);
3077
3078         spin_unlock_irq(&cmd->t_state_lock);
3079         ret = cmd->se_tfo->queue_status(cmd);
3080         if (ret)
3081                 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
3082         spin_lock_irq(&cmd->t_state_lock);
3083
3084         return 1;
3085 }
3086
3087 int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3088 {
3089         int ret;
3090
3091         spin_lock_irq(&cmd->t_state_lock);
3092         ret = __transport_check_aborted_status(cmd, send_status);
3093         spin_unlock_irq(&cmd->t_state_lock);
3094
3095         return ret;
3096 }
3097 EXPORT_SYMBOL(transport_check_aborted_status);
3098
3099 void transport_send_task_abort(struct se_cmd *cmd)
3100 {
3101         unsigned long flags;
3102         int ret;
3103
3104         spin_lock_irqsave(&cmd->t_state_lock, flags);
3105         if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) {
3106                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3107                 return;
3108         }
3109         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3110
3111         /*
3112          * If there are still expected incoming fabric WRITEs, we wait
3113          * until until they have completed before sending a TASK_ABORTED
3114          * response.  This response with TASK_ABORTED status will be
3115          * queued back to fabric module by transport_check_aborted_status().
3116          */
3117         if (cmd->data_direction == DMA_TO_DEVICE) {
3118                 if (cmd->se_tfo->write_pending_status(cmd) != 0) {
3119                         spin_lock_irqsave(&cmd->t_state_lock, flags);
3120                         if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) {
3121                                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3122                                 goto send_abort;
3123                         }
3124                         cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS;
3125                         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3126                         return;
3127                 }
3128         }
3129 send_abort:
3130         cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3131
3132         transport_lun_remove_cmd(cmd);
3133
3134         pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
3135                  cmd->t_task_cdb[0], cmd->tag);
3136
3137         trace_target_cmd_complete(cmd);
3138         ret = cmd->se_tfo->queue_status(cmd);
3139         if (ret)
3140                 transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
3141 }
3142
3143 static void target_tmr_work(struct work_struct *work)
3144 {
3145         struct se_cmd *cmd = container_of(work, struct se_cmd, work);
3146         struct se_device *dev = cmd->se_dev;
3147         struct se_tmr_req *tmr = cmd->se_tmr_req;
3148         unsigned long flags;
3149         int ret;
3150
3151         spin_lock_irqsave(&cmd->t_state_lock, flags);
3152         if (cmd->transport_state & CMD_T_ABORTED) {
3153                 tmr->response = TMR_FUNCTION_REJECTED;
3154                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3155                 goto check_stop;
3156         }
3157         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3158
3159         switch (tmr->function) {
3160         case TMR_ABORT_TASK:
3161                 core_tmr_abort_task(dev, tmr, cmd->se_sess);
3162                 break;
3163         case TMR_ABORT_TASK_SET:
3164         case TMR_CLEAR_ACA:
3165         case TMR_CLEAR_TASK_SET:
3166                 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
3167                 break;
3168         case TMR_LUN_RESET:
3169                 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
3170                 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
3171                                          TMR_FUNCTION_REJECTED;
3172                 if (tmr->response == TMR_FUNCTION_COMPLETE) {
3173                         target_ua_allocate_lun(cmd->se_sess->se_node_acl,
3174                                                cmd->orig_fe_lun, 0x29,
3175                                                ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED);
3176                 }
3177                 break;
3178         case TMR_TARGET_WARM_RESET:
3179                 tmr->response = TMR_FUNCTION_REJECTED;
3180                 break;
3181         case TMR_TARGET_COLD_RESET:
3182                 tmr->response = TMR_FUNCTION_REJECTED;
3183                 break;
3184         default:
3185                 pr_err("Uknown TMR function: 0x%02x.\n",
3186                                 tmr->function);
3187                 tmr->response = TMR_FUNCTION_REJECTED;
3188                 break;
3189         }
3190
3191         spin_lock_irqsave(&cmd->t_state_lock, flags);
3192         if (cmd->transport_state & CMD_T_ABORTED) {
3193                 spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3194                 goto check_stop;
3195         }
3196         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3197
3198         cmd->se_tfo->queue_tm_rsp(cmd);
3199
3200 check_stop:
3201         transport_cmd_check_stop_to_fabric(cmd);
3202 }
3203
3204 int transport_generic_handle_tmr(
3205         struct se_cmd *cmd)
3206 {
3207         unsigned long flags;
3208         bool aborted = false;
3209
3210         spin_lock_irqsave(&cmd->t_state_lock, flags);
3211         if (cmd->transport_state & CMD_T_ABORTED) {
3212                 aborted = true;
3213         } else {
3214                 cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
3215                 cmd->transport_state |= CMD_T_ACTIVE;
3216         }
3217         spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3218
3219         if (aborted) {
3220                 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d"
3221                         "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function,
3222                         cmd->se_tmr_req->ref_task_tag, cmd->tag);
3223                 transport_cmd_check_stop_to_fabric(cmd);
3224                 return 0;
3225         }
3226
3227         INIT_WORK(&cmd->work, target_tmr_work);
3228         queue_work(cmd->se_dev->tmr_wq, &cmd->work);
3229         return 0;
3230 }
3231 EXPORT_SYMBOL(transport_generic_handle_tmr);
3232
3233 bool
3234 target_check_wce(struct se_device *dev)
3235 {
3236         bool wce = false;
3237
3238         if (dev->transport->get_write_cache)
3239                 wce = dev->transport->get_write_cache(dev);
3240         else if (dev->dev_attrib.emulate_write_cache > 0)
3241                 wce = true;
3242
3243         return wce;
3244 }
3245
3246 bool
3247 target_check_fua(struct se_device *dev)
3248 {
3249         return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0;
3250 }