drivers/scsi/lpfc/lpfc_nvmet.c

   1 /*******************************************************************
   2  * This file is part of the Emulex Linux Device Driver for         *
   3  * Fibre Channsel Host Bus Adapters.                               *
   4  * Copyright (C) 2017 Broadcom. All Rights Reserved. The term      *
   5  * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
   6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
   7  * EMULEX and SLI are trademarks of Emulex.                        *
   8  * www.broadcom.com                                                *
   9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
  10  *                                                                 *
  11  * This program is free software; you can redistribute it and/or   *
  12  * modify it under the terms of version 2 of the GNU General       *
  13  * Public License as published by the Free Software Foundation.    *
  14  * This program is distributed in the hope that it will be useful. *
  15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
  16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
  17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
  18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
  20  * more details, a copy of which can be found in the file COPYING  *
  21  * included with this package.                                     *
  22  ********************************************************************/
  23 #include <linux/pci.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/delay.h>
  27 #include <asm/unaligned.h>
  28 #include <linux/crc-t10dif.h>
  29 #include <net/checksum.h>
  30
  31 #include <scsi/scsi.h>
  32 #include <scsi/scsi_device.h>
  33 #include <scsi/scsi_eh.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi_tcq.h>
  36 #include <scsi/scsi_transport_fc.h>
  37 #include <scsi/fc/fc_fs.h>
  38
  39 #include <../drivers/nvme/host/nvme.h>
  40 #include <linux/nvme-fc-driver.h>
  41
  42 #include "lpfc_version.h"
  43 #include "lpfc_hw4.h"
  44 #include "lpfc_hw.h"
  45 #include "lpfc_sli.h"
  46 #include "lpfc_sli4.h"
  47 #include "lpfc_nl.h"
  48 #include "lpfc_disc.h"
  49 #include "lpfc.h"
  50 #include "lpfc_scsi.h"
  51 #include "lpfc_nvme.h"
  52 #include "lpfc_nvmet.h"
  53 #include "lpfc_logmsg.h"
  54 #include "lpfc_crtn.h"
  55 #include "lpfc_vport.h"
  56 #include "lpfc_debugfs.h"
  57
  58 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
  59                                                  struct lpfc_nvmet_rcv_ctx *,
  60                                                  dma_addr_t rspbuf,
  61                                                  uint16_t rspsize);
  62 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
  63                                                   struct lpfc_nvmet_rcv_ctx *);
  64 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
  65                                           struct lpfc_nvmet_rcv_ctx *,
  66                                           uint32_t, uint16_t);
  67 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
  68                                             struct lpfc_nvmet_rcv_ctx *,
  69                                             uint32_t, uint16_t);
  70 static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *,
  71                                            struct lpfc_nvmet_rcv_ctx *,
  72                                            uint32_t, uint16_t);
  73
  74 void
  75 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp)
  76 {
  77         unsigned long iflag;
  78
  79         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
  80                         "6313 NVMET Defer ctx release xri x%x flg x%x\n",
  81                         ctxp->oxid, ctxp->flag);
  82
  83         spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
  84         if (ctxp->flag & LPFC_NVMET_CTX_RLS) {
  85                 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock,
  86                                        iflag);
  87                 return;
  88         }
  89         ctxp->flag |= LPFC_NVMET_CTX_RLS;
  90         list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
  91         spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
  92 }
  93
  94 /**
  95  * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
  96  * @phba: Pointer to HBA context object.
  97  * @cmdwqe: Pointer to driver command WQE object.
  98  * @wcqe: Pointer to driver response CQE object.
  99  *
 100  * The function is called from SLI ring event handler with no
 101  * lock held. This function is the completion handler for NVME LS commands
 102  * The function frees memory resources used for the NVME commands.
 103  **/
 104 static void
 105 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 106                           struct lpfc_wcqe_complete *wcqe)
 107 {
 108         struct lpfc_nvmet_tgtport *tgtp;
 109         struct nvmefc_tgt_ls_req *rsp;
 110         struct lpfc_nvmet_rcv_ctx *ctxp;
 111         uint32_t status, result;
 112
 113         status = bf_get(lpfc_wcqe_c_status, wcqe);
 114         result = wcqe->parameter;
 115         ctxp = cmdwqe->context2;
 116
 117         if (ctxp->state != LPFC_NVMET_STE_LS_RSP || ctxp->entry_cnt != 2) {
 118                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 119                                 "6410 NVMET LS cmpl state mismatch IO x%x: "
 120                                 "%d %d\n",
 121                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
 122         }
 123
 124         if (!phba->targetport)
 125                 goto out;
 126
 127         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 128
 129         if (status)
 130                 atomic_inc(&tgtp->xmt_ls_rsp_error);
 131         else
 132                 atomic_inc(&tgtp->xmt_ls_rsp_cmpl);
 133
 134 out:
 135         rsp = &ctxp->ctx.ls_req;
 136
 137         lpfc_nvmeio_data(phba, "NVMET LS  CMPL: xri x%x stat x%x result x%x\n",
 138                          ctxp->oxid, status, result);
 139
 140         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 141                         "6038 NVMET LS rsp cmpl: %d %d oxid x%x\n",
 142                         status, result, ctxp->oxid);
 143
 144         lpfc_nlp_put(cmdwqe->context1);
 145         cmdwqe->context2 = NULL;
 146         cmdwqe->context3 = NULL;
 147         lpfc_sli_release_iocbq(phba, cmdwqe);
 148         rsp->done(rsp);
 149         kfree(ctxp);
 150 }
 151
 152 /**
 153  * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context
 154  * @phba: HBA buffer is associated with
 155  * @ctxp: context to clean up
 156  * @mp: Buffer to free
 157  *
 158  * Description: Frees the given DMA buffer in the appropriate way given by
 159  * reposting it to its associated RQ so it can be reused.
 160  *
 161  * Notes: Takes phba->hbalock.  Can be called with or without other locks held.
 162  *
 163  * Returns: None
 164  **/
 165 void
 166 lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 167 {
 168 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
 169         struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context;
 170         struct lpfc_nvmet_tgtport *tgtp;
 171         struct fc_frame_header *fc_hdr;
 172         struct rqb_dmabuf *nvmebuf;
 173         uint32_t *payload;
 174         uint32_t size, oxid, sid, rc;
 175         unsigned long iflag;
 176
 177         if (ctxp->txrdy) {
 178                 pci_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
 179                               ctxp->txrdy_phys);
 180                 ctxp->txrdy = NULL;
 181                 ctxp->txrdy_phys = 0;
 182         }
 183
 184         if (ctxp->state == LPFC_NVMET_STE_FREE) {
 185                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 186                                 "6411 NVMET free, already free IO x%x: %d %d\n",
 187                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
 188         }
 189         ctxp->state = LPFC_NVMET_STE_FREE;
 190
 191         spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
 192         if (phba->sli4_hba.nvmet_io_wait_cnt) {
 193                 list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list,
 194                                  nvmebuf, struct rqb_dmabuf,
 195                                  hbuf.list);
 196                 phba->sli4_hba.nvmet_io_wait_cnt--;
 197                 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
 198                                        iflag);
 199
 200                 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
 201                 oxid = be16_to_cpu(fc_hdr->fh_ox_id);
 202                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 203                 payload = (uint32_t *)(nvmebuf->dbuf.virt);
 204                 size = nvmebuf->bytes_recv;
 205                 sid = sli4_sid_from_fc_hdr(fc_hdr);
 206
 207                 ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
 208                 ctxp->wqeq = NULL;
 209                 ctxp->txrdy = NULL;
 210                 ctxp->offset = 0;
 211                 ctxp->phba = phba;
 212                 ctxp->size = size;
 213                 ctxp->oxid = oxid;
 214                 ctxp->sid = sid;
 215                 ctxp->state = LPFC_NVMET_STE_RCV;
 216                 ctxp->entry_cnt = 1;
 217                 ctxp->flag = 0;
 218                 ctxp->ctxbuf = ctx_buf;
 219                 spin_lock_init(&ctxp->ctxlock);
 220
 221 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 222                 if (phba->ktime_on) {
 223                         ctxp->ts_cmd_nvme = ktime_get_ns();
 224                         ctxp->ts_isr_cmd = ctxp->ts_cmd_nvme;
 225                         ctxp->ts_nvme_data = 0;
 226                         ctxp->ts_data_wqput = 0;
 227                         ctxp->ts_isr_data = 0;
 228                         ctxp->ts_data_nvme = 0;
 229                         ctxp->ts_nvme_status = 0;
 230                         ctxp->ts_status_wqput = 0;
 231                         ctxp->ts_isr_status = 0;
 232                         ctxp->ts_status_nvme = 0;
 233                 }
 234 #endif
 235                 atomic_inc(&tgtp->rcv_fcp_cmd_in);
 236                 /*
 237                  * The calling sequence should be:
 238                  * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done
 239                  * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
 240                  * When we return from nvmet_fc_rcv_fcp_req, all relevant info
 241                  * the NVME command / FC header is stored.
 242                  * A buffer has already been reposted for this IO, so just free
 243                  * the nvmebuf.
 244                  */
 245                 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
 246                                           payload, size);
 247
 248                 /* Process FCP command */
 249                 if (rc == 0) {
 250                         atomic_inc(&tgtp->rcv_fcp_cmd_out);
 251                         nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
 252                         return;
 253                 }
 254
 255                 atomic_inc(&tgtp->rcv_fcp_cmd_drop);
 256                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 257                                 "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
 258                                 ctxp->oxid, rc,
 259                                 atomic_read(&tgtp->rcv_fcp_cmd_in),
 260                                 atomic_read(&tgtp->rcv_fcp_cmd_out),
 261                                 atomic_read(&tgtp->xmt_fcp_release));
 262
 263                 lpfc_nvmet_defer_release(phba, ctxp);
 264                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
 265                 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
 266                 return;
 267         }
 268         spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
 269
 270         spin_lock_irqsave(&phba->sli4_hba.nvmet_io_lock, iflag);
 271         list_add_tail(&ctx_buf->list,
 272                       &phba->sli4_hba.lpfc_nvmet_ctx_list);
 273         phba->sli4_hba.nvmet_ctx_cnt++;
 274         spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_lock, iflag);
 275 #endif
 276 }
 277
 278 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 279 static void
 280 lpfc_nvmet_ktime(struct lpfc_hba *phba,
 281                  struct lpfc_nvmet_rcv_ctx *ctxp)
 282 {
 283         uint64_t seg1, seg2, seg3, seg4, seg5;
 284         uint64_t seg6, seg7, seg8, seg9, seg10;
 285
 286         if (!phba->ktime_on)
 287                 return;
 288
 289         if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
 290             !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
 291             !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
 292             !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
 293             !ctxp->ts_isr_status || !ctxp->ts_status_nvme)
 294                 return;
 295
 296         if (ctxp->ts_isr_cmd  > ctxp->ts_cmd_nvme)
 297                 return;
 298         if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
 299                 return;
 300         if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
 301                 return;
 302         if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
 303                 return;
 304         if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
 305                 return;
 306         if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
 307                 return;
 308         if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
 309                 return;
 310         if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
 311                 return;
 312         if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
 313                 return;
 314         /*
 315          * Segment 1 - Time from FCP command received by MSI-X ISR
 316          * to FCP command is passed to NVME Layer.
 317          * Segment 2 - Time from FCP command payload handed
 318          * off to NVME Layer to Driver receives a Command op
 319          * from NVME Layer.
 320          * Segment 3 - Time from Driver receives a Command op
 321          * from NVME Layer to Command is put on WQ.
 322          * Segment 4 - Time from Driver WQ put is done
 323          * to MSI-X ISR for Command cmpl.
 324          * Segment 5 - Time from MSI-X ISR for Command cmpl to
 325          * Command cmpl is passed to NVME Layer.
 326          * Segment 6 - Time from Command cmpl is passed to NVME
 327          * Layer to Driver receives a RSP op from NVME Layer.
 328          * Segment 7 - Time from Driver receives a RSP op from
 329          * NVME Layer to WQ put is done on TRSP FCP Status.
 330          * Segment 8 - Time from Driver WQ put is done on TRSP
 331          * FCP Status to MSI-X ISR for TRSP cmpl.
 332          * Segment 9 - Time from MSI-X ISR for TRSP cmpl to
 333          * TRSP cmpl is passed to NVME Layer.
 334          * Segment 10 - Time from FCP command received by
 335          * MSI-X ISR to command is completed on wire.
 336          * (Segments 1 thru 8) for READDATA / WRITEDATA
 337          * (Segments 1 thru 4) for READDATA_RSP
 338          */
 339         seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
 340         seg2 = (ctxp->ts_nvme_data - ctxp->ts_isr_cmd) - seg1;
 341         seg3 = (ctxp->ts_data_wqput - ctxp->ts_isr_cmd) -
 342                 seg1 - seg2;
 343         seg4 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd) -
 344                 seg1 - seg2 - seg3;
 345         seg5 = (ctxp->ts_data_nvme - ctxp->ts_isr_cmd) -
 346                 seg1 - seg2 - seg3 - seg4;
 347
 348         /* For auto rsp commands seg6 thru seg10 will be 0 */
 349         if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
 350                 seg6 = (ctxp->ts_nvme_status -
 351                         ctxp->ts_isr_cmd) -
 352                         seg1 - seg2 - seg3 - seg4 - seg5;
 353                 seg7 = (ctxp->ts_status_wqput -
 354                         ctxp->ts_isr_cmd) -
 355                         seg1 - seg2 - seg3 -
 356                         seg4 - seg5 - seg6;
 357                 seg8 = (ctxp->ts_isr_status -
 358                         ctxp->ts_isr_cmd) -
 359                         seg1 - seg2 - seg3 - seg4 -
 360                         seg5 - seg6 - seg7;
 361                 seg9 = (ctxp->ts_status_nvme -
 362                         ctxp->ts_isr_cmd) -
 363                         seg1 - seg2 - seg3 - seg4 -
 364                         seg5 - seg6 - seg7 - seg8;
 365                 seg10 = (ctxp->ts_isr_status -
 366                         ctxp->ts_isr_cmd);
 367         } else {
 368                 seg6 =  0;
 369                 seg7 =  0;
 370                 seg8 =  0;
 371                 seg9 =  0;
 372                 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
 373         }
 374
 375         phba->ktime_seg1_total += seg1;
 376         if (seg1 < phba->ktime_seg1_min)
 377                 phba->ktime_seg1_min = seg1;
 378         else if (seg1 > phba->ktime_seg1_max)
 379                 phba->ktime_seg1_max = seg1;
 380
 381         phba->ktime_seg2_total += seg2;
 382         if (seg2 < phba->ktime_seg2_min)
 383                 phba->ktime_seg2_min = seg2;
 384         else if (seg2 > phba->ktime_seg2_max)
 385                 phba->ktime_seg2_max = seg2;
 386
 387         phba->ktime_seg3_total += seg3;
 388         if (seg3 < phba->ktime_seg3_min)
 389                 phba->ktime_seg3_min = seg3;
 390         else if (seg3 > phba->ktime_seg3_max)
 391                 phba->ktime_seg3_max = seg3;
 392
 393         phba->ktime_seg4_total += seg4;
 394         if (seg4 < phba->ktime_seg4_min)
 395                 phba->ktime_seg4_min = seg4;
 396         else if (seg4 > phba->ktime_seg4_max)
 397                 phba->ktime_seg4_max = seg4;
 398
 399         phba->ktime_seg5_total += seg5;
 400         if (seg5 < phba->ktime_seg5_min)
 401                 phba->ktime_seg5_min = seg5;
 402         else if (seg5 > phba->ktime_seg5_max)
 403                 phba->ktime_seg5_max = seg5;
 404
 405         phba->ktime_data_samples++;
 406         if (!seg6)
 407                 goto out;
 408
 409         phba->ktime_seg6_total += seg6;
 410         if (seg6 < phba->ktime_seg6_min)
 411                 phba->ktime_seg6_min = seg6;
 412         else if (seg6 > phba->ktime_seg6_max)
 413                 phba->ktime_seg6_max = seg6;
 414
 415         phba->ktime_seg7_total += seg7;
 416         if (seg7 < phba->ktime_seg7_min)
 417                 phba->ktime_seg7_min = seg7;
 418         else if (seg7 > phba->ktime_seg7_max)
 419                 phba->ktime_seg7_max = seg7;
 420
 421         phba->ktime_seg8_total += seg8;
 422         if (seg8 < phba->ktime_seg8_min)
 423                 phba->ktime_seg8_min = seg8;
 424         else if (seg8 > phba->ktime_seg8_max)
 425                 phba->ktime_seg8_max = seg8;
 426
 427         phba->ktime_seg9_total += seg9;
 428         if (seg9 < phba->ktime_seg9_min)
 429                 phba->ktime_seg9_min = seg9;
 430         else if (seg9 > phba->ktime_seg9_max)
 431                 phba->ktime_seg9_max = seg9;
 432 out:
 433         phba->ktime_seg10_total += seg10;
 434         if (seg10 < phba->ktime_seg10_min)
 435                 phba->ktime_seg10_min = seg10;
 436         else if (seg10 > phba->ktime_seg10_max)
 437                 phba->ktime_seg10_max = seg10;
 438         phba->ktime_status_samples++;
 439 }
 440 #endif
 441
 442 /**
 443  * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
 444  * @phba: Pointer to HBA context object.
 445  * @cmdwqe: Pointer to driver command WQE object.
 446  * @wcqe: Pointer to driver response CQE object.
 447  *
 448  * The function is called from SLI ring event handler with no
 449  * lock held. This function is the completion handler for NVME FCP commands
 450  * The function frees memory resources used for the NVME commands.
 451  **/
 452 static void
 453 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 454                           struct lpfc_wcqe_complete *wcqe)
 455 {
 456         struct lpfc_nvmet_tgtport *tgtp;
 457         struct nvmefc_tgt_fcp_req *rsp;
 458         struct lpfc_nvmet_rcv_ctx *ctxp;
 459         uint32_t status, result, op, start_clean;
 460 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 461         uint32_t id;
 462 #endif
 463
 464         ctxp = cmdwqe->context2;
 465         ctxp->flag &= ~LPFC_NVMET_IO_INP;
 466
 467         rsp = &ctxp->ctx.fcp_req;
 468         op = rsp->op;
 469
 470         status = bf_get(lpfc_wcqe_c_status, wcqe);
 471         result = wcqe->parameter;
 472
 473         if (phba->targetport)
 474                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 475         else
 476                 tgtp = NULL;
 477
 478         lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
 479                          ctxp->oxid, op, status);
 480
 481         if (status) {
 482                 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
 483                 rsp->transferred_length = 0;
 484                 if (tgtp)
 485                         atomic_inc(&tgtp->xmt_fcp_rsp_error);
 486
 487                 /* pick up SLI4 exhange busy condition */
 488                 if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
 489                         ctxp->flag |= LPFC_NVMET_XBUSY;
 490
 491                         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 492                                         "6315 IO Cmpl XBUSY: xri x%x: %x/%x\n",
 493                                         ctxp->oxid, status, result);
 494                 } else {
 495                         ctxp->flag &= ~LPFC_NVMET_XBUSY;
 496                 }
 497
 498         } else {
 499                 rsp->fcp_error = NVME_SC_SUCCESS;
 500                 if (op == NVMET_FCOP_RSP)
 501                         rsp->transferred_length = rsp->rsplen;
 502                 else
 503                         rsp->transferred_length = rsp->transfer_length;
 504                 if (tgtp)
 505                         atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);
 506         }
 507
 508         if ((op == NVMET_FCOP_READDATA_RSP) ||
 509             (op == NVMET_FCOP_RSP)) {
 510                 /* Sanity check */
 511                 ctxp->state = LPFC_NVMET_STE_DONE;
 512                 ctxp->entry_cnt++;
 513
 514 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 515                 if (phba->ktime_on) {
 516                         if (rsp->op == NVMET_FCOP_READDATA_RSP) {
 517                                 ctxp->ts_isr_data =
 518                                         cmdwqe->isr_timestamp;
 519                                 ctxp->ts_data_nvme =
 520                                         ktime_get_ns();
 521                                 ctxp->ts_nvme_status =
 522                                         ctxp->ts_data_nvme;
 523                                 ctxp->ts_status_wqput =
 524                                         ctxp->ts_data_nvme;
 525                                 ctxp->ts_isr_status =
 526                                         ctxp->ts_data_nvme;
 527                                 ctxp->ts_status_nvme =
 528                                         ctxp->ts_data_nvme;
 529                         } else {
 530                                 ctxp->ts_isr_status =
 531                                         cmdwqe->isr_timestamp;
 532                                 ctxp->ts_status_nvme =
 533                                         ktime_get_ns();
 534                         }
 535                 }
 536                 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 537                         id = smp_processor_id();
 538                         if (ctxp->cpu != id)
 539                                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 540                                                 "6703 CPU Check cmpl: "
 541                                                 "cpu %d expect %d\n",
 542                                                 id, ctxp->cpu);
 543                         if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
 544                                 phba->cpucheck_cmpl_io[id]++;
 545                 }
 546 #endif
 547                 rsp->done(rsp);
 548 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 549                 if (phba->ktime_on)
 550                         lpfc_nvmet_ktime(phba, ctxp);
 551 #endif
 552                 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */
 553         } else {
 554                 ctxp->entry_cnt++;
 555                 start_clean = offsetof(struct lpfc_iocbq, wqe);
 556                 memset(((char *)cmdwqe) + start_clean, 0,
 557                        (sizeof(struct lpfc_iocbq) - start_clean));
 558 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 559                 if (phba->ktime_on) {
 560                         ctxp->ts_isr_data = cmdwqe->isr_timestamp;
 561                         ctxp->ts_data_nvme = ktime_get_ns();
 562                 }
 563                 if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 564                         id = smp_processor_id();
 565                         if (ctxp->cpu != id)
 566                                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 567                                                 "6704 CPU Check cmdcmpl: "
 568                                                 "cpu %d expect %d\n",
 569                                                 id, ctxp->cpu);
 570                         if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
 571                                 phba->cpucheck_ccmpl_io[id]++;
 572                 }
 573 #endif
 574                 rsp->done(rsp);
 575         }
 576 }
 577
 578 static int
 579 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
 580                       struct nvmefc_tgt_ls_req *rsp)
 581 {
 582         struct lpfc_nvmet_rcv_ctx *ctxp =
 583                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req);
 584         struct lpfc_hba *phba = ctxp->phba;
 585         struct hbq_dmabuf *nvmebuf =
 586                 (struct hbq_dmabuf *)ctxp->rqb_buffer;
 587         struct lpfc_iocbq *nvmewqeq;
 588         struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
 589         struct lpfc_dmabuf dmabuf;
 590         struct ulp_bde64 bpl;
 591         int rc;
 592
 593         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
 594                         "6023 NVMET LS rsp oxid x%x\n", ctxp->oxid);
 595
 596         if ((ctxp->state != LPFC_NVMET_STE_LS_RCV) ||
 597             (ctxp->entry_cnt != 1)) {
 598                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 599                                 "6412 NVMET LS rsp state mismatch "
 600                                 "oxid x%x: %d %d\n",
 601                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
 602         }
 603         ctxp->state = LPFC_NVMET_STE_LS_RSP;
 604         ctxp->entry_cnt++;
 605
 606         nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma,
 607                                       rsp->rsplen);
 608         if (nvmewqeq == NULL) {
 609                 atomic_inc(&nvmep->xmt_ls_drop);
 610                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 611                                 "6150 LS Drop IO x%x: Prep\n",
 612                                 ctxp->oxid);
 613                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 614                 atomic_inc(&nvmep->xmt_ls_abort);
 615                 lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp,
 616                                                 ctxp->sid, ctxp->oxid);
 617                 return -ENOMEM;
 618         }
 619
 620         /* Save numBdes for bpl2sgl */
 621         nvmewqeq->rsvd2 = 1;
 622         nvmewqeq->hba_wqidx = 0;
 623         nvmewqeq->context3 = &dmabuf;
 624         dmabuf.virt = &bpl;
 625         bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
 626         bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
 627         bpl.tus.f.bdeSize = rsp->rsplen;
 628         bpl.tus.f.bdeFlags = 0;
 629         bpl.tus.w = le32_to_cpu(bpl.tus.w);
 630
 631         nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp;
 632         nvmewqeq->iocb_cmpl = NULL;
 633         nvmewqeq->context2 = ctxp;
 634
 635         lpfc_nvmeio_data(phba, "NVMET LS  RESP: xri x%x wqidx x%x len x%x\n",
 636                          ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen);
 637
 638         rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq);
 639         if (rc == WQE_SUCCESS) {
 640                 /*
 641                  * Okay to repost buffer here, but wait till cmpl
 642                  * before freeing ctxp and iocbq.
 643                  */
 644                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 645                 ctxp->rqb_buffer = 0;
 646                 atomic_inc(&nvmep->xmt_ls_rsp);
 647                 return 0;
 648         }
 649         /* Give back resources */
 650         atomic_inc(&nvmep->xmt_ls_drop);
 651         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 652                         "6151 LS Drop IO x%x: Issue %d\n",
 653                         ctxp->oxid, rc);
 654
 655         lpfc_nlp_put(nvmewqeq->context1);
 656
 657         lpfc_in_buf_free(phba, &nvmebuf->dbuf);
 658         atomic_inc(&nvmep->xmt_ls_abort);
 659         lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
 660         return -ENXIO;
 661 }
 662
 663 static int
 664 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
 665                       struct nvmefc_tgt_fcp_req *rsp)
 666 {
 667         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
 668         struct lpfc_nvmet_rcv_ctx *ctxp =
 669                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 670         struct lpfc_hba *phba = ctxp->phba;
 671         struct lpfc_iocbq *nvmewqeq;
 672         int rc;
 673
 674 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 675         if (phba->ktime_on) {
 676                 if (rsp->op == NVMET_FCOP_RSP)
 677                         ctxp->ts_nvme_status = ktime_get_ns();
 678                 else
 679                         ctxp->ts_nvme_data = ktime_get_ns();
 680         }
 681         if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
 682                 int id = smp_processor_id();
 683                 ctxp->cpu = id;
 684                 if (id < LPFC_CHECK_CPU_CNT)
 685                         phba->cpucheck_xmt_io[id]++;
 686                 if (rsp->hwqid != id) {
 687                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 688                                         "6705 CPU Check OP: "
 689                                         "cpu %d expect %d\n",
 690                                         id, rsp->hwqid);
 691                         ctxp->cpu = rsp->hwqid;
 692                 }
 693         }
 694 #endif
 695
 696         /* Sanity check */
 697         if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) ||
 698             (ctxp->state == LPFC_NVMET_STE_ABORT)) {
 699                 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 700                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 701                                 "6102 IO xri x%x aborted\n",
 702                                 ctxp->oxid);
 703                 rc = -ENXIO;
 704                 goto aerr;
 705         }
 706
 707         nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
 708         if (nvmewqeq == NULL) {
 709                 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 710                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 711                                 "6152 FCP Drop IO x%x: Prep\n",
 712                                 ctxp->oxid);
 713                 rc = -ENXIO;
 714                 goto aerr;
 715         }
 716
 717         nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
 718         nvmewqeq->iocb_cmpl = NULL;
 719         nvmewqeq->context2 = ctxp;
 720         nvmewqeq->iocb_flag |=  LPFC_IO_NVMET;
 721         ctxp->wqeq->hba_wqidx = rsp->hwqid;
 722
 723         lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
 724                          ctxp->oxid, rsp->op, rsp->rsplen);
 725
 726         ctxp->flag |= LPFC_NVMET_IO_INP;
 727         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);
 728         if (rc == WQE_SUCCESS) {
 729 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 730                 if (!phba->ktime_on)
 731                         return 0;
 732                 if (rsp->op == NVMET_FCOP_RSP)
 733                         ctxp->ts_status_wqput = ktime_get_ns();
 734                 else
 735                         ctxp->ts_data_wqput = ktime_get_ns();
 736 #endif
 737                 return 0;
 738         }
 739
 740         /* Give back resources */
 741         atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
 742         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 743                         "6153 FCP Drop IO x%x: Issue: %d\n",
 744                         ctxp->oxid, rc);
 745
 746         ctxp->wqeq->hba_wqidx = 0;
 747         nvmewqeq->context2 = NULL;
 748         nvmewqeq->context3 = NULL;
 749         rc = -EBUSY;
 750 aerr:
 751         return rc;
 752 }
 753
 754 static void
 755 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
 756 {
 757         struct lpfc_nvmet_tgtport *tport = targetport->private;
 758
 759         /* release any threads waiting for the unreg to complete */
 760         complete(&tport->tport_unreg_done);
 761 }
 762
 763 static void
 764 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
 765                          struct nvmefc_tgt_fcp_req *req)
 766 {
 767         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
 768         struct lpfc_nvmet_rcv_ctx *ctxp =
 769                 container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 770         struct lpfc_hba *phba = ctxp->phba;
 771         unsigned long flags;
 772
 773         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 774                         "6103 NVMET Abort op: oxri x%x flg x%x ste %d\n",
 775                         ctxp->oxid, ctxp->flag, ctxp->state);
 776
 777         lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n",
 778                          ctxp->oxid, ctxp->flag, ctxp->state);
 779
 780         atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
 781
 782         spin_lock_irqsave(&ctxp->ctxlock, flags);
 783
 784         /* Since iaab/iaar are NOT set, we need to check
 785          * if the firmware is in process of aborting IO
 786          */
 787         if (ctxp->flag & LPFC_NVMET_XBUSY) {
 788                 spin_unlock_irqrestore(&ctxp->ctxlock, flags);
 789                 return;
 790         }
 791         ctxp->flag |= LPFC_NVMET_ABORT_OP;
 792
 793         /* An state of LPFC_NVMET_STE_RCV means we have just received
 794          * the NVME command and have not started processing it.
 795          * (by issuing any IO WQEs on this exchange yet)
 796          */
 797         if (ctxp->state == LPFC_NVMET_STE_RCV)
 798                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
 799                                                  ctxp->oxid);
 800         else
 801                 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
 802                                                ctxp->oxid);
 803         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
 804 }
 805
 806 static void
 807 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
 808                            struct nvmefc_tgt_fcp_req *rsp)
 809 {
 810         struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
 811         struct lpfc_nvmet_rcv_ctx *ctxp =
 812                 container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
 813         struct lpfc_hba *phba = ctxp->phba;
 814         unsigned long flags;
 815         bool aborting = false;
 816
 817         if (ctxp->state != LPFC_NVMET_STE_DONE &&
 818             ctxp->state != LPFC_NVMET_STE_ABORT) {
 819                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
 820                                 "6413 NVMET release bad state %d %d oxid x%x\n",
 821                                 ctxp->state, ctxp->entry_cnt, ctxp->oxid);
 822         }
 823
 824         spin_lock_irqsave(&ctxp->ctxlock, flags);
 825         if ((ctxp->flag & LPFC_NVMET_ABORT_OP) ||
 826             (ctxp->flag & LPFC_NVMET_XBUSY)) {
 827                 aborting = true;
 828                 /* let the abort path do the real release */
 829                 lpfc_nvmet_defer_release(phba, ctxp);
 830         }
 831         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
 832
 833         lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid,
 834                          ctxp->state, aborting);
 835
 836         atomic_inc(&lpfc_nvmep->xmt_fcp_release);
 837
 838         if (aborting)
 839                 return;
 840
 841         lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
 842 }
 843
 844 static struct nvmet_fc_target_template lpfc_tgttemplate = {
 845         .targetport_delete = lpfc_nvmet_targetport_delete,
 846         .xmt_ls_rsp     = lpfc_nvmet_xmt_ls_rsp,
 847         .fcp_op         = lpfc_nvmet_xmt_fcp_op,
 848         .fcp_abort      = lpfc_nvmet_xmt_fcp_abort,
 849         .fcp_req_release = lpfc_nvmet_xmt_fcp_release,
 850
 851         .max_hw_queues  = 1,
 852         .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
 853         .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
 854         .dma_boundary = 0xFFFFFFFF,
 855
 856         /* optional features */
 857         .target_features = 0,
 858         /* sizes of additional private data for data structures */
 859         .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
 860 };
 861
 862 static void
 863 lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
 864 {
 865         struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;
 866         unsigned long flags;
 867
 868         list_for_each_entry_safe(
 869                 ctx_buf, next_ctx_buf,
 870                 &phba->sli4_hba.lpfc_nvmet_ctx_list, list) {
 871                 spin_lock_irqsave(
 872                         &phba->sli4_hba.abts_nvme_buf_list_lock, flags);
 873                 list_del_init(&ctx_buf->list);
 874                 spin_unlock_irqrestore(
 875                         &phba->sli4_hba.abts_nvme_buf_list_lock, flags);
 876                 __lpfc_clear_active_sglq(phba,
 877                                          ctx_buf->sglq->sli4_lxritag);
 878                 ctx_buf->sglq->state = SGL_FREED;
 879                 ctx_buf->sglq->ndlp = NULL;
 880
 881                 spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock, flags);
 882                 list_add_tail(&ctx_buf->sglq->list,
 883                               &phba->sli4_hba.lpfc_nvmet_sgl_list);
 884                 spin_unlock_irqrestore(&phba->sli4_hba.sgl_list_lock,
 885                                        flags);
 886
 887                 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
 888                 kfree(ctx_buf->context);
 889         }
 890 }
 891
 892 static int
 893 lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
 894 {
 895         struct lpfc_nvmet_ctxbuf *ctx_buf;
 896         struct lpfc_iocbq *nvmewqe;
 897         union lpfc_wqe128 *wqe;
 898         int i;
 899
 900         lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
 901                         "6403 Allocate NVMET resources for %d XRIs\n",
 902                         phba->sli4_hba.nvmet_xri_cnt);
 903
 904         /* For all nvmet xris, allocate resources needed to process a
 905          * received command on a per xri basis.
 906          */
 907         for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
 908                 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
 909                 if (!ctx_buf) {
 910                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
 911                                         "6404 Ran out of memory for NVMET\n");
 912                         return -ENOMEM;
 913                 }
 914
 915                 ctx_buf->context = kzalloc(sizeof(*ctx_buf->context),
 916                                            GFP_KERNEL);
 917                 if (!ctx_buf->context) {
 918                         kfree(ctx_buf);
 919                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
 920                                         "6405 Ran out of NVMET "
 921                                         "context memory\n");
 922                         return -ENOMEM;
 923                 }
 924                 ctx_buf->context->ctxbuf = ctx_buf;
 925                 ctx_buf->context->state = LPFC_NVMET_STE_FREE;
 926
 927                 ctx_buf->iocbq = lpfc_sli_get_iocbq(phba);
 928                 if (!ctx_buf->iocbq) {
 929                         kfree(ctx_buf->context);
 930                         kfree(ctx_buf);
 931                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
 932                                         "6406 Ran out of NVMET iocb/WQEs\n");
 933                         return -ENOMEM;
 934                 }
 935                 ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET;
 936                 nvmewqe = ctx_buf->iocbq;
 937                 wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
 938                 /* Initialize WQE */
 939                 memset(wqe, 0, sizeof(union lpfc_wqe));
 940                 /* Word 7 */
 941                 bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI);
 942                 bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3);
 943                 bf_set(wqe_pu, &wqe->generic.wqe_com, 1);
 944                 /* Word 10 */
 945                 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
 946                 bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
 947                 bf_set(wqe_qosd, &wqe->generic.wqe_com, 0);
 948
 949                 ctx_buf->iocbq->context1 = NULL;
 950                 spin_lock(&phba->sli4_hba.sgl_list_lock);
 951                 ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq);
 952                 spin_unlock(&phba->sli4_hba.sgl_list_lock);
 953                 if (!ctx_buf->sglq) {
 954                         lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
 955                         kfree(ctx_buf->context);
 956                         kfree(ctx_buf);
 957                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
 958                                         "6407 Ran out of NVMET XRIs\n");
 959                         return -ENOMEM;
 960                 }
 961                 spin_lock(&phba->sli4_hba.nvmet_io_lock);
 962                 list_add_tail(&ctx_buf->list,
 963                               &phba->sli4_hba.lpfc_nvmet_ctx_list);
 964                 spin_unlock(&phba->sli4_hba.nvmet_io_lock);
 965         }
 966         phba->sli4_hba.nvmet_ctx_cnt = phba->sli4_hba.nvmet_xri_cnt;
 967         return 0;
 968 }
 969
 970 int
 971 lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
 972 {
 973         struct lpfc_vport  *vport = phba->pport;
 974         struct lpfc_nvmet_tgtport *tgtp;
 975         struct nvmet_fc_port_info pinfo;
 976         int error;
 977
 978         if (phba->targetport)
 979                 return 0;
 980
 981         error = lpfc_nvmet_setup_io_context(phba);
 982         if (error)
 983                 return error;
 984
 985         memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
 986         pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
 987         pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
 988         pinfo.port_id = vport->fc_myDID;
 989
 990         /* Limit to LPFC_MAX_NVME_SEG_CNT.
 991          * For now need + 1 to get around NVME transport logic.
 992          */
 993         if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) {
 994                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
 995                                 "6400 Reducing sg segment cnt to %d\n",
 996                                 LPFC_MAX_NVME_SEG_CNT);
 997                 phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT;
 998         } else {
 999                 phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt;
1000         }
1001         lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
1002         lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel;
1003         lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP |
1004                                            NVMET_FCTGTFEAT_CMD_IN_ISR |
1005                                            NVMET_FCTGTFEAT_OPDONE_IN_ISR;
1006
1007 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1008         error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
1009                                              &phba->pcidev->dev,
1010                                              &phba->targetport);
1011 #else
1012         error = -ENOENT;
1013 #endif
1014         if (error) {
1015                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1016                                 "6025 Cannot register NVME targetport "
1017                                 "x%x\n", error);
1018                 phba->targetport = NULL;
1019
1020                 lpfc_nvmet_cleanup_io_context(phba);
1021
1022         } else {
1023                 tgtp = (struct lpfc_nvmet_tgtport *)
1024                         phba->targetport->private;
1025                 tgtp->phba = phba;
1026
1027                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1028                                 "6026 Registered NVME "
1029                                 "targetport: %p, private %p "
1030                                 "portnm %llx nodenm %llx\n",
1031                                 phba->targetport, tgtp,
1032                                 pinfo.port_name, pinfo.node_name);
1033
1034                 atomic_set(&tgtp->rcv_ls_req_in, 0);
1035                 atomic_set(&tgtp->rcv_ls_req_out, 0);
1036                 atomic_set(&tgtp->rcv_ls_req_drop, 0);
1037                 atomic_set(&tgtp->xmt_ls_abort, 0);
1038                 atomic_set(&tgtp->xmt_ls_abort_cmpl, 0);
1039                 atomic_set(&tgtp->xmt_ls_rsp, 0);
1040                 atomic_set(&tgtp->xmt_ls_drop, 0);
1041                 atomic_set(&tgtp->xmt_ls_rsp_error, 0);
1042                 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
1043                 atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
1044                 atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
1045                 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
1046                 atomic_set(&tgtp->xmt_fcp_drop, 0);
1047                 atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
1048                 atomic_set(&tgtp->xmt_fcp_read, 0);
1049                 atomic_set(&tgtp->xmt_fcp_write, 0);
1050                 atomic_set(&tgtp->xmt_fcp_rsp, 0);
1051                 atomic_set(&tgtp->xmt_fcp_release, 0);
1052                 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
1053                 atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
1054                 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
1055                 atomic_set(&tgtp->xmt_fcp_abort, 0);
1056                 atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0);
1057                 atomic_set(&tgtp->xmt_abort_unsol, 0);
1058                 atomic_set(&tgtp->xmt_abort_sol, 0);
1059                 atomic_set(&tgtp->xmt_abort_rsp, 0);
1060                 atomic_set(&tgtp->xmt_abort_rsp_error, 0);
1061         }
1062         return error;
1063 }
1064
1065 int
1066 lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
1067 {
1068         struct lpfc_vport  *vport = phba->pport;
1069
1070         if (!phba->targetport)
1071                 return 0;
1072
1073         lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1074                          "6007 Update NVMET port %p did x%x\n",
1075                          phba->targetport, vport->fc_myDID);
1076
1077         phba->targetport->port_id = vport->fc_myDID;
1078         return 0;
1079 }
1080
1081 /**
1082  * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort
1083  * @phba: pointer to lpfc hba data structure.
1084  * @axri: pointer to the nvmet xri abort wcqe structure.
1085  *
1086  * This routine is invoked by the worker thread to process a SLI4 fast-path
1087  * NVMET aborted xri.
1088  **/
1089 void
1090 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,
1091                             struct sli4_wcqe_xri_aborted *axri)
1092 {
1093         uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
1094         uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
1095         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
1096         struct lpfc_nodelist *ndlp;
1097         unsigned long iflag = 0;
1098         int rrq_empty = 0;
1099         bool released = false;
1100
1101         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1102                         "6317 XB aborted xri x%x rxid x%x\n", xri, rxid);
1103
1104         if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
1105                 return;
1106         spin_lock_irqsave(&phba->hbalock, iflag);
1107         spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1108         list_for_each_entry_safe(ctxp, next_ctxp,
1109                                  &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1110                                  list) {
1111                 if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1112                         continue;
1113
1114                 /* Check if we already received a free context call
1115                  * and we have completed processing an abort situation.
1116                  */
1117                 if (ctxp->flag & LPFC_NVMET_CTX_RLS &&
1118                     !(ctxp->flag & LPFC_NVMET_ABORT_OP)) {
1119                         list_del(&ctxp->list);
1120                         released = true;
1121                 }
1122                 ctxp->flag &= ~LPFC_NVMET_XBUSY;
1123                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1124
1125                 rrq_empty = list_empty(&phba->active_rrq_list);
1126                 spin_unlock_irqrestore(&phba->hbalock, iflag);
1127                 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1128                 if (ndlp && NLP_CHK_NODE_ACT(ndlp) &&
1129                     (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||
1130                      ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {
1131                         lpfc_set_rrq_active(phba, ndlp,
1132                                 ctxp->ctxbuf->sglq->sli4_lxritag,
1133                                 rxid, 1);
1134                         lpfc_sli4_abts_err_handler(phba, ndlp, axri);
1135                 }
1136
1137                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1138                                 "6318 XB aborted oxid %x flg x%x (%x)\n",
1139                                 ctxp->oxid, ctxp->flag, released);
1140                 if (released)
1141                         lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
1142
1143                 if (rrq_empty)
1144                         lpfc_worker_wake_up(phba);
1145                 return;
1146         }
1147         spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1148         spin_unlock_irqrestore(&phba->hbalock, iflag);
1149 }
1150
1151 int
1152 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
1153                            struct fc_frame_header *fc_hdr)
1154
1155 {
1156 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1157         struct lpfc_hba *phba = vport->phba;
1158         struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
1159         struct nvmefc_tgt_fcp_req *rsp;
1160         uint16_t xri;
1161         unsigned long iflag = 0;
1162
1163         xri = be16_to_cpu(fc_hdr->fh_ox_id);
1164
1165         spin_lock_irqsave(&phba->hbalock, iflag);
1166         spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1167         list_for_each_entry_safe(ctxp, next_ctxp,
1168                                  &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1169                                  list) {
1170                 if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1171                         continue;
1172
1173                 spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1174                 spin_unlock_irqrestore(&phba->hbalock, iflag);
1175
1176                 spin_lock_irqsave(&ctxp->ctxlock, iflag);
1177                 ctxp->flag |= LPFC_NVMET_ABTS_RCV;
1178                 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
1179
1180                 lpfc_nvmeio_data(phba,
1181                         "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1182                         xri, smp_processor_id(), 0);
1183
1184                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1185                                 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
1186
1187                 rsp = &ctxp->ctx.fcp_req;
1188                 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp);
1189
1190                 /* Respond with BA_ACC accordingly */
1191                 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
1192                 return 0;
1193         }
1194         spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1195         spin_unlock_irqrestore(&phba->hbalock, iflag);
1196
1197         lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1198                          xri, smp_processor_id(), 1);
1199
1200         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1201                         "6320 NVMET Rcv ABTS:rjt xri x%x\n", xri);
1202
1203         /* Respond with BA_RJT accordingly */
1204         lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0);
1205 #endif
1206         return 0;
1207 }
1208
1209 void
1210 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
1211 {
1212 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1213         struct lpfc_nvmet_tgtport *tgtp;
1214
1215         if (phba->nvmet_support == 0)
1216                 return;
1217         if (phba->targetport) {
1218                 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1219                 init_completion(&tgtp->tport_unreg_done);
1220                 nvmet_fc_unregister_targetport(phba->targetport);
1221                 wait_for_completion_timeout(&tgtp->tport_unreg_done, 5);
1222                 lpfc_nvmet_cleanup_io_context(phba);
1223         }
1224         phba->targetport = NULL;
1225 #endif
1226 }
1227
1228 /**
1229  * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer
1230  * @phba: pointer to lpfc hba data structure.
1231  * @pring: pointer to a SLI ring.
1232  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1233  *
1234  * This routine is used for processing the WQE associated with a unsolicited
1235  * event. It first determines whether there is an existing ndlp that matches
1236  * the DID from the unsolicited WQE. If not, it will create a new one with
1237  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1238  * WQE is then used to invoke the proper routine and to set up proper state
1239  * of the discovery state machine.
1240  **/
1241 static void
1242 lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1243                            struct hbq_dmabuf *nvmebuf)
1244 {
1245 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1246         struct lpfc_nvmet_tgtport *tgtp;
1247         struct fc_frame_header *fc_hdr;
1248         struct lpfc_nvmet_rcv_ctx *ctxp;
1249         uint32_t *payload;
1250         uint32_t size, oxid, sid, rc;
1251
1252         if (!nvmebuf || !phba->targetport) {
1253                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1254                                 "6154 LS Drop IO\n");
1255                 oxid = 0;
1256                 size = 0;
1257                 sid = 0;
1258                 ctxp = NULL;
1259                 goto dropit;
1260         }
1261
1262         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1263         payload = (uint32_t *)(nvmebuf->dbuf.virt);
1264         fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1265         size = bf_get(lpfc_rcqe_length,  &nvmebuf->cq_event.cqe.rcqe_cmpl);
1266         oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1267         sid = sli4_sid_from_fc_hdr(fc_hdr);
1268
1269         ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC);
1270         if (ctxp == NULL) {
1271                 atomic_inc(&tgtp->rcv_ls_req_drop);
1272                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1273                                 "6155 LS Drop IO x%x: Alloc\n",
1274                                 oxid);
1275 dropit:
1276                 lpfc_nvmeio_data(phba, "NVMET LS  DROP: "
1277                                  "xri x%x sz %d from %06x\n",
1278                                  oxid, size, sid);
1279                 if (nvmebuf)
1280                         lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1281                 return;
1282         }
1283         ctxp->phba = phba;
1284         ctxp->size = size;
1285         ctxp->oxid = oxid;
1286         ctxp->sid = sid;
1287         ctxp->wqeq = NULL;
1288         ctxp->state = LPFC_NVMET_STE_LS_RCV;
1289         ctxp->entry_cnt = 1;
1290         ctxp->rqb_buffer = (void *)nvmebuf;
1291
1292         lpfc_nvmeio_data(phba, "NVMET LS   RCV: xri x%x sz %d from %06x\n",
1293                          oxid, size, sid);
1294         /*
1295          * The calling sequence should be:
1296          * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done
1297          * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp.
1298          */
1299         atomic_inc(&tgtp->rcv_ls_req_in);
1300         rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req,
1301                                  payload, size);
1302
1303         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1304                         "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x "
1305                         "%08x %08x %08x\n", size, rc,
1306                         *payload, *(payload+1), *(payload+2),
1307                         *(payload+3), *(payload+4), *(payload+5));
1308
1309         if (rc == 0) {
1310                 atomic_inc(&tgtp->rcv_ls_req_out);
1311                 return;
1312         }
1313
1314         lpfc_nvmeio_data(phba, "NVMET LS  DROP: xri x%x sz %d from %06x\n",
1315                          oxid, size, sid);
1316
1317         atomic_inc(&tgtp->rcv_ls_req_drop);
1318         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1319                         "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n",
1320                         ctxp->oxid, rc);
1321
1322         /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
1323         if (nvmebuf)
1324                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1325
1326         atomic_inc(&tgtp->xmt_ls_abort);
1327         lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid);
1328 #endif
1329 }
1330
1331 /**
1332  * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
1333  * @phba: pointer to lpfc hba data structure.
1334  * @pring: pointer to a SLI ring.
1335  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1336  *
1337  * This routine is used for processing the WQE associated with a unsolicited
1338  * event. It first determines whether there is an existing ndlp that matches
1339  * the DID from the unsolicited WQE. If not, it will create a new one with
1340  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1341  * WQE is then used to invoke the proper routine and to set up proper state
1342  * of the discovery state machine.
1343  **/
1344 static void
1345 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
1346                             struct lpfc_sli_ring *pring,
1347                             struct rqb_dmabuf *nvmebuf,
1348                             uint64_t isr_timestamp)
1349 {
1350 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1351         struct lpfc_nvmet_rcv_ctx *ctxp;
1352         struct lpfc_nvmet_tgtport *tgtp;
1353         struct fc_frame_header *fc_hdr;
1354         struct lpfc_nvmet_ctxbuf *ctx_buf;
1355         uint32_t *payload;
1356         uint32_t size, oxid, sid, rc, qno;
1357         unsigned long iflag;
1358 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1359         uint32_t id;
1360 #endif
1361
1362         ctx_buf = NULL;
1363         if (!nvmebuf || !phba->targetport) {
1364                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1365                                 "6157 NVMET FCP Drop IO\n");
1366                 oxid = 0;
1367                 size = 0;
1368                 sid = 0;
1369                 ctxp = NULL;
1370                 goto dropit;
1371         }
1372
1373         spin_lock_irqsave(&phba->sli4_hba.nvmet_io_lock, iflag);
1374         if (phba->sli4_hba.nvmet_ctx_cnt) {
1375                 list_remove_head(&phba->sli4_hba.lpfc_nvmet_ctx_list,
1376                                  ctx_buf, struct lpfc_nvmet_ctxbuf, list);
1377                 phba->sli4_hba.nvmet_ctx_cnt--;
1378         }
1379         spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_lock, iflag);
1380
1381         fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1382         oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1383         size = nvmebuf->bytes_recv;
1384
1385 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1386         if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) {
1387                 id = smp_processor_id();
1388                 if (id < LPFC_CHECK_CPU_CNT)
1389                         phba->cpucheck_rcv_io[id]++;
1390         }
1391 #endif
1392
1393         lpfc_nvmeio_data(phba, "NVMET FCP  RCV: xri x%x sz %d CPU %02x\n",
1394                          oxid, size, smp_processor_id());
1395
1396         if (!ctx_buf) {
1397                 /* Queue this NVME IO to process later */
1398                 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
1399                 list_add_tail(&nvmebuf->hbuf.list,
1400                               &phba->sli4_hba.lpfc_nvmet_io_wait_list);
1401                 phba->sli4_hba.nvmet_io_wait_cnt++;
1402                 phba->sli4_hba.nvmet_io_wait_total++;
1403                 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
1404                                        iflag);
1405
1406                 /* Post a brand new DMA buffer to RQ */
1407                 qno = nvmebuf->idx;
1408                 lpfc_post_rq_buffer(
1409                         phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
1410                         phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
1411                 return;
1412         }
1413
1414         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1415         payload = (uint32_t *)(nvmebuf->dbuf.virt);
1416         sid = sli4_sid_from_fc_hdr(fc_hdr);
1417
1418         ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
1419         if (ctxp->state != LPFC_NVMET_STE_FREE) {
1420                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1421                                 "6414 NVMET Context corrupt %d %d oxid x%x\n",
1422                                 ctxp->state, ctxp->entry_cnt, ctxp->oxid);
1423         }
1424         ctxp->wqeq = NULL;
1425         ctxp->txrdy = NULL;
1426         ctxp->offset = 0;
1427         ctxp->phba = phba;
1428         ctxp->size = size;
1429         ctxp->oxid = oxid;
1430         ctxp->sid = sid;
1431         ctxp->state = LPFC_NVMET_STE_RCV;
1432         ctxp->entry_cnt = 1;
1433         ctxp->flag = 0;
1434         ctxp->ctxbuf = ctx_buf;
1435         spin_lock_init(&ctxp->ctxlock);
1436
1437 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1438         if (phba->ktime_on) {
1439                 ctxp->ts_isr_cmd = isr_timestamp;
1440                 ctxp->ts_cmd_nvme = ktime_get_ns();
1441                 ctxp->ts_nvme_data = 0;
1442                 ctxp->ts_data_wqput = 0;
1443                 ctxp->ts_isr_data = 0;
1444                 ctxp->ts_data_nvme = 0;
1445                 ctxp->ts_nvme_status = 0;
1446                 ctxp->ts_status_wqput = 0;
1447                 ctxp->ts_isr_status = 0;
1448                 ctxp->ts_status_nvme = 0;
1449         }
1450 #endif
1451
1452         atomic_inc(&tgtp->rcv_fcp_cmd_in);
1453         /*
1454          * The calling sequence should be:
1455          * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done
1456          * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
1457          * When we return from nvmet_fc_rcv_fcp_req, all relevant info in
1458          * the NVME command / FC header is stored, so we are free to repost
1459          * the buffer.
1460          */
1461         rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
1462                                   payload, size);
1463
1464         /* Process FCP command */
1465         if (rc == 0) {
1466                 atomic_inc(&tgtp->rcv_fcp_cmd_out);
1467                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
1468                 return;
1469         }
1470
1471         atomic_inc(&tgtp->rcv_fcp_cmd_drop);
1472         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1473                         "6159 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
1474                         ctxp->oxid, rc,
1475                         atomic_read(&tgtp->rcv_fcp_cmd_in),
1476                         atomic_read(&tgtp->rcv_fcp_cmd_out),
1477                         atomic_read(&tgtp->xmt_fcp_release));
1478 dropit:
1479         lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
1480                          oxid, size, sid);
1481         if (oxid) {
1482                 lpfc_nvmet_defer_release(phba, ctxp);
1483                 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
1484                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
1485                 return;
1486         }
1487
1488         if (ctx_buf)
1489                 lpfc_nvmet_ctxbuf_post(phba, ctx_buf);
1490
1491         if (nvmebuf)
1492                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
1493 #endif
1494 }
1495
1496 /**
1497  * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport
1498  * @phba: pointer to lpfc hba data structure.
1499  * @pring: pointer to a SLI ring.
1500  * @nvmebuf: pointer to received nvme data structure.
1501  *
1502  * This routine is used to process an unsolicited event received from a SLI
1503  * (Service Level Interface) ring. The actual processing of the data buffer
1504  * associated with the unsolicited event is done by invoking the routine
1505  * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the
1506  * SLI RQ on which the unsolicited event was received.
1507  **/
1508 void
1509 lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1510                           struct lpfc_iocbq *piocb)
1511 {
1512         struct lpfc_dmabuf *d_buf;
1513         struct hbq_dmabuf *nvmebuf;
1514
1515         d_buf = piocb->context2;
1516         nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
1517
1518         if (phba->nvmet_support == 0) {
1519                 lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1520                 return;
1521         }
1522         lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf);
1523 }
1524
1525 /**
1526  * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
1527  * @phba: pointer to lpfc hba data structure.
1528  * @pring: pointer to a SLI ring.
1529  * @nvmebuf: pointer to received nvme data structure.
1530  *
1531  * This routine is used to process an unsolicited event received from a SLI
1532  * (Service Level Interface) ring. The actual processing of the data buffer
1533  * associated with the unsolicited event is done by invoking the routine
1534  * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
1535  * SLI RQ on which the unsolicited event was received.
1536  **/
1537 void
1538 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
1539                            struct lpfc_sli_ring *pring,
1540                            struct rqb_dmabuf *nvmebuf,
1541                            uint64_t isr_timestamp)
1542 {
1543         if (phba->nvmet_support == 0) {
1544                 lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
1545                 return;
1546         }
1547         lpfc_nvmet_unsol_fcp_buffer(phba, pring, nvmebuf,
1548                                     isr_timestamp);
1549 }
1550
1551 /**
1552  * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
1553  * @phba: pointer to a host N_Port data structure.
1554  * @ctxp: Context info for NVME LS Request
1555  * @rspbuf: DMA buffer of NVME command.
1556  * @rspsize: size of the NVME command.
1557  *
1558  * This routine is used for allocating a lpfc-WQE data structure from
1559  * the driver lpfc-WQE free-list and prepare the WQE with the parameters
1560  * passed into the routine for discovery state machine to issue an Extended
1561  * Link Service (NVME) commands. It is a generic lpfc-WQE allocation
1562  * and preparation routine that is used by all the discovery state machine
1563  * routines and the NVME command-specific fields will be later set up by
1564  * the individual discovery machine routines after calling this routine
1565  * allocating and preparing a generic WQE data structure. It fills in the
1566  * Buffer Descriptor Entries (BDEs), allocates buffers for both command
1567  * payload and response payload (if expected). The reference count on the
1568  * ndlp is incremented by 1 and the reference to the ndlp is put into
1569  * context1 of the WQE data structure for this WQE to hold the ndlp
1570  * reference for the command's callback function to access later.
1571  *
1572  * Return code
1573  *   Pointer to the newly allocated/prepared nvme wqe data structure
1574  *   NULL - when nvme wqe data structure allocation/preparation failed
1575  **/
1576 static struct lpfc_iocbq *
1577 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
1578                        struct lpfc_nvmet_rcv_ctx *ctxp,
1579                        dma_addr_t rspbuf, uint16_t rspsize)
1580 {
1581         struct lpfc_nodelist *ndlp;
1582         struct lpfc_iocbq *nvmewqe;
1583         union lpfc_wqe *wqe;
1584
1585         if (!lpfc_is_link_up(phba)) {
1586                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1587                                 "6104 NVMET prep LS wqe: link err: "
1588                                 "NPORT x%x oxid:x%x ste %d\n",
1589                                 ctxp->sid, ctxp->oxid, ctxp->state);
1590                 return NULL;
1591         }
1592
1593         /* Allocate buffer for  command wqe */
1594         nvmewqe = lpfc_sli_get_iocbq(phba);
1595         if (nvmewqe == NULL) {
1596                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1597                                 "6105 NVMET prep LS wqe: No WQE: "
1598                                 "NPORT x%x oxid x%x ste %d\n",
1599                                 ctxp->sid, ctxp->oxid, ctxp->state);
1600                 return NULL;
1601         }
1602
1603         ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1604         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1605             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1606             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1607                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1608                                 "6106 NVMET prep LS wqe: No ndlp: "
1609                                 "NPORT x%x oxid x%x ste %d\n",
1610                                 ctxp->sid, ctxp->oxid, ctxp->state);
1611                 goto nvme_wqe_free_wqeq_exit;
1612         }
1613         ctxp->wqeq = nvmewqe;
1614
1615         /* prevent preparing wqe with NULL ndlp reference */
1616         nvmewqe->context1 = lpfc_nlp_get(ndlp);
1617         if (nvmewqe->context1 == NULL)
1618                 goto nvme_wqe_free_wqeq_exit;
1619         nvmewqe->context2 = ctxp;
1620
1621         wqe = &nvmewqe->wqe;
1622         memset(wqe, 0, sizeof(union lpfc_wqe));
1623
1624         /* Words 0 - 2 */
1625         wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1626         wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
1627         wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
1628         wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
1629
1630         /* Word 3 */
1631
1632         /* Word 4 */
1633
1634         /* Word 5 */
1635         bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
1636         bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
1637         bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
1638         bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);
1639         bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
1640
1641         /* Word 6 */
1642         bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
1643                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1644         bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
1645
1646         /* Word 7 */
1647         bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
1648                CMD_XMIT_SEQUENCE64_WQE);
1649         bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
1650         bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
1651         bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
1652
1653         /* Word 8 */
1654         wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
1655
1656         /* Word 9 */
1657         bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
1658         /* Needs to be set by caller */
1659         bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
1660
1661         /* Word 10 */
1662         bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
1663         bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
1664         bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
1665                LPFC_WQE_LENLOC_WORD12);
1666         bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
1667
1668         /* Word 11 */
1669         bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
1670                LPFC_WQE_CQ_ID_DEFAULT);
1671         bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
1672                OTHER_COMMAND);
1673
1674         /* Word 12 */
1675         wqe->xmit_sequence.xmit_len = rspsize;
1676
1677         nvmewqe->retry = 1;
1678         nvmewqe->vport = phba->pport;
1679         nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
1680         nvmewqe->iocb_flag |= LPFC_IO_NVME_LS;
1681
1682         /* Xmit NVMET response to remote NPORT <did> */
1683         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1684                         "6039 Xmit NVMET LS response to remote "
1685                         "NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
1686                         ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
1687                         rspsize);
1688         return nvmewqe;
1689
1690 nvme_wqe_free_wqeq_exit:
1691         nvmewqe->context2 = NULL;
1692         nvmewqe->context3 = NULL;
1693         lpfc_sli_release_iocbq(phba, nvmewqe);
1694         return NULL;
1695 }
1696
1697
1698 static struct lpfc_iocbq *
1699 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
1700                         struct lpfc_nvmet_rcv_ctx *ctxp)
1701 {
1702         struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req;
1703         struct lpfc_nvmet_tgtport *tgtp;
1704         struct sli4_sge *sgl;
1705         struct lpfc_nodelist *ndlp;
1706         struct lpfc_iocbq *nvmewqe;
1707         struct scatterlist *sgel;
1708         union lpfc_wqe128 *wqe;
1709         uint32_t *txrdy;
1710         dma_addr_t physaddr;
1711         int i, cnt;
1712         int xc = 1;
1713
1714         if (!lpfc_is_link_up(phba)) {
1715                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1716                                 "6107 NVMET prep FCP wqe: link err:"
1717                                 "NPORT x%x oxid x%x ste %d\n",
1718                                 ctxp->sid, ctxp->oxid, ctxp->state);
1719                 return NULL;
1720         }
1721
1722         ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1723         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1724             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1725              (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1726                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1727                                 "6108 NVMET prep FCP wqe: no ndlp: "
1728                                 "NPORT x%x oxid x%x ste %d\n",
1729                                 ctxp->sid, ctxp->oxid, ctxp->state);
1730                 return NULL;
1731         }
1732
1733         if (rsp->sg_cnt > phba->cfg_nvme_seg_cnt) {
1734                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1735                                 "6109 NVMET prep FCP wqe: seg cnt err: "
1736                                 "NPORT x%x oxid x%x ste %d cnt %d\n",
1737                                 ctxp->sid, ctxp->oxid, ctxp->state,
1738                                 phba->cfg_nvme_seg_cnt);
1739                 return NULL;
1740         }
1741
1742         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1743         nvmewqe = ctxp->wqeq;
1744         if (nvmewqe == NULL) {
1745                 /* Allocate buffer for  command wqe */
1746                 nvmewqe = ctxp->ctxbuf->iocbq;
1747                 if (nvmewqe == NULL) {
1748                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1749                                         "6110 NVMET prep FCP wqe: No "
1750                                         "WQE: NPORT x%x oxid x%x ste %d\n",
1751                                         ctxp->sid, ctxp->oxid, ctxp->state);
1752                         return NULL;
1753                 }
1754                 ctxp->wqeq = nvmewqe;
1755                 xc = 0; /* create new XRI */
1756                 nvmewqe->sli4_lxritag = NO_XRI;
1757                 nvmewqe->sli4_xritag = NO_XRI;
1758         }
1759
1760         /* Sanity check */
1761         if (((ctxp->state == LPFC_NVMET_STE_RCV) &&
1762             (ctxp->entry_cnt == 1)) ||
1763             (ctxp->state == LPFC_NVMET_STE_DATA)) {
1764                 wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
1765         } else {
1766                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1767                                 "6111 Wrong state NVMET FCP: %d  cnt %d\n",
1768                                 ctxp->state, ctxp->entry_cnt);
1769                 return NULL;
1770         }
1771
1772         sgl  = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl;
1773         switch (rsp->op) {
1774         case NVMET_FCOP_READDATA:
1775         case NVMET_FCOP_READDATA_RSP:
1776                 /* Words 0 - 2 : The first sg segment */
1777                 sgel = &rsp->sg[0];
1778                 physaddr = sg_dma_address(sgel);
1779                 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1780                 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
1781                 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
1782                 wqe->fcp_tsend.bde.addrHigh =
1783                         cpu_to_le32(putPaddrHigh(physaddr));
1784
1785                 /* Word 3 */
1786                 wqe->fcp_tsend.payload_offset_len = 0;
1787
1788                 /* Word 4 */
1789                 wqe->fcp_tsend.relative_offset = ctxp->offset;
1790
1791                 /* Word 5 */
1792
1793                 /* Word 6 */
1794                 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
1795                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1796                 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
1797                        nvmewqe->sli4_xritag);
1798
1799                 /* Word 7 */
1800                 bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
1801
1802                 /* Word 8 */
1803                 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
1804
1805                 /* Word 9 */
1806                 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
1807                 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
1808
1809                 /* Word 10 */
1810                 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
1811                 bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
1812                 bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
1813                 bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com,
1814                        LPFC_WQE_LENLOC_WORD12);
1815                 bf_set(wqe_ebde_cnt, &wqe->fcp_tsend.wqe_com, 0);
1816                 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, xc);
1817                 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
1818                 if (phba->cfg_nvme_oas)
1819                         bf_set(wqe_oas, &wqe->fcp_tsend.wqe_com, 1);
1820
1821                 /* Word 11 */
1822                 bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com,
1823                        LPFC_WQE_CQ_ID_DEFAULT);
1824                 bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com,
1825                        FCP_COMMAND_TSEND);
1826
1827                 /* Word 12 */
1828                 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
1829
1830                 /* Setup 2 SKIP SGEs */
1831                 sgl->addr_hi = 0;
1832                 sgl->addr_lo = 0;
1833                 sgl->word2 = 0;
1834                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1835                 sgl->word2 = cpu_to_le32(sgl->word2);
1836                 sgl->sge_len = 0;
1837                 sgl++;
1838                 sgl->addr_hi = 0;
1839                 sgl->addr_lo = 0;
1840                 sgl->word2 = 0;
1841                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1842                 sgl->word2 = cpu_to_le32(sgl->word2);
1843                 sgl->sge_len = 0;
1844                 sgl++;
1845                 if (rsp->op == NVMET_FCOP_READDATA_RSP) {
1846                         atomic_inc(&tgtp->xmt_fcp_read_rsp);
1847                         bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
1848                         if ((ndlp->nlp_flag & NLP_SUPPRESS_RSP) &&
1849                             (rsp->rsplen == 12)) {
1850                                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 1);
1851                                 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
1852                                 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
1853                                 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
1854                         } else {
1855                                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1856                                 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
1857                                 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
1858                                 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
1859                                        ((rsp->rsplen >> 2) - 1));
1860                                 memcpy(&wqe->words[16], rsp->rspaddr,
1861                                        rsp->rsplen);
1862                         }
1863                 } else {
1864                         atomic_inc(&tgtp->xmt_fcp_read);
1865
1866                         bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1867                         bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
1868                         bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
1869                         bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
1870                         bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
1871                 }
1872                 break;
1873
1874         case NVMET_FCOP_WRITEDATA:
1875                 /* Words 0 - 2 : The first sg segment */
1876                 txrdy = pci_pool_alloc(phba->txrdy_payload_pool,
1877                                        GFP_KERNEL, &physaddr);
1878                 if (!txrdy) {
1879                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1880                                         "6041 Bad txrdy buffer: oxid x%x\n",
1881                                         ctxp->oxid);
1882                         return NULL;
1883                 }
1884                 ctxp->txrdy = txrdy;
1885                 ctxp->txrdy_phys = physaddr;
1886                 wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1887                 wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN;
1888                 wqe->fcp_treceive.bde.addrLow =
1889                         cpu_to_le32(putPaddrLow(physaddr));
1890                 wqe->fcp_treceive.bde.addrHigh =
1891                         cpu_to_le32(putPaddrHigh(physaddr));
1892
1893                 /* Word 3 */
1894                 wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
1895
1896                 /* Word 4 */
1897                 wqe->fcp_treceive.relative_offset = ctxp->offset;
1898
1899                 /* Word 5 */
1900
1901                 /* Word 6 */
1902                 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
1903                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1904                 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
1905                        nvmewqe->sli4_xritag);
1906
1907                 /* Word 7 */
1908                 bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
1909                 bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com,
1910                        CMD_FCP_TRECEIVE64_WQE);
1911
1912                 /* Word 8 */
1913                 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
1914
1915                 /* Word 9 */
1916                 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
1917                 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
1918
1919                 /* Word 10 */
1920                 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
1921                 bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
1922                 bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
1923                 bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com,
1924                        LPFC_WQE_LENLOC_WORD12);
1925                 bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, xc);
1926                 bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
1927                 bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
1928                 bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
1929                 bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
1930                 if (phba->cfg_nvme_oas)
1931                         bf_set(wqe_oas, &wqe->fcp_treceive.wqe_com, 1);
1932
1933                 /* Word 11 */
1934                 bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com,
1935                        LPFC_WQE_CQ_ID_DEFAULT);
1936                 bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com,
1937                        FCP_COMMAND_TRECEIVE);
1938                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1939
1940                 /* Word 12 */
1941                 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
1942
1943                 /* Setup 1 TXRDY and 1 SKIP SGE */
1944                 txrdy[0] = 0;
1945                 txrdy[1] = cpu_to_be32(rsp->transfer_length);
1946                 txrdy[2] = 0;
1947
1948                 sgl->addr_hi = putPaddrHigh(physaddr);
1949                 sgl->addr_lo = putPaddrLow(physaddr);
1950                 sgl->word2 = 0;
1951                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
1952                 sgl->word2 = cpu_to_le32(sgl->word2);
1953                 sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN);
1954                 sgl++;
1955                 sgl->addr_hi = 0;
1956                 sgl->addr_lo = 0;
1957                 sgl->word2 = 0;
1958                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1959                 sgl->word2 = cpu_to_le32(sgl->word2);
1960                 sgl->sge_len = 0;
1961                 sgl++;
1962                 atomic_inc(&tgtp->xmt_fcp_write);
1963                 break;
1964
1965         case NVMET_FCOP_RSP:
1966                 /* Words 0 - 2 */
1967                 physaddr = rsp->rspdma;
1968                 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1969                 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
1970                 wqe->fcp_trsp.bde.addrLow =
1971                         cpu_to_le32(putPaddrLow(physaddr));
1972                 wqe->fcp_trsp.bde.addrHigh =
1973                         cpu_to_le32(putPaddrHigh(physaddr));
1974
1975                 /* Word 3 */
1976                 wqe->fcp_trsp.response_len = rsp->rsplen;
1977
1978                 /* Word 4 */
1979                 wqe->fcp_trsp.rsvd_4_5[0] = 0;
1980
1981
1982                 /* Word 5 */
1983
1984                 /* Word 6 */
1985                 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
1986                        phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1987                 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
1988                        nvmewqe->sli4_xritag);
1989
1990                 /* Word 7 */
1991                 bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1);
1992                 bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
1993
1994                 /* Word 8 */
1995                 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
1996
1997                 /* Word 9 */
1998                 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
1999                 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
2000
2001                 /* Word 10 */
2002                 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
2003                 bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 0);
2004                 bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_WRITE);
2005                 bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com,
2006                        LPFC_WQE_LENLOC_WORD3);
2007                 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, xc);
2008                 bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
2009                 if (phba->cfg_nvme_oas)
2010                         bf_set(wqe_oas, &wqe->fcp_trsp.wqe_com, 1);
2011
2012                 /* Word 11 */
2013                 bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com,
2014                        LPFC_WQE_CQ_ID_DEFAULT);
2015                 bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com,
2016                        FCP_COMMAND_TRSP);
2017                 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
2018
2019                 if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
2020                         /* Good response - all zero's on wire */
2021                         bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
2022                         bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
2023                         bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
2024                 } else {
2025                         bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
2026                         bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
2027                         bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
2028                                ((rsp->rsplen >> 2) - 1));
2029                         memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
2030                 }
2031
2032                 /* Use rspbuf, NOT sg list */
2033                 rsp->sg_cnt = 0;
2034                 sgl->word2 = 0;
2035                 atomic_inc(&tgtp->xmt_fcp_rsp);
2036                 break;
2037
2038         default:
2039                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2040                                 "6064 Unknown Rsp Op %d\n",
2041                                 rsp->op);
2042                 return NULL;
2043         }
2044
2045         nvmewqe->retry = 1;
2046         nvmewqe->vport = phba->pport;
2047         nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
2048         nvmewqe->context1 = ndlp;
2049
2050         for (i = 0; i < rsp->sg_cnt; i++) {
2051                 sgel = &rsp->sg[i];
2052                 physaddr = sg_dma_address(sgel);
2053                 cnt = sg_dma_len(sgel);
2054                 sgl->addr_hi = putPaddrHigh(physaddr);
2055                 sgl->addr_lo = putPaddrLow(physaddr);
2056                 sgl->word2 = 0;
2057                 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
2058                 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
2059                 if ((i+1) == rsp->sg_cnt)
2060                         bf_set(lpfc_sli4_sge_last, sgl, 1);
2061                 sgl->word2 = cpu_to_le32(sgl->word2);
2062                 sgl->sge_len = cpu_to_le32(cnt);
2063                 sgl++;
2064                 ctxp->offset += cnt;
2065         }
2066         ctxp->state = LPFC_NVMET_STE_DATA;
2067         ctxp->entry_cnt++;
2068         return nvmewqe;
2069 }
2070
2071 /**
2072  * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
2073  * @phba: Pointer to HBA context object.
2074  * @cmdwqe: Pointer to driver command WQE object.
2075  * @wcqe: Pointer to driver response CQE object.
2076  *
2077  * The function is called from SLI ring event handler with no
2078  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
2079  * The function frees memory resources used for the NVME commands.
2080  **/
2081 static void
2082 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2083                              struct lpfc_wcqe_complete *wcqe)
2084 {
2085         struct lpfc_nvmet_rcv_ctx *ctxp;
2086         struct lpfc_nvmet_tgtport *tgtp;
2087         uint32_t status, result;
2088         unsigned long flags;
2089         bool released = false;
2090
2091         ctxp = cmdwqe->context2;
2092         status = bf_get(lpfc_wcqe_c_status, wcqe);
2093         result = wcqe->parameter;
2094
2095         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2096         if (ctxp->flag & LPFC_NVMET_ABORT_OP)
2097                 atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
2098
2099         ctxp->state = LPFC_NVMET_STE_DONE;
2100
2101         /* Check if we already received a free context call
2102          * and we have completed processing an abort situation.
2103          */
2104         spin_lock_irqsave(&ctxp->ctxlock, flags);
2105         if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
2106             !(ctxp->flag & LPFC_NVMET_XBUSY)) {
2107                 list_del(&ctxp->list);
2108                 released = true;
2109         }
2110         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2111         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
2112         atomic_inc(&tgtp->xmt_abort_rsp);
2113
2114         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2115                         "6165 ABORT cmpl: xri x%x flg x%x (%d) "
2116                         "WCQE: %08x %08x %08x %08x\n",
2117                         ctxp->oxid, ctxp->flag, released,
2118                         wcqe->word0, wcqe->total_data_placed,
2119                         result, wcqe->word3);
2120
2121         cmdwqe->context2 = NULL;
2122         cmdwqe->context3 = NULL;
2123         /*
2124          * if transport has released ctx, then can reuse it. Otherwise,
2125          * will be recycled by transport release call.
2126          */
2127         if (released)
2128                 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
2129
2130         /* This is the iocbq for the abort, not the command */
2131         lpfc_sli_release_iocbq(phba, cmdwqe);
2132
2133         /* Since iaab/iaar are NOT set, there is no work left.
2134          * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
2135          * should have been called already.
2136          */
2137 }
2138
2139 /**
2140  * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS
2141  * @phba: Pointer to HBA context object.
2142  * @cmdwqe: Pointer to driver command WQE object.
2143  * @wcqe: Pointer to driver response CQE object.
2144  *
2145  * The function is called from SLI ring event handler with no
2146  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
2147  * The function frees memory resources used for the NVME commands.
2148  **/
2149 static void
2150 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2151                                struct lpfc_wcqe_complete *wcqe)
2152 {
2153         struct lpfc_nvmet_rcv_ctx *ctxp;
2154         struct lpfc_nvmet_tgtport *tgtp;
2155         unsigned long flags;
2156         uint32_t status, result;
2157         bool released = false;
2158
2159         ctxp = cmdwqe->context2;
2160         status = bf_get(lpfc_wcqe_c_status, wcqe);
2161         result = wcqe->parameter;
2162
2163         if (!ctxp) {
2164                 /* if context is clear, related io alrady complete */
2165                 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2166                                 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",
2167                                 wcqe->word0, wcqe->total_data_placed,
2168                                 result, wcqe->word3);
2169                 return;
2170         }
2171
2172         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2173         if (ctxp->flag & LPFC_NVMET_ABORT_OP)
2174                 atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
2175
2176         /* Sanity check */
2177         if (ctxp->state != LPFC_NVMET_STE_ABORT) {
2178                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2179                                 "6112 ABTS Wrong state:%d oxid x%x\n",
2180                                 ctxp->state, ctxp->oxid);
2181         }
2182
2183         /* Check if we already received a free context call
2184          * and we have completed processing an abort situation.
2185          */
2186         ctxp->state = LPFC_NVMET_STE_DONE;
2187         spin_lock_irqsave(&ctxp->ctxlock, flags);
2188         if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
2189             !(ctxp->flag & LPFC_NVMET_XBUSY)) {
2190                 list_del(&ctxp->list);
2191                 released = true;
2192         }
2193         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2194         spin_unlock_irqrestore(&ctxp->ctxlock, flags);
2195         atomic_inc(&tgtp->xmt_abort_rsp);
2196
2197         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2198                         "6316 ABTS cmpl xri x%x flg x%x (%x) "
2199                         "WCQE: %08x %08x %08x %08x\n",
2200                         ctxp->oxid, ctxp->flag, released,
2201                         wcqe->word0, wcqe->total_data_placed,
2202                         result, wcqe->word3);
2203
2204         cmdwqe->context2 = NULL;
2205         cmdwqe->context3 = NULL;
2206         /*
2207          * if transport has released ctx, then can reuse it. Otherwise,
2208          * will be recycled by transport release call.
2209          */
2210         if (released)
2211                 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
2212
2213         /* Since iaab/iaar are NOT set, there is no work left.
2214          * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
2215          * should have been called already.
2216          */
2217 }
2218
2219 /**
2220  * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
2221  * @phba: Pointer to HBA context object.
2222  * @cmdwqe: Pointer to driver command WQE object.
2223  * @wcqe: Pointer to driver response CQE object.
2224  *
2225  * The function is called from SLI ring event handler with no
2226  * lock held. This function is the completion handler for NVME ABTS for LS cmds
2227  * The function frees memory resources used for the NVME commands.
2228  **/
2229 static void
2230 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2231                             struct lpfc_wcqe_complete *wcqe)
2232 {
2233         struct lpfc_nvmet_rcv_ctx *ctxp;
2234         struct lpfc_nvmet_tgtport *tgtp;
2235         uint32_t status, result;
2236
2237         ctxp = cmdwqe->context2;
2238         status = bf_get(lpfc_wcqe_c_status, wcqe);
2239         result = wcqe->parameter;
2240
2241         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2242         atomic_inc(&tgtp->xmt_ls_abort_cmpl);
2243
2244         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2245                         "6083 Abort cmpl: ctx %p WCQE:%08x %08x %08x %08x\n",
2246                         ctxp, wcqe->word0, wcqe->total_data_placed,
2247                         result, wcqe->word3);
2248
2249         if (!ctxp) {
2250                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2251                                 "6415 NVMET LS Abort No ctx: WCQE: "
2252                                  "%08x %08x %08x %08x\n",
2253                                 wcqe->word0, wcqe->total_data_placed,
2254                                 result, wcqe->word3);
2255
2256                 lpfc_sli_release_iocbq(phba, cmdwqe);
2257                 return;
2258         }
2259
2260         if (ctxp->state != LPFC_NVMET_STE_LS_ABORT) {
2261                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2262                                 "6416 NVMET LS abort cmpl state mismatch: "
2263                                 "oxid x%x: %d %d\n",
2264                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
2265         }
2266
2267         cmdwqe->context2 = NULL;
2268         cmdwqe->context3 = NULL;
2269         lpfc_sli_release_iocbq(phba, cmdwqe);
2270         kfree(ctxp);
2271 }
2272
2273 static int
2274 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
2275                              struct lpfc_nvmet_rcv_ctx *ctxp,
2276                              uint32_t sid, uint16_t xri)
2277 {
2278         struct lpfc_nvmet_tgtport *tgtp;
2279         struct lpfc_iocbq *abts_wqeq;
2280         union lpfc_wqe *wqe_abts;
2281         struct lpfc_nodelist *ndlp;
2282
2283         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2284                         "6067 ABTS: sid %x xri x%x/x%x\n",
2285                         sid, xri, ctxp->wqeq->sli4_xritag);
2286
2287         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2288
2289         ndlp = lpfc_findnode_did(phba->pport, sid);
2290         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2291             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2292             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2293                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2294                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2295                                 "6134 Drop ABTS - wrong NDLP state x%x.\n",
2296                                 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2297
2298                 /* No failure to an ABTS request. */
2299                 return 0;
2300         }
2301
2302         abts_wqeq = ctxp->wqeq;
2303         wqe_abts = &abts_wqeq->wqe;
2304
2305         /*
2306          * Since we zero the whole WQE, we need to ensure we set the WQE fields
2307          * that were initialized in lpfc_sli4_nvmet_alloc.
2308          */
2309         memset(wqe_abts, 0, sizeof(union lpfc_wqe));
2310
2311         /* Word 5 */
2312         bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
2313         bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
2314         bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
2315         bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
2316         bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
2317
2318         /* Word 6 */
2319         bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
2320                phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2321         bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
2322                abts_wqeq->sli4_xritag);
2323
2324         /* Word 7 */
2325         bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
2326                CMD_XMIT_SEQUENCE64_WQE);
2327         bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
2328         bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
2329         bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
2330
2331         /* Word 8 */
2332         wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
2333
2334         /* Word 9 */
2335         bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
2336         /* Needs to be set by caller */
2337         bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
2338
2339         /* Word 10 */
2340         bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1);
2341         bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
2342         bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
2343                LPFC_WQE_LENLOC_WORD12);
2344         bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
2345         bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
2346
2347         /* Word 11 */
2348         bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
2349                LPFC_WQE_CQ_ID_DEFAULT);
2350         bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
2351                OTHER_COMMAND);
2352
2353         abts_wqeq->vport = phba->pport;
2354         abts_wqeq->context1 = ndlp;
2355         abts_wqeq->context2 = ctxp;
2356         abts_wqeq->context3 = NULL;
2357         abts_wqeq->rsvd2 = 0;
2358         /* hba_wqidx should already be setup from command we are aborting */
2359         abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
2360         abts_wqeq->iocb.ulpLe = 1;
2361
2362         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2363                         "6069 Issue ABTS to xri x%x reqtag x%x\n",
2364                         xri, abts_wqeq->iotag);
2365         return 1;
2366 }
2367
2368 static int
2369 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
2370                                struct lpfc_nvmet_rcv_ctx *ctxp,
2371                                uint32_t sid, uint16_t xri)
2372 {
2373         struct lpfc_nvmet_tgtport *tgtp;
2374         struct lpfc_iocbq *abts_wqeq;
2375         union lpfc_wqe *abts_wqe;
2376         struct lpfc_nodelist *ndlp;
2377         unsigned long flags;
2378         int rc;
2379
2380         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2381         if (!ctxp->wqeq) {
2382                 ctxp->wqeq = ctxp->ctxbuf->iocbq;
2383                 ctxp->wqeq->hba_wqidx = 0;
2384         }
2385
2386         ndlp = lpfc_findnode_did(phba->pport, sid);
2387         if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2388             ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2389             (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2390                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2391                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2392                                 "6160 Drop ABORT - wrong NDLP state x%x.\n",
2393                                 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2394
2395                 /* No failure to an ABTS request. */
2396                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2397                 return 0;
2398         }
2399
2400         /* Issue ABTS for this WQE based on iotag */
2401         ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
2402         if (!ctxp->abort_wqeq) {
2403                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2404                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2405                                 "6161 ABORT failed: No wqeqs: "
2406                                 "xri: x%x\n", ctxp->oxid);
2407                 /* No failure to an ABTS request. */
2408                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2409                 return 0;
2410         }
2411         abts_wqeq = ctxp->abort_wqeq;
2412         abts_wqe = &abts_wqeq->wqe;
2413         ctxp->state = LPFC_NVMET_STE_ABORT;
2414
2415         /* Announce entry to new IO submit field. */
2416         lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2417                         "6162 ABORT Request to rport DID x%06x "
2418                         "for xri x%x x%x\n",
2419                         ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
2420
2421         /* If the hba is getting reset, this flag is set.  It is
2422          * cleared when the reset is complete and rings reestablished.
2423          */
2424         spin_lock_irqsave(&phba->hbalock, flags);
2425         /* driver queued commands are in process of being flushed */
2426         if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) {
2427                 spin_unlock_irqrestore(&phba->hbalock, flags);
2428                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2429                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
2430                                 "6163 Driver in reset cleanup - flushing "
2431                                 "NVME Req now. hba_flag x%x oxid x%x\n",
2432                                 phba->hba_flag, ctxp->oxid);
2433                 lpfc_sli_release_iocbq(phba, abts_wqeq);
2434                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2435                 return 0;
2436         }
2437
2438         /* Outstanding abort is in progress */
2439         if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) {
2440                 spin_unlock_irqrestore(&phba->hbalock, flags);
2441                 atomic_inc(&tgtp->xmt_abort_rsp_error);
2442                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
2443                                 "6164 Outstanding NVME I/O Abort Request "
2444                                 "still pending on oxid x%x\n",
2445                                 ctxp->oxid);
2446                 lpfc_sli_release_iocbq(phba, abts_wqeq);
2447                 ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2448                 return 0;
2449         }
2450
2451         /* Ready - mark outstanding as aborted by driver. */
2452         abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED;
2453
2454         /* WQEs are reused.  Clear stale data and set key fields to
2455          * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
2456          */
2457         memset(abts_wqe, 0, sizeof(union lpfc_wqe));
2458
2459         /* word 3 */
2460         bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
2461
2462         /* word 7 */
2463         bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
2464         bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
2465
2466         /* word 8 - tell the FW to abort the IO associated with this
2467          * outstanding exchange ID.
2468          */
2469         abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag;
2470
2471         /* word 9 - this is the iotag for the abts_wqe completion. */
2472         bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
2473                abts_wqeq->iotag);
2474
2475         /* word 10 */
2476         bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
2477         bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
2478
2479         /* word 11 */
2480         bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
2481         bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
2482         bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
2483
2484         /* ABTS WQE must go to the same WQ as the WQE to be aborted */
2485         abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
2486         abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
2487         abts_wqeq->iocb_cmpl = 0;
2488         abts_wqeq->iocb_flag |= LPFC_IO_NVME;
2489         abts_wqeq->context2 = ctxp;
2490         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
2491         spin_unlock_irqrestore(&phba->hbalock, flags);
2492         if (rc == WQE_SUCCESS) {
2493                 atomic_inc(&tgtp->xmt_abort_sol);
2494                 return 0;
2495         }
2496
2497         atomic_inc(&tgtp->xmt_abort_rsp_error);
2498         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2499         lpfc_sli_release_iocbq(phba, abts_wqeq);
2500         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2501                         "6166 Failed ABORT issue_wqe with status x%x "
2502                         "for oxid x%x.\n",
2503                         rc, ctxp->oxid);
2504         return 1;
2505 }
2506
2507
2508 static int
2509 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
2510                                  struct lpfc_nvmet_rcv_ctx *ctxp,
2511                                  uint32_t sid, uint16_t xri)
2512 {
2513         struct lpfc_nvmet_tgtport *tgtp;
2514         struct lpfc_iocbq *abts_wqeq;
2515         unsigned long flags;
2516         int rc;
2517
2518         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2519         if (!ctxp->wqeq) {
2520                 ctxp->wqeq = ctxp->ctxbuf->iocbq;
2521                 ctxp->wqeq->hba_wqidx = 0;
2522         }
2523
2524         if (ctxp->state == LPFC_NVMET_STE_FREE) {
2525                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2526                                 "6417 NVMET ABORT ctx freed %d %d oxid x%x\n",
2527                                 ctxp->state, ctxp->entry_cnt, ctxp->oxid);
2528                 rc = WQE_BUSY;
2529                 goto aerr;
2530         }
2531         ctxp->state = LPFC_NVMET_STE_ABORT;
2532         ctxp->entry_cnt++;
2533         rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
2534         if (rc == 0)
2535                 goto aerr;
2536
2537         spin_lock_irqsave(&phba->hbalock, flags);
2538         abts_wqeq = ctxp->wqeq;
2539         abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;
2540         abts_wqeq->iocb_cmpl = NULL;
2541         abts_wqeq->iocb_flag |= LPFC_IO_NVMET;
2542         rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
2543         spin_unlock_irqrestore(&phba->hbalock, flags);
2544         if (rc == WQE_SUCCESS) {
2545                 return 0;
2546         }
2547
2548 aerr:
2549         atomic_inc(&tgtp->xmt_abort_rsp_error);
2550         ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2551         atomic_inc(&tgtp->xmt_abort_rsp_error);
2552         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2553                         "6135 Failed to Issue ABTS for oxid x%x. Status x%x\n",
2554                         ctxp->oxid, rc);
2555         return 1;
2556 }
2557
2558 static int
2559 lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
2560                                 struct lpfc_nvmet_rcv_ctx *ctxp,
2561                                 uint32_t sid, uint16_t xri)
2562 {
2563         struct lpfc_nvmet_tgtport *tgtp;
2564         struct lpfc_iocbq *abts_wqeq;
2565         union lpfc_wqe *wqe_abts;
2566         unsigned long flags;
2567         int rc;
2568
2569         if ((ctxp->state == LPFC_NVMET_STE_LS_RCV && ctxp->entry_cnt == 1) ||
2570             (ctxp->state == LPFC_NVMET_STE_LS_RSP && ctxp->entry_cnt == 2)) {
2571                 ctxp->state = LPFC_NVMET_STE_LS_ABORT;
2572                 ctxp->entry_cnt++;
2573         } else {
2574                 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2575                                 "6418 NVMET LS abort state mismatch "
2576                                 "IO x%x: %d %d\n",
2577                                 ctxp->oxid, ctxp->state, ctxp->entry_cnt);
2578                 ctxp->state = LPFC_NVMET_STE_LS_ABORT;
2579         }
2580
2581         tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2582         if (!ctxp->wqeq) {
2583                 /* Issue ABTS for this WQE based on iotag */
2584                 ctxp->wqeq = lpfc_sli_get_iocbq(phba);
2585                 if (!ctxp->wqeq) {
2586                         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2587                                         "6068 Abort failed: No wqeqs: "
2588                                         "xri: x%x\n", xri);
2589                         /* No failure to an ABTS request. */
2590                         kfree(ctxp);
2591                         return 0;
2592                 }
2593         }
2594         abts_wqeq = ctxp->wqeq;
2595         wqe_abts = &abts_wqeq->wqe;
2596
2597         if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {
2598                 rc = WQE_BUSY;
2599                 goto out;
2600         }
2601
2602         spin_lock_irqsave(&phba->hbalock, flags);
2603         abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
2604         abts_wqeq->iocb_cmpl = 0;
2605         abts_wqeq->iocb_flag |=  LPFC_IO_NVME_LS;
2606         rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq);
2607         spin_unlock_irqrestore(&phba->hbalock, flags);
2608         if (rc == WQE_SUCCESS) {
2609                 atomic_inc(&tgtp->xmt_abort_unsol);
2610                 return 0;
2611         }
2612 out:
2613         atomic_inc(&tgtp->xmt_abort_rsp_error);
2614         abts_wqeq->context2 = NULL;
2615         abts_wqeq->context3 = NULL;
2616         lpfc_sli_release_iocbq(phba, abts_wqeq);
2617         kfree(ctxp);
2618         lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2619                         "6056 Failed to Issue ABTS. Status x%x\n", rc);
2620         return 0;
2621 }