2 * Copyright (c) 2016 Avago Technologies. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful.
9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
13 * See the GNU General Public License for more details, a copy of which
14 * can be found in the file COPYING included with this package
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/blk-mq.h>
21 #include <linux/parser.h>
22 #include <linux/random.h>
23 #include <uapi/scsi/fc/fc_fs.h>
24 #include <uapi/scsi/fc/fc_els.h>
27 #include <linux/nvme-fc-driver.h>
28 #include <linux/nvme-fc.h>
31 /* *************************** Data Structures/Defines ****************** */
34 #define NVMET_LS_CTX_COUNT 4
36 /* for this implementation, assume small single frame rqst/rsp */
37 #define NVME_FC_MAX_LS_BUFFER_SIZE 2048
39 struct nvmet_fc_tgtport;
40 struct nvmet_fc_tgt_assoc;
42 struct nvmet_fc_ls_iod {
43 struct nvmefc_tgt_ls_req *lsreq;
44 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
46 struct list_head ls_list; /* tgtport->ls_list */
48 struct nvmet_fc_tgtport *tgtport;
49 struct nvmet_fc_tgt_assoc *assoc;
56 struct scatterlist sg[2];
58 struct work_struct work;
59 } __aligned(sizeof(unsigned long long));
61 #define NVMET_FC_MAX_KB_PER_XFR 256
63 enum nvmet_fcp_datadir {
70 struct nvmet_fc_fcp_iod {
71 struct nvmefc_tgt_fcp_req *fcpreq;
73 struct nvme_fc_cmd_iu cmdiubuf;
74 struct nvme_fc_ersp_iu rspiubuf;
76 struct scatterlist *data_sg;
77 struct scatterlist *next_sg;
82 enum nvmet_fcp_datadir io_dir;
90 struct work_struct work;
91 struct work_struct done_work;
93 struct nvmet_fc_tgtport *tgtport;
94 struct nvmet_fc_tgt_queue *queue;
96 struct list_head fcp_list; /* tgtport->fcp_list */
99 struct nvmet_fc_tgtport {
101 struct nvmet_fc_target_port fc_target_port;
103 struct list_head tgt_list; /* nvmet_fc_target_list */
104 struct device *dev; /* dev for dma mapping */
105 struct nvmet_fc_target_template *ops;
107 struct nvmet_fc_ls_iod *iod;
109 struct list_head ls_list;
110 struct list_head ls_busylist;
111 struct list_head assoc_list;
112 struct ida assoc_cnt;
113 struct nvmet_port *port;
117 struct nvmet_fc_tgt_queue {
129 struct nvmet_port *port;
130 struct nvmet_cq nvme_cq;
131 struct nvmet_sq nvme_sq;
132 struct nvmet_fc_tgt_assoc *assoc;
133 struct nvmet_fc_fcp_iod *fod; /* array of fcp_iods */
134 struct list_head fod_list;
135 struct workqueue_struct *work_q;
137 } __aligned(sizeof(unsigned long long));
139 struct nvmet_fc_tgt_assoc {
142 struct nvmet_fc_tgtport *tgtport;
143 struct list_head a_list;
144 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES];
150 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
152 return (iodptr - iodptr->tgtport->iod);
156 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
158 return (fodptr - fodptr->queue->fod);
163 * Association and Connection IDs:
165 * Association ID will have random number in upper 6 bytes and zero
168 * Connection IDs will be Association ID with QID or'd in lower 2 bytes
170 * note: Association ID = Connection ID for queue 0
172 #define BYTES_FOR_QID sizeof(u16)
173 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
174 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
177 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
179 return (assoc->association_id | qid);
183 nvmet_fc_getassociationid(u64 connectionid)
185 return connectionid & ~NVMET_FC_QUEUEID_MASK;
189 nvmet_fc_getqueueid(u64 connectionid)
191 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
194 static inline struct nvmet_fc_tgtport *
195 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
197 return container_of(targetport, struct nvmet_fc_tgtport,
201 static inline struct nvmet_fc_fcp_iod *
202 nvmet_req_to_fod(struct nvmet_req *nvme_req)
204 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
208 /* *************************** Globals **************************** */
211 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
213 static LIST_HEAD(nvmet_fc_target_list);
214 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
217 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
218 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
219 static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work);
220 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
221 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
222 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
223 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
224 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
225 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
228 /* *********************** FC-NVME DMA Handling **************************** */
231 * The fcloop device passes in a NULL device pointer. Real LLD's will
232 * pass in a valid device pointer. If NULL is passed to the dma mapping
233 * routines, depending on the platform, it may or may not succeed, and
237 * Wrapper all the dma routines and check the dev pointer.
239 * If simple mappings (return just a dma address, we'll noop them,
240 * returning a dma address of 0.
242 * On more complex mappings (dma_map_sg), a pseudo routine fills
243 * in the scatter list, setting all dma addresses to 0.
246 static inline dma_addr_t
247 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
248 enum dma_data_direction dir)
250 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
254 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
256 return dev ? dma_mapping_error(dev, dma_addr) : 0;
260 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
261 enum dma_data_direction dir)
264 dma_unmap_single(dev, addr, size, dir);
268 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
269 enum dma_data_direction dir)
272 dma_sync_single_for_cpu(dev, addr, size, dir);
276 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
277 enum dma_data_direction dir)
280 dma_sync_single_for_device(dev, addr, size, dir);
283 /* pseudo dma_map_sg call */
285 fc_map_sg(struct scatterlist *sg, int nents)
287 struct scatterlist *s;
290 WARN_ON(nents == 0 || sg[0].length == 0);
292 for_each_sg(sg, s, nents, i) {
294 #ifdef CONFIG_NEED_SG_DMA_LENGTH
295 s->dma_length = s->length;
302 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
303 enum dma_data_direction dir)
305 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
309 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
310 enum dma_data_direction dir)
313 dma_unmap_sg(dev, sg, nents, dir);
317 /* *********************** FC-NVME Port Management ************************ */
321 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
323 struct nvmet_fc_ls_iod *iod;
326 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
333 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
334 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
335 iod->tgtport = tgtport;
336 list_add_tail(&iod->ls_list, &tgtport->ls_list);
338 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
343 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
345 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
346 NVME_FC_MAX_LS_BUFFER_SIZE,
348 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
356 list_del(&iod->ls_list);
357 for (iod--, i--; i >= 0; iod--, i--) {
358 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
359 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
361 list_del(&iod->ls_list);
370 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
372 struct nvmet_fc_ls_iod *iod = tgtport->iod;
375 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
376 fc_dma_unmap_single(tgtport->dev,
377 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
380 list_del(&iod->ls_list);
385 static struct nvmet_fc_ls_iod *
386 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
388 static struct nvmet_fc_ls_iod *iod;
391 spin_lock_irqsave(&tgtport->lock, flags);
392 iod = list_first_entry_or_null(&tgtport->ls_list,
393 struct nvmet_fc_ls_iod, ls_list);
395 list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
396 spin_unlock_irqrestore(&tgtport->lock, flags);
402 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
403 struct nvmet_fc_ls_iod *iod)
407 spin_lock_irqsave(&tgtport->lock, flags);
408 list_move(&iod->ls_list, &tgtport->ls_list);
409 spin_unlock_irqrestore(&tgtport->lock, flags);
413 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
414 struct nvmet_fc_tgt_queue *queue)
416 struct nvmet_fc_fcp_iod *fod = queue->fod;
419 for (i = 0; i < queue->sqsize; fod++, i++) {
420 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work);
421 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work);
422 fod->tgtport = tgtport;
426 fod->aborted = false;
428 list_add_tail(&fod->fcp_list, &queue->fod_list);
429 spin_lock_init(&fod->flock);
431 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
432 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
433 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
434 list_del(&fod->fcp_list);
435 for (fod--, i--; i >= 0; fod--, i--) {
436 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
437 sizeof(fod->rspiubuf),
440 list_del(&fod->fcp_list);
449 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
450 struct nvmet_fc_tgt_queue *queue)
452 struct nvmet_fc_fcp_iod *fod = queue->fod;
455 for (i = 0; i < queue->sqsize; fod++, i++) {
457 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
458 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
462 static struct nvmet_fc_fcp_iod *
463 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
465 static struct nvmet_fc_fcp_iod *fod;
468 spin_lock_irqsave(&queue->qlock, flags);
469 fod = list_first_entry_or_null(&queue->fod_list,
470 struct nvmet_fc_fcp_iod, fcp_list);
472 list_del(&fod->fcp_list);
475 * no queue reference is taken, as it was taken by the
476 * queue lookup just prior to the allocation. The iod
477 * will "inherit" that reference.
480 spin_unlock_irqrestore(&queue->qlock, flags);
486 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
487 struct nvmet_fc_fcp_iod *fod)
489 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
490 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
493 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
494 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
496 fcpreq->nvmet_fc_private = NULL;
498 spin_lock_irqsave(&queue->qlock, flags);
499 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
502 fod->aborted = false;
503 fod->writedataactive = false;
505 spin_unlock_irqrestore(&queue->qlock, flags);
508 * release the reference taken at queue lookup and fod allocation
510 nvmet_fc_tgt_q_put(queue);
512 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
516 nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid)
520 if (tgtport->ops->max_hw_queues == 1)
521 return WORK_CPU_UNBOUND;
523 /* Simple cpu selection based on qid modulo active cpu count */
524 idx = !qid ? 0 : (qid - 1) % num_active_cpus();
526 /* find the n'th active cpu */
527 for (cpu = 0, cnt = 0; ; ) {
528 if (cpu_active(cpu)) {
533 cpu = (cpu + 1) % num_possible_cpus();
539 static struct nvmet_fc_tgt_queue *
540 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
543 struct nvmet_fc_tgt_queue *queue;
547 if (qid >= NVMET_NR_QUEUES)
550 queue = kzalloc((sizeof(*queue) +
551 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
556 if (!nvmet_fc_tgt_a_get(assoc))
559 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
560 assoc->tgtport->fc_target_port.port_num,
565 queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
567 queue->sqsize = sqsize;
568 queue->assoc = assoc;
569 queue->port = assoc->tgtport->port;
570 queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid);
571 INIT_LIST_HEAD(&queue->fod_list);
572 atomic_set(&queue->connected, 0);
573 atomic_set(&queue->sqtail, 0);
574 atomic_set(&queue->rsn, 1);
575 atomic_set(&queue->zrspcnt, 0);
576 spin_lock_init(&queue->qlock);
577 kref_init(&queue->ref);
579 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
581 ret = nvmet_sq_init(&queue->nvme_sq);
583 goto out_fail_iodlist;
585 WARN_ON(assoc->queues[qid]);
586 spin_lock_irqsave(&assoc->tgtport->lock, flags);
587 assoc->queues[qid] = queue;
588 spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
593 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
594 destroy_workqueue(queue->work_q);
596 nvmet_fc_tgt_a_put(assoc);
604 nvmet_fc_tgt_queue_free(struct kref *ref)
606 struct nvmet_fc_tgt_queue *queue =
607 container_of(ref, struct nvmet_fc_tgt_queue, ref);
610 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
611 queue->assoc->queues[queue->qid] = NULL;
612 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
614 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
616 nvmet_fc_tgt_a_put(queue->assoc);
618 destroy_workqueue(queue->work_q);
624 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
626 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
630 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
632 return kref_get_unless_zero(&queue->ref);
637 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
639 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
640 struct nvmet_fc_fcp_iod *fod = queue->fod;
642 int i, writedataactive;
645 disconnect = atomic_xchg(&queue->connected, 0);
647 spin_lock_irqsave(&queue->qlock, flags);
648 /* about outstanding io's */
649 for (i = 0; i < queue->sqsize; fod++, i++) {
651 spin_lock(&fod->flock);
653 writedataactive = fod->writedataactive;
654 spin_unlock(&fod->flock);
656 * only call lldd abort routine if waiting for
657 * writedata. other outstanding ops should finish
660 if (writedataactive) {
661 spin_lock(&fod->flock);
663 spin_unlock(&fod->flock);
664 tgtport->ops->fcp_abort(
665 &tgtport->fc_target_port, fod->fcpreq);
669 spin_unlock_irqrestore(&queue->qlock, flags);
671 flush_workqueue(queue->work_q);
674 nvmet_sq_destroy(&queue->nvme_sq);
676 nvmet_fc_tgt_q_put(queue);
679 static struct nvmet_fc_tgt_queue *
680 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
683 struct nvmet_fc_tgt_assoc *assoc;
684 struct nvmet_fc_tgt_queue *queue;
685 u64 association_id = nvmet_fc_getassociationid(connection_id);
686 u16 qid = nvmet_fc_getqueueid(connection_id);
689 spin_lock_irqsave(&tgtport->lock, flags);
690 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
691 if (association_id == assoc->association_id) {
692 queue = assoc->queues[qid];
694 (!atomic_read(&queue->connected) ||
695 !nvmet_fc_tgt_q_get(queue)))
697 spin_unlock_irqrestore(&tgtport->lock, flags);
701 spin_unlock_irqrestore(&tgtport->lock, flags);
705 static struct nvmet_fc_tgt_assoc *
706 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
708 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
712 bool needrandom = true;
714 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
718 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
722 if (!nvmet_fc_tgtport_get(tgtport))
725 assoc->tgtport = tgtport;
727 INIT_LIST_HEAD(&assoc->a_list);
728 kref_init(&assoc->ref);
731 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
732 ran = ran << BYTES_FOR_QID_SHIFT;
734 spin_lock_irqsave(&tgtport->lock, flags);
736 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
737 if (ran == tmpassoc->association_id) {
742 assoc->association_id = ran;
743 list_add_tail(&assoc->a_list, &tgtport->assoc_list);
745 spin_unlock_irqrestore(&tgtport->lock, flags);
751 ida_simple_remove(&tgtport->assoc_cnt, idx);
758 nvmet_fc_target_assoc_free(struct kref *ref)
760 struct nvmet_fc_tgt_assoc *assoc =
761 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
762 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
765 spin_lock_irqsave(&tgtport->lock, flags);
766 list_del(&assoc->a_list);
767 spin_unlock_irqrestore(&tgtport->lock, flags);
768 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
770 nvmet_fc_tgtport_put(tgtport);
774 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
776 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
780 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
782 return kref_get_unless_zero(&assoc->ref);
786 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
788 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
789 struct nvmet_fc_tgt_queue *queue;
793 spin_lock_irqsave(&tgtport->lock, flags);
794 for (i = NVMET_NR_QUEUES - 1; i >= 0; i--) {
795 queue = assoc->queues[i];
797 if (!nvmet_fc_tgt_q_get(queue))
799 spin_unlock_irqrestore(&tgtport->lock, flags);
800 nvmet_fc_delete_target_queue(queue);
801 nvmet_fc_tgt_q_put(queue);
802 spin_lock_irqsave(&tgtport->lock, flags);
805 spin_unlock_irqrestore(&tgtport->lock, flags);
807 nvmet_fc_tgt_a_put(assoc);
810 static struct nvmet_fc_tgt_assoc *
811 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
814 struct nvmet_fc_tgt_assoc *assoc;
815 struct nvmet_fc_tgt_assoc *ret = NULL;
818 spin_lock_irqsave(&tgtport->lock, flags);
819 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
820 if (association_id == assoc->association_id) {
822 nvmet_fc_tgt_a_get(assoc);
826 spin_unlock_irqrestore(&tgtport->lock, flags);
833 * nvme_fc_register_targetport - transport entry point called by an
834 * LLDD to register the existence of a local
835 * NVME subystem FC port.
836 * @pinfo: pointer to information about the port to be registered
837 * @template: LLDD entrypoints and operational parameters for the port
838 * @dev: physical hardware device node port corresponds to. Will be
839 * used for DMA mappings
840 * @portptr: pointer to a local port pointer. Upon success, the routine
841 * will allocate a nvme_fc_local_port structure and place its
842 * address in the local port pointer. Upon failure, local port
843 * pointer will be set to NULL.
846 * a completion status. Must be 0 upon success; a negative errno
847 * (ex: -ENXIO) upon failure.
850 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
851 struct nvmet_fc_target_template *template,
853 struct nvmet_fc_target_port **portptr)
855 struct nvmet_fc_tgtport *newrec;
859 if (!template->xmt_ls_rsp || !template->fcp_op ||
860 !template->fcp_abort ||
861 !template->fcp_req_release || !template->targetport_delete ||
862 !template->max_hw_queues || !template->max_sgl_segments ||
863 !template->max_dif_sgl_segments || !template->dma_boundary) {
865 goto out_regtgt_failed;
868 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
872 goto out_regtgt_failed;
875 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
881 if (!get_device(dev) && dev) {
886 newrec->fc_target_port.node_name = pinfo->node_name;
887 newrec->fc_target_port.port_name = pinfo->port_name;
888 newrec->fc_target_port.private = &newrec[1];
889 newrec->fc_target_port.port_id = pinfo->port_id;
890 newrec->fc_target_port.port_num = idx;
891 INIT_LIST_HEAD(&newrec->tgt_list);
893 newrec->ops = template;
894 spin_lock_init(&newrec->lock);
895 INIT_LIST_HEAD(&newrec->ls_list);
896 INIT_LIST_HEAD(&newrec->ls_busylist);
897 INIT_LIST_HEAD(&newrec->assoc_list);
898 kref_init(&newrec->ref);
899 ida_init(&newrec->assoc_cnt);
901 ret = nvmet_fc_alloc_ls_iodlist(newrec);
904 goto out_free_newrec;
907 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
908 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
909 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
911 *portptr = &newrec->fc_target_port;
917 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
924 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
928 nvmet_fc_free_tgtport(struct kref *ref)
930 struct nvmet_fc_tgtport *tgtport =
931 container_of(ref, struct nvmet_fc_tgtport, ref);
932 struct device *dev = tgtport->dev;
935 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
936 list_del(&tgtport->tgt_list);
937 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
939 nvmet_fc_free_ls_iodlist(tgtport);
941 /* let the LLDD know we've finished tearing it down */
942 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
944 ida_simple_remove(&nvmet_fc_tgtport_cnt,
945 tgtport->fc_target_port.port_num);
947 ida_destroy(&tgtport->assoc_cnt);
955 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
957 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
961 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
963 return kref_get_unless_zero(&tgtport->ref);
967 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
969 struct nvmet_fc_tgt_assoc *assoc, *next;
972 spin_lock_irqsave(&tgtport->lock, flags);
973 list_for_each_entry_safe(assoc, next,
974 &tgtport->assoc_list, a_list) {
975 if (!nvmet_fc_tgt_a_get(assoc))
977 spin_unlock_irqrestore(&tgtport->lock, flags);
978 nvmet_fc_delete_target_assoc(assoc);
979 nvmet_fc_tgt_a_put(assoc);
980 spin_lock_irqsave(&tgtport->lock, flags);
982 spin_unlock_irqrestore(&tgtport->lock, flags);
986 * nvmet layer has called to terminate an association
989 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
991 struct nvmet_fc_tgtport *tgtport, *next;
992 struct nvmet_fc_tgt_assoc *assoc;
993 struct nvmet_fc_tgt_queue *queue;
995 bool found_ctrl = false;
997 /* this is a bit ugly, but don't want to make locks layered */
998 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
999 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1001 if (!nvmet_fc_tgtport_get(tgtport))
1003 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1005 spin_lock_irqsave(&tgtport->lock, flags);
1006 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1007 queue = assoc->queues[0];
1008 if (queue && queue->nvme_sq.ctrl == ctrl) {
1009 if (nvmet_fc_tgt_a_get(assoc))
1014 spin_unlock_irqrestore(&tgtport->lock, flags);
1016 nvmet_fc_tgtport_put(tgtport);
1019 nvmet_fc_delete_target_assoc(assoc);
1020 nvmet_fc_tgt_a_put(assoc);
1024 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1026 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1030 * nvme_fc_unregister_targetport - transport entry point called by an
1031 * LLDD to deregister/remove a previously
1032 * registered a local NVME subsystem FC port.
1033 * @tgtport: pointer to the (registered) target port that is to be
1037 * a completion status. Must be 0 upon success; a negative errno
1038 * (ex: -ENXIO) upon failure.
1041 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1043 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1045 /* terminate any outstanding associations */
1046 __nvmet_fc_free_assocs(tgtport);
1048 nvmet_fc_tgtport_put(tgtport);
1052 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1055 /* *********************** FC-NVME LS Handling **************************** */
1059 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd)
1061 struct fcnvme_ls_acc_hdr *acc = buf;
1063 acc->w0.ls_cmd = ls_cmd;
1064 acc->desc_list_len = desc_len;
1065 acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1066 acc->rqst.desc_len =
1067 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1068 acc->rqst.w0.ls_cmd = rqst_ls_cmd;
1072 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
1073 u8 reason, u8 explanation, u8 vendor)
1075 struct fcnvme_ls_rjt *rjt = buf;
1077 nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
1078 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
1080 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1081 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1082 rjt->rjt.reason_code = reason;
1083 rjt->rjt.reason_explanation = explanation;
1084 rjt->rjt.vendor = vendor;
1086 return sizeof(struct fcnvme_ls_rjt);
1089 /* Validation Error indexes into the string table below */
1092 VERR_CR_ASSOC_LEN = 1,
1093 VERR_CR_ASSOC_RQST_LEN = 2,
1094 VERR_CR_ASSOC_CMD = 3,
1095 VERR_CR_ASSOC_CMD_LEN = 4,
1096 VERR_ERSP_RATIO = 5,
1097 VERR_ASSOC_ALLOC_FAIL = 6,
1098 VERR_QUEUE_ALLOC_FAIL = 7,
1099 VERR_CR_CONN_LEN = 8,
1100 VERR_CR_CONN_RQST_LEN = 9,
1102 VERR_ASSOC_ID_LEN = 11,
1105 VERR_CONN_ID_LEN = 14,
1107 VERR_CR_CONN_CMD = 16,
1108 VERR_CR_CONN_CMD_LEN = 17,
1109 VERR_DISCONN_LEN = 18,
1110 VERR_DISCONN_RQST_LEN = 19,
1111 VERR_DISCONN_CMD = 20,
1112 VERR_DISCONN_CMD_LEN = 21,
1113 VERR_DISCONN_SCOPE = 22,
1115 VERR_RS_RQST_LEN = 24,
1117 VERR_RS_CMD_LEN = 26,
1122 static char *validation_errors[] = {
1124 "Bad CR_ASSOC Length",
1125 "Bad CR_ASSOC Rqst Length",
1127 "Bad CR_ASSOC Cmd Length",
1129 "Association Allocation Failed",
1130 "Queue Allocation Failed",
1131 "Bad CR_CONN Length",
1132 "Bad CR_CONN Rqst Length",
1133 "Not Association ID",
1134 "Bad Association ID Length",
1136 "Not Connection ID",
1137 "Bad Connection ID Length",
1140 "Bad CR_CONN Cmd Length",
1141 "Bad DISCONN Length",
1142 "Bad DISCONN Rqst Length",
1144 "Bad DISCONN Cmd Length",
1145 "Bad Disconnect Scope",
1147 "Bad RS Rqst Length",
1149 "Bad RS Cmd Length",
1151 "Bad RS Relative Offset",
1155 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1156 struct nvmet_fc_ls_iod *iod)
1158 struct fcnvme_ls_cr_assoc_rqst *rqst =
1159 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
1160 struct fcnvme_ls_cr_assoc_acc *acc =
1161 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
1162 struct nvmet_fc_tgt_queue *queue;
1165 memset(acc, 0, sizeof(*acc));
1167 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_assoc_rqst))
1168 ret = VERR_CR_ASSOC_LEN;
1169 else if (rqst->desc_list_len !=
1171 sizeof(struct fcnvme_ls_cr_assoc_rqst)))
1172 ret = VERR_CR_ASSOC_RQST_LEN;
1173 else if (rqst->assoc_cmd.desc_tag !=
1174 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1175 ret = VERR_CR_ASSOC_CMD;
1176 else if (rqst->assoc_cmd.desc_len !=
1178 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd)))
1179 ret = VERR_CR_ASSOC_CMD_LEN;
1180 else if (!rqst->assoc_cmd.ersp_ratio ||
1181 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1182 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1183 ret = VERR_ERSP_RATIO;
1186 /* new association w/ admin queue */
1187 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
1189 ret = VERR_ASSOC_ALLOC_FAIL;
1191 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1192 be16_to_cpu(rqst->assoc_cmd.sqsize));
1194 ret = VERR_QUEUE_ALLOC_FAIL;
1199 dev_err(tgtport->dev,
1200 "Create Association LS failed: %s\n",
1201 validation_errors[ret]);
1202 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1203 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1204 FCNVME_RJT_RC_LOGIC,
1205 FCNVME_RJT_EXP_NONE, 0);
1209 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1210 atomic_set(&queue->connected, 1);
1211 queue->sqhd = 0; /* best place to init value */
1213 /* format a response */
1215 iod->lsreq->rsplen = sizeof(*acc);
1217 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1219 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1220 FCNVME_LS_CREATE_ASSOCIATION);
1221 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1222 acc->associd.desc_len =
1224 sizeof(struct fcnvme_lsdesc_assoc_id));
1225 acc->associd.association_id =
1226 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1227 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1228 acc->connectid.desc_len =
1230 sizeof(struct fcnvme_lsdesc_conn_id));
1231 acc->connectid.connection_id = acc->associd.association_id;
1235 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1236 struct nvmet_fc_ls_iod *iod)
1238 struct fcnvme_ls_cr_conn_rqst *rqst =
1239 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
1240 struct fcnvme_ls_cr_conn_acc *acc =
1241 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
1242 struct nvmet_fc_tgt_queue *queue;
1245 memset(acc, 0, sizeof(*acc));
1247 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1248 ret = VERR_CR_CONN_LEN;
1249 else if (rqst->desc_list_len !=
1251 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1252 ret = VERR_CR_CONN_RQST_LEN;
1253 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1254 ret = VERR_ASSOC_ID;
1255 else if (rqst->associd.desc_len !=
1257 sizeof(struct fcnvme_lsdesc_assoc_id)))
1258 ret = VERR_ASSOC_ID_LEN;
1259 else if (rqst->connect_cmd.desc_tag !=
1260 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1261 ret = VERR_CR_CONN_CMD;
1262 else if (rqst->connect_cmd.desc_len !=
1264 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1265 ret = VERR_CR_CONN_CMD_LEN;
1266 else if (!rqst->connect_cmd.ersp_ratio ||
1267 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1268 be16_to_cpu(rqst->connect_cmd.sqsize)))
1269 ret = VERR_ERSP_RATIO;
1273 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1274 be64_to_cpu(rqst->associd.association_id));
1276 ret = VERR_NO_ASSOC;
1278 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1279 be16_to_cpu(rqst->connect_cmd.qid),
1280 be16_to_cpu(rqst->connect_cmd.sqsize));
1282 ret = VERR_QUEUE_ALLOC_FAIL;
1284 /* release get taken in nvmet_fc_find_target_assoc */
1285 nvmet_fc_tgt_a_put(iod->assoc);
1290 dev_err(tgtport->dev,
1291 "Create Connection LS failed: %s\n",
1292 validation_errors[ret]);
1293 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1294 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1295 (ret == VERR_NO_ASSOC) ?
1296 FCNVME_RJT_RC_INV_ASSOC :
1297 FCNVME_RJT_RC_LOGIC,
1298 FCNVME_RJT_EXP_NONE, 0);
1302 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1303 atomic_set(&queue->connected, 1);
1304 queue->sqhd = 0; /* best place to init value */
1306 /* format a response */
1308 iod->lsreq->rsplen = sizeof(*acc);
1310 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1311 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1312 FCNVME_LS_CREATE_CONNECTION);
1313 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1314 acc->connectid.desc_len =
1316 sizeof(struct fcnvme_lsdesc_conn_id));
1317 acc->connectid.connection_id =
1318 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1319 be16_to_cpu(rqst->connect_cmd.qid)));
1323 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1324 struct nvmet_fc_ls_iod *iod)
1326 struct fcnvme_ls_disconnect_rqst *rqst =
1327 (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
1328 struct fcnvme_ls_disconnect_acc *acc =
1329 (struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
1330 struct nvmet_fc_tgt_queue *queue = NULL;
1331 struct nvmet_fc_tgt_assoc *assoc;
1333 bool del_assoc = false;
1335 memset(acc, 0, sizeof(*acc));
1337 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
1338 ret = VERR_DISCONN_LEN;
1339 else if (rqst->desc_list_len !=
1341 sizeof(struct fcnvme_ls_disconnect_rqst)))
1342 ret = VERR_DISCONN_RQST_LEN;
1343 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1344 ret = VERR_ASSOC_ID;
1345 else if (rqst->associd.desc_len !=
1347 sizeof(struct fcnvme_lsdesc_assoc_id)))
1348 ret = VERR_ASSOC_ID_LEN;
1349 else if (rqst->discon_cmd.desc_tag !=
1350 cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
1351 ret = VERR_DISCONN_CMD;
1352 else if (rqst->discon_cmd.desc_len !=
1354 sizeof(struct fcnvme_lsdesc_disconn_cmd)))
1355 ret = VERR_DISCONN_CMD_LEN;
1356 else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
1357 (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
1358 ret = VERR_DISCONN_SCOPE;
1360 /* match an active association */
1361 assoc = nvmet_fc_find_target_assoc(tgtport,
1362 be64_to_cpu(rqst->associd.association_id));
1365 if (rqst->discon_cmd.scope ==
1366 FCNVME_DISCONN_CONNECTION) {
1367 queue = nvmet_fc_find_target_queue(tgtport,
1369 rqst->discon_cmd.id));
1371 nvmet_fc_tgt_a_put(assoc);
1376 ret = VERR_NO_ASSOC;
1380 dev_err(tgtport->dev,
1381 "Disconnect LS failed: %s\n",
1382 validation_errors[ret]);
1383 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1384 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1385 (ret == VERR_NO_ASSOC) ?
1386 FCNVME_RJT_RC_INV_ASSOC :
1387 (ret == VERR_NO_CONN) ?
1388 FCNVME_RJT_RC_INV_CONN :
1389 FCNVME_RJT_RC_LOGIC,
1390 FCNVME_RJT_EXP_NONE, 0);
1394 /* format a response */
1396 iod->lsreq->rsplen = sizeof(*acc);
1398 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1400 sizeof(struct fcnvme_ls_disconnect_acc)),
1401 FCNVME_LS_DISCONNECT);
1404 /* are we to delete a Connection ID (queue) */
1406 int qid = queue->qid;
1408 nvmet_fc_delete_target_queue(queue);
1410 /* release the get taken by find_target_queue */
1411 nvmet_fc_tgt_q_put(queue);
1413 /* tear association down if io queue terminated */
1418 /* release get taken in nvmet_fc_find_target_assoc */
1419 nvmet_fc_tgt_a_put(iod->assoc);
1422 nvmet_fc_delete_target_assoc(iod->assoc);
1426 /* *********************** NVME Ctrl Routines **************************** */
1429 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1431 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1434 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
1436 struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
1437 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1439 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1440 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1441 nvmet_fc_free_ls_iod(tgtport, iod);
1442 nvmet_fc_tgtport_put(tgtport);
1446 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1447 struct nvmet_fc_ls_iod *iod)
1451 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1452 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1454 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
1456 nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
1460 * Actual processing routine for received FC-NVME LS Requests from the LLD
1463 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1464 struct nvmet_fc_ls_iod *iod)
1466 struct fcnvme_ls_rqst_w0 *w0 =
1467 (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
1469 iod->lsreq->nvmet_fc_private = iod;
1470 iod->lsreq->rspbuf = iod->rspbuf;
1471 iod->lsreq->rspdma = iod->rspdma;
1472 iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
1473 /* Be preventative. handlers will later set to valid length */
1474 iod->lsreq->rsplen = 0;
1480 * parse request input, execute the request, and format the
1483 switch (w0->ls_cmd) {
1484 case FCNVME_LS_CREATE_ASSOCIATION:
1485 /* Creates Association and initial Admin Queue/Connection */
1486 nvmet_fc_ls_create_association(tgtport, iod);
1488 case FCNVME_LS_CREATE_CONNECTION:
1489 /* Creates an IO Queue/Connection */
1490 nvmet_fc_ls_create_connection(tgtport, iod);
1492 case FCNVME_LS_DISCONNECT:
1493 /* Terminate a Queue/Connection or the Association */
1494 nvmet_fc_ls_disconnect(tgtport, iod);
1497 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
1498 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
1499 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1502 nvmet_fc_xmt_ls_rsp(tgtport, iod);
1506 * Actual processing routine for received FC-NVME LS Requests from the LLD
1509 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1511 struct nvmet_fc_ls_iod *iod =
1512 container_of(work, struct nvmet_fc_ls_iod, work);
1513 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1515 nvmet_fc_handle_ls_rqst(tgtport, iod);
1520 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
1521 * upon the reception of a NVME LS request.
1523 * The nvmet-fc layer will copy payload to an internal structure for
1524 * processing. As such, upon completion of the routine, the LLDD may
1525 * immediately free/reuse the LS request buffer passed in the call.
1527 * If this routine returns error, the LLDD should abort the exchange.
1529 * @tgtport: pointer to the (registered) target port the LS was
1531 * @lsreq: pointer to a lsreq request structure to be used to reference
1532 * the exchange corresponding to the LS.
1533 * @lsreqbuf: pointer to the buffer containing the LS Request
1534 * @lsreqbuf_len: length, in bytes, of the received LS request
1537 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
1538 struct nvmefc_tgt_ls_req *lsreq,
1539 void *lsreqbuf, u32 lsreqbuf_len)
1541 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1542 struct nvmet_fc_ls_iod *iod;
1544 if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
1547 if (!nvmet_fc_tgtport_get(tgtport))
1550 iod = nvmet_fc_alloc_ls_iod(tgtport);
1552 nvmet_fc_tgtport_put(tgtport);
1558 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
1559 iod->rqstdatalen = lsreqbuf_len;
1561 schedule_work(&iod->work);
1565 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
1569 * **********************
1570 * Start of FCP handling
1571 * **********************
1575 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1577 struct scatterlist *sg;
1580 u32 page_len, length;
1583 length = fod->total_length;
1584 nent = DIV_ROUND_UP(length, PAGE_SIZE);
1585 sg = kmalloc_array(nent, sizeof(struct scatterlist), GFP_KERNEL);
1589 sg_init_table(sg, nent);
1592 page_len = min_t(u32, length, PAGE_SIZE);
1594 page = alloc_page(GFP_KERNEL);
1596 goto out_free_pages;
1598 sg_set_page(&sg[i], page, page_len, 0);
1604 fod->data_sg_cnt = nent;
1605 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
1606 ((fod->io_dir == NVMET_FCP_WRITE) ?
1607 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1608 /* note: write from initiator perspective */
1615 __free_page(sg_page(&sg[i]));
1618 fod->data_sg = NULL;
1619 fod->data_sg_cnt = 0;
1621 return NVME_SC_INTERNAL;
1625 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1627 struct scatterlist *sg;
1630 if (!fod->data_sg || !fod->data_sg_cnt)
1633 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
1634 ((fod->io_dir == NVMET_FCP_WRITE) ?
1635 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1636 for_each_sg(fod->data_sg, sg, fod->data_sg_cnt, count)
1637 __free_page(sg_page(sg));
1638 kfree(fod->data_sg);
1639 fod->data_sg = NULL;
1640 fod->data_sg_cnt = 0;
1645 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
1649 /* egad, this is ugly. And sqtail is just a best guess */
1650 sqtail = atomic_read(&q->sqtail) % q->sqsize;
1652 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
1653 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
1658 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
1661 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1662 struct nvmet_fc_fcp_iod *fod)
1664 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
1665 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1666 struct nvme_completion *cqe = &ersp->cqe;
1667 u32 *cqewd = (u32 *)cqe;
1668 bool send_ersp = false;
1669 u32 rsn, rspcnt, xfr_length;
1671 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
1672 xfr_length = fod->total_length;
1674 xfr_length = fod->offset;
1677 * check to see if we can send a 0's rsp.
1678 * Note: to send a 0's response, the NVME-FC host transport will
1679 * recreate the CQE. The host transport knows: sq id, SQHD (last
1680 * seen in an ersp), and command_id. Thus it will create a
1681 * zero-filled CQE with those known fields filled in. Transport
1682 * must send an ersp for any condition where the cqe won't match
1685 * Here are the FC-NVME mandated cases where we must send an ersp:
1686 * every N responses, where N=ersp_ratio
1687 * force fabric commands to send ersp's (not in FC-NVME but good
1689 * normal cmds: any time status is non-zero, or status is zero
1690 * but words 0 or 1 are non-zero.
1691 * the SQ is 90% or more full
1692 * the cmd is a fused command
1693 * transferred data length not equal to cmd iu length
1695 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
1696 if (!(rspcnt % fod->queue->ersp_ratio) ||
1697 sqe->opcode == nvme_fabrics_command ||
1698 xfr_length != fod->total_length ||
1699 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
1700 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
1701 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
1704 /* re-set the fields */
1705 fod->fcpreq->rspaddr = ersp;
1706 fod->fcpreq->rspdma = fod->rspdma;
1709 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
1710 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
1712 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
1713 rsn = atomic_inc_return(&fod->queue->rsn);
1714 ersp->rsn = cpu_to_be32(rsn);
1715 ersp->xfrd_len = cpu_to_be32(xfr_length);
1716 fod->fcpreq->rsplen = sizeof(*ersp);
1719 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
1720 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
1723 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
1726 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
1727 struct nvmet_fc_fcp_iod *fod)
1729 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1731 /* data no longer needed */
1732 nvmet_fc_free_tgt_pgs(fod);
1735 * if an ABTS was received or we issued the fcp_abort early
1736 * don't call abort routine again.
1738 /* no need to take lock - lock was taken earlier to get here */
1740 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
1742 nvmet_fc_free_fcp_iod(fod->queue, fod);
1746 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1747 struct nvmet_fc_fcp_iod *fod)
1751 fod->fcpreq->op = NVMET_FCOP_RSP;
1752 fod->fcpreq->timeout = 0;
1754 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1756 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1758 nvmet_fc_abort_op(tgtport, fod);
1762 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
1763 struct nvmet_fc_fcp_iod *fod, u8 op)
1765 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1766 struct scatterlist *sg, *datasg;
1767 unsigned long flags;
1772 fcpreq->offset = fod->offset;
1773 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
1774 tlen = min_t(u32, (NVMET_FC_MAX_KB_PER_XFR * 1024),
1775 (fod->total_length - fod->offset));
1776 tlen = min_t(u32, tlen, NVME_FC_MAX_SEGMENTS * PAGE_SIZE);
1777 tlen = min_t(u32, tlen, fod->tgtport->ops->max_sgl_segments
1779 fcpreq->transfer_length = tlen;
1780 fcpreq->transferred_length = 0;
1781 fcpreq->fcp_error = 0;
1786 datasg = fod->next_sg;
1787 sg_off = fod->next_sg_offset;
1789 for (sg = fcpreq->sg ; tlen; sg++) {
1792 sg->offset += sg_off;
1793 sg->length -= sg_off;
1794 sg->dma_address += sg_off;
1797 if (tlen < sg->length) {
1799 fod->next_sg = datasg;
1800 fod->next_sg_offset += tlen;
1801 } else if (tlen == sg->length) {
1802 fod->next_sg_offset = 0;
1803 fod->next_sg = sg_next(datasg);
1805 fod->next_sg_offset = 0;
1806 datasg = sg_next(datasg);
1813 * If the last READDATA request: check if LLDD supports
1814 * combined xfr with response.
1816 if ((op == NVMET_FCOP_READDATA) &&
1817 ((fod->offset + fcpreq->transfer_length) == fod->total_length) &&
1818 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
1819 fcpreq->op = NVMET_FCOP_READDATA_RSP;
1820 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1823 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1826 * should be ok to set w/o lock as its in the thread of
1827 * execution (not an async timer routine) and doesn't
1828 * contend with any clearing action
1832 if (op == NVMET_FCOP_WRITEDATA) {
1833 spin_lock_irqsave(&fod->flock, flags);
1834 fod->writedataactive = false;
1835 spin_unlock_irqrestore(&fod->flock, flags);
1836 nvmet_req_complete(&fod->req,
1837 NVME_SC_FC_TRANSPORT_ERROR);
1838 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
1839 fcpreq->fcp_error = ret;
1840 fcpreq->transferred_length = 0;
1841 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
1847 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
1849 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1850 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1852 /* if in the middle of an io and we need to tear down */
1854 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
1855 nvmet_req_complete(&fod->req,
1856 NVME_SC_FC_TRANSPORT_ERROR);
1860 nvmet_fc_abort_op(tgtport, fod);
1868 * actual done handler for FCP operations when completed by the lldd
1871 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
1873 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1874 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
1875 unsigned long flags;
1878 spin_lock_irqsave(&fod->flock, flags);
1880 fod->writedataactive = false;
1881 spin_unlock_irqrestore(&fod->flock, flags);
1883 switch (fcpreq->op) {
1885 case NVMET_FCOP_WRITEDATA:
1886 if (__nvmet_fc_fod_op_abort(fod, abort))
1888 if (fcpreq->fcp_error ||
1889 fcpreq->transferred_length != fcpreq->transfer_length) {
1890 spin_lock(&fod->flock);
1892 spin_unlock(&fod->flock);
1894 nvmet_req_complete(&fod->req,
1895 NVME_SC_FC_TRANSPORT_ERROR);
1899 fod->offset += fcpreq->transferred_length;
1900 if (fod->offset != fod->total_length) {
1901 spin_lock_irqsave(&fod->flock, flags);
1902 fod->writedataactive = true;
1903 spin_unlock_irqrestore(&fod->flock, flags);
1905 /* transfer the next chunk */
1906 nvmet_fc_transfer_fcp_data(tgtport, fod,
1907 NVMET_FCOP_WRITEDATA);
1911 /* data transfer complete, resume with nvmet layer */
1913 fod->req.execute(&fod->req);
1917 case NVMET_FCOP_READDATA:
1918 case NVMET_FCOP_READDATA_RSP:
1919 if (__nvmet_fc_fod_op_abort(fod, abort))
1921 if (fcpreq->fcp_error ||
1922 fcpreq->transferred_length != fcpreq->transfer_length) {
1923 nvmet_fc_abort_op(tgtport, fod);
1929 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
1930 /* data no longer needed */
1931 nvmet_fc_free_tgt_pgs(fod);
1932 nvmet_fc_free_fcp_iod(fod->queue, fod);
1936 fod->offset += fcpreq->transferred_length;
1937 if (fod->offset != fod->total_length) {
1938 /* transfer the next chunk */
1939 nvmet_fc_transfer_fcp_data(tgtport, fod,
1940 NVMET_FCOP_READDATA);
1944 /* data transfer complete, send response */
1946 /* data no longer needed */
1947 nvmet_fc_free_tgt_pgs(fod);
1949 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
1953 case NVMET_FCOP_RSP:
1954 if (__nvmet_fc_fod_op_abort(fod, abort))
1956 nvmet_fc_free_fcp_iod(fod->queue, fod);
1965 nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work)
1967 struct nvmet_fc_fcp_iod *fod =
1968 container_of(work, struct nvmet_fc_fcp_iod, done_work);
1970 nvmet_fc_fod_op_done(fod);
1974 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
1976 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
1977 struct nvmet_fc_tgt_queue *queue = fod->queue;
1979 if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR)
1980 /* context switch so completion is not in ISR context */
1981 queue_work_on(queue->cpu, queue->work_q, &fod->done_work);
1983 nvmet_fc_fod_op_done(fod);
1987 * actual completion handler after execution by the nvmet layer
1990 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
1991 struct nvmet_fc_fcp_iod *fod, int status)
1993 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1994 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
1995 unsigned long flags;
1998 spin_lock_irqsave(&fod->flock, flags);
2000 spin_unlock_irqrestore(&fod->flock, flags);
2002 /* if we have a CQE, snoop the last sq_head value */
2004 fod->queue->sqhd = cqe->sq_head;
2007 nvmet_fc_abort_op(tgtport, fod);
2011 /* if an error handling the cmd post initial parsing */
2013 /* fudge up a failed CQE status for our transport error */
2014 memset(cqe, 0, sizeof(*cqe));
2015 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */
2016 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2017 cqe->command_id = sqe->command_id;
2018 cqe->status = cpu_to_le16(status);
2022 * try to push the data even if the SQE status is non-zero.
2023 * There may be a status where data still was intended to
2026 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2027 /* push the data over before sending rsp */
2028 nvmet_fc_transfer_fcp_data(tgtport, fod,
2029 NVMET_FCOP_READDATA);
2033 /* writes & no data - fall thru */
2036 /* data no longer needed */
2037 nvmet_fc_free_tgt_pgs(fod);
2039 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2044 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2046 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2047 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2049 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2054 * Actual processing routine for received FC-NVME LS Requests from the LLD
2057 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2058 struct nvmet_fc_fcp_iod *fod)
2060 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2064 * Fused commands are currently not supported in the linux
2067 * As such, the implementation of the FC transport does not
2068 * look at the fused commands and order delivery to the upper
2069 * layer until we have both based on csn.
2072 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2074 fod->total_length = be32_to_cpu(cmdiu->data_len);
2075 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2076 fod->io_dir = NVMET_FCP_WRITE;
2077 if (!nvme_is_write(&cmdiu->sqe))
2078 goto transport_error;
2079 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2080 fod->io_dir = NVMET_FCP_READ;
2081 if (nvme_is_write(&cmdiu->sqe))
2082 goto transport_error;
2084 fod->io_dir = NVMET_FCP_NODATA;
2085 if (fod->total_length)
2086 goto transport_error;
2089 fod->req.cmd = &fod->cmdiubuf.sqe;
2090 fod->req.rsp = &fod->rspiubuf.cqe;
2091 fod->req.port = fod->queue->port;
2093 /* ensure nvmet handlers will set cmd handler callback */
2094 fod->req.execute = NULL;
2096 /* clear any response payload */
2097 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2099 ret = nvmet_req_init(&fod->req,
2100 &fod->queue->nvme_cq,
2101 &fod->queue->nvme_sq,
2102 &nvmet_fc_tgt_fcp_ops);
2103 if (!ret) { /* bad SQE content or invalid ctrl state */
2104 nvmet_fc_abort_op(tgtport, fod);
2108 /* keep a running counter of tail position */
2109 atomic_inc(&fod->queue->sqtail);
2111 fod->data_sg = NULL;
2112 fod->data_sg_cnt = 0;
2113 if (fod->total_length) {
2114 ret = nvmet_fc_alloc_tgt_pgs(fod);
2116 nvmet_req_complete(&fod->req, ret);
2120 fod->req.sg = fod->data_sg;
2121 fod->req.sg_cnt = fod->data_sg_cnt;
2123 fod->next_sg = fod->data_sg;
2124 fod->next_sg_offset = 0;
2126 if (fod->io_dir == NVMET_FCP_WRITE) {
2127 /* pull the data over before invoking nvmet layer */
2128 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2135 * can invoke the nvmet_layer now. If read data, cmd completion will
2139 fod->req.execute(&fod->req);
2144 nvmet_fc_abort_op(tgtport, fod);
2148 * Actual processing routine for received FC-NVME LS Requests from the LLD
2151 nvmet_fc_handle_fcp_rqst_work(struct work_struct *work)
2153 struct nvmet_fc_fcp_iod *fod =
2154 container_of(work, struct nvmet_fc_fcp_iod, work);
2155 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2157 nvmet_fc_handle_fcp_rqst(tgtport, fod);
2161 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2162 * upon the reception of a NVME FCP CMD IU.
2164 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2165 * layer for processing.
2167 * The nvmet-fc layer will copy cmd payload to an internal structure for
2168 * processing. As such, upon completion of the routine, the LLDD may
2169 * immediately free/reuse the CMD IU buffer passed in the call.
2171 * If this routine returns error, the lldd should abort the exchange.
2173 * @target_port: pointer to the (registered) target port the FCP CMD IU
2175 * @fcpreq: pointer to a fcpreq request structure to be used to reference
2176 * the exchange corresponding to the FCP Exchange.
2177 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU
2178 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2181 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2182 struct nvmefc_tgt_fcp_req *fcpreq,
2183 void *cmdiubuf, u32 cmdiubuf_len)
2185 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2186 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2187 struct nvmet_fc_tgt_queue *queue;
2188 struct nvmet_fc_fcp_iod *fod;
2190 /* validate iu, so the connection id can be used to find the queue */
2191 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2192 (cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
2193 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2194 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2197 queue = nvmet_fc_find_target_queue(tgtport,
2198 be64_to_cpu(cmdiu->connection_id));
2203 * note: reference taken by find_target_queue
2204 * After successful fod allocation, the fod will inherit the
2205 * ownership of that reference and will remove the reference
2206 * when the fod is freed.
2209 fod = nvmet_fc_alloc_fcp_iod(queue);
2211 /* release the queue lookup reference */
2212 nvmet_fc_tgt_q_put(queue);
2216 fcpreq->nvmet_fc_private = fod;
2217 fod->fcpreq = fcpreq;
2219 * put all admin cmds on hw queue id 0. All io commands go to
2220 * the respective hw queue based on a modulo basis
2222 fcpreq->hwqid = queue->qid ?
2223 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
2224 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2226 if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR)
2227 queue_work_on(queue->cpu, queue->work_q, &fod->work);
2229 nvmet_fc_handle_fcp_rqst(tgtport, fod);
2233 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2236 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2237 * upon the reception of an ABTS for a FCP command
2239 * Notify the transport that an ABTS has been received for a FCP command
2240 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2241 * LLDD believes the command is still being worked on
2242 * (template_ops->fcp_req_release() has not been called).
2244 * The transport will wait for any outstanding work (an op to the LLDD,
2245 * which the lldd should complete with error due to the ABTS; or the
2246 * completion from the nvmet layer of the nvme command), then will
2247 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2248 * return the i/o context to the LLDD. The LLDD may send the BA_ACC
2249 * to the ABTS either after return from this function (assuming any
2250 * outstanding op work has been terminated) or upon the callback being
2253 * @target_port: pointer to the (registered) target port the FCP CMD IU
2255 * @fcpreq: pointer to the fcpreq request structure that corresponds
2256 * to the exchange that received the ABTS.
2259 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2260 struct nvmefc_tgt_fcp_req *fcpreq)
2262 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2263 struct nvmet_fc_tgt_queue *queue;
2264 unsigned long flags;
2266 if (!fod || fod->fcpreq != fcpreq)
2267 /* job appears to have already completed, ignore abort */
2272 spin_lock_irqsave(&queue->qlock, flags);
2275 * mark as abort. The abort handler, invoked upon completion
2276 * of any work, will detect the aborted status and do the
2279 spin_lock(&fod->flock);
2281 fod->aborted = true;
2282 spin_unlock(&fod->flock);
2284 spin_unlock_irqrestore(&queue->qlock, flags);
2286 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2290 FCT_TRADDR_WWNN = 1 << 0,
2291 FCT_TRADDR_WWPN = 1 << 1,
2294 struct nvmet_fc_traddr {
2299 static const match_table_t traddr_opt_tokens = {
2300 { FCT_TRADDR_WWNN, "nn-%s" },
2301 { FCT_TRADDR_WWPN, "pn-%s" },
2302 { FCT_TRADDR_ERR, NULL }
2306 nvmet_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf)
2308 substring_t args[MAX_OPT_ARGS];
2309 char *options, *o, *p;
2313 options = o = kstrdup(buf, GFP_KERNEL);
2317 while ((p = strsep(&o, ":\n")) != NULL) {
2321 token = match_token(p, traddr_opt_tokens, args);
2323 case FCT_TRADDR_WWNN:
2324 if (match_u64(args, &token64)) {
2328 traddr->nn = token64;
2330 case FCT_TRADDR_WWPN:
2331 if (match_u64(args, &token64)) {
2335 traddr->pn = token64;
2338 pr_warn("unknown traddr token or missing value '%s'\n",
2351 nvmet_fc_add_port(struct nvmet_port *port)
2353 struct nvmet_fc_tgtport *tgtport;
2354 struct nvmet_fc_traddr traddr = { 0L, 0L };
2355 unsigned long flags;
2358 /* validate the address info */
2359 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2360 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2363 /* map the traddr address info to a target port */
2365 ret = nvmet_fc_parse_traddr(&traddr, port->disc_addr.traddr);
2370 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2371 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2372 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2373 (tgtport->fc_target_port.port_name == traddr.pn)) {
2374 /* a FC port can only be 1 nvmet port id */
2375 if (!tgtport->port) {
2376 tgtport->port = port;
2377 port->priv = tgtport;
2378 nvmet_fc_tgtport_get(tgtport);
2385 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2390 nvmet_fc_remove_port(struct nvmet_port *port)
2392 struct nvmet_fc_tgtport *tgtport = port->priv;
2393 unsigned long flags;
2395 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2396 if (tgtport->port == port) {
2397 nvmet_fc_tgtport_put(tgtport);
2398 tgtport->port = NULL;
2400 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2403 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2404 .owner = THIS_MODULE,
2405 .type = NVMF_TRTYPE_FC,
2407 .add_port = nvmet_fc_add_port,
2408 .remove_port = nvmet_fc_remove_port,
2409 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2410 .delete_ctrl = nvmet_fc_delete_ctrl,
2413 static int __init nvmet_fc_init_module(void)
2415 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2418 static void __exit nvmet_fc_exit_module(void)
2420 /* sanity check - all lports should be removed */
2421 if (!list_empty(&nvmet_fc_target_list))
2422 pr_warn("%s: targetport list not empty\n", __func__);
2424 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2426 ida_destroy(&nvmet_fc_tgtport_cnt);
2429 module_init(nvmet_fc_init_module);
2430 module_exit(nvmet_fc_exit_module);
2432 MODULE_LICENSE("GPL v2");