2 * Broadcom NetXtreme-E RoCE driver.
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Description: Fast Path Operators
39 #include <linux/interrupt.h>
40 #include <linux/spinlock.h>
41 #include <linux/sched.h>
42 #include <linux/slab.h>
43 #include <linux/pci.h>
44 #include <linux/prefetch.h>
48 #include "qplib_res.h"
49 #include "qplib_rcfw.h"
53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
55 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
56 struct bnxt_qplib_qp *qp)
58 struct bnxt_qplib_q *rq = &qp->rq;
59 struct bnxt_qplib_q *sq = &qp->sq;
62 dma_free_coherent(&res->pdev->dev,
63 rq->hwq.max_elements * qp->rq_hdr_buf_size,
64 qp->rq_hdr_buf, qp->rq_hdr_buf_map);
66 dma_free_coherent(&res->pdev->dev,
67 sq->hwq.max_elements * qp->sq_hdr_buf_size,
68 qp->sq_hdr_buf, qp->sq_hdr_buf_map);
69 qp->rq_hdr_buf = NULL;
70 qp->sq_hdr_buf = NULL;
71 qp->rq_hdr_buf_map = 0;
72 qp->sq_hdr_buf_map = 0;
73 qp->sq_hdr_buf_size = 0;
74 qp->rq_hdr_buf_size = 0;
77 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
78 struct bnxt_qplib_qp *qp)
80 struct bnxt_qplib_q *rq = &qp->rq;
81 struct bnxt_qplib_q *sq = &qp->rq;
84 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
85 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
86 sq->hwq.max_elements *
88 &qp->sq_hdr_buf_map, GFP_KERNEL);
89 if (!qp->sq_hdr_buf) {
91 dev_err(&res->pdev->dev,
92 "QPLIB: Failed to create sq_hdr_buf");
97 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
98 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
99 rq->hwq.max_elements *
103 if (!qp->rq_hdr_buf) {
105 dev_err(&res->pdev->dev,
106 "QPLIB: Failed to create rq_hdr_buf");
113 bnxt_qplib_free_qp_hdr_buf(res, qp);
117 static void bnxt_qplib_service_nq(unsigned long data)
119 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
120 struct bnxt_qplib_hwq *hwq = &nq->hwq;
121 struct nq_base *nqe, **nq_ptr;
122 int num_cqne_processed = 0;
123 u32 sw_cons, raw_cons;
125 int budget = nq->budget;
128 /* Service the NQ until empty */
129 raw_cons = hwq->cons;
131 sw_cons = HWQ_CMP(raw_cons, hwq);
132 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
133 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
134 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
137 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
139 case NQ_BASE_TYPE_CQ_NOTIFICATION:
141 struct nq_cn *nqcne = (struct nq_cn *)nqe;
143 q_handle = le32_to_cpu(nqcne->cq_handle_low);
144 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
146 bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
147 ((unsigned long)q_handle));
148 if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
149 ((unsigned long)q_handle)))
150 num_cqne_processed++;
152 dev_warn(&nq->pdev->dev,
153 "QPLIB: cqn - type 0x%x not handled",
157 case NQ_BASE_TYPE_DBQ_EVENT:
160 dev_warn(&nq->pdev->dev,
161 "QPLIB: nqe with type = 0x%x not handled",
167 if (hwq->cons != raw_cons) {
168 hwq->cons = raw_cons;
169 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
173 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
175 struct bnxt_qplib_nq *nq = dev_instance;
176 struct bnxt_qplib_hwq *hwq = &nq->hwq;
177 struct nq_base **nq_ptr;
180 /* Prefetch the NQ element */
181 sw_cons = HWQ_CMP(hwq->cons, hwq);
182 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
183 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
185 /* Fan out to CPU affinitized kthreads? */
186 tasklet_schedule(&nq->worker);
191 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
193 /* Make sure the HW is stopped! */
194 synchronize_irq(nq->vector);
195 tasklet_disable(&nq->worker);
196 tasklet_kill(&nq->worker);
199 free_irq(nq->vector, nq);
200 nq->requested = false;
202 if (nq->bar_reg_iomem)
203 iounmap(nq->bar_reg_iomem);
204 nq->bar_reg_iomem = NULL;
206 nq->cqn_handler = NULL;
207 nq->srqn_handler = NULL;
211 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
212 int msix_vector, int bar_reg_offset,
213 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
214 struct bnxt_qplib_cq *),
215 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
218 resource_size_t nq_base;
222 nq->vector = msix_vector;
224 nq->cqn_handler = cqn_handler;
226 nq->srqn_handler = srqn_handler;
228 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
230 nq->requested = false;
231 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
233 dev_err(&nq->pdev->dev,
234 "Failed to request IRQ for NQ: %#x", rc);
235 bnxt_qplib_disable_nq(nq);
238 nq->requested = true;
239 nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
240 nq->bar_reg_off = bar_reg_offset;
241 nq_base = pci_resource_start(pdev, nq->bar_reg);
246 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
247 if (!nq->bar_reg_iomem) {
251 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
255 bnxt_qplib_disable_nq(nq);
259 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
261 if (nq->hwq.max_elements)
262 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
265 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
268 if (!nq->hwq.max_elements ||
269 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
270 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
272 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
273 &nq->hwq.max_elements,
274 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
275 PAGE_SIZE, HWQ_TYPE_L2_CMPL))
283 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
285 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
286 struct cmdq_create_qp1 req;
287 struct creq_create_qp1_resp *resp;
288 struct bnxt_qplib_pbl *pbl;
289 struct bnxt_qplib_q *sq = &qp->sq;
290 struct bnxt_qplib_q *rq = &qp->rq;
295 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
299 req.dpi = cpu_to_le32(qp->dpi->dpi);
300 req.qp_handle = cpu_to_le64(qp->qp_handle);
303 sq->hwq.max_elements = sq->max_wqe;
304 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
305 &sq->hwq.max_elements,
306 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
307 PAGE_SIZE, HWQ_TYPE_QUEUE);
311 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
316 pbl = &sq->hwq.pbl[PBL_LVL_0];
317 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
318 req.sq_pg_size_sq_lvl =
319 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
320 << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
321 (pbl->pg_size == ROCE_PG_SIZE_4K ?
322 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
323 pbl->pg_size == ROCE_PG_SIZE_8K ?
324 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
325 pbl->pg_size == ROCE_PG_SIZE_64K ?
326 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
327 pbl->pg_size == ROCE_PG_SIZE_2M ?
328 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
329 pbl->pg_size == ROCE_PG_SIZE_8M ?
330 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
331 pbl->pg_size == ROCE_PG_SIZE_1G ?
332 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
333 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
336 req.scq_cid = cpu_to_le32(qp->scq->id);
338 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
342 rq->hwq.max_elements = qp->rq.max_wqe;
343 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
344 &rq->hwq.max_elements,
345 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
346 PAGE_SIZE, HWQ_TYPE_QUEUE);
350 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
356 pbl = &rq->hwq.pbl[PBL_LVL_0];
357 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
358 req.rq_pg_size_rq_lvl =
359 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
360 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
361 (pbl->pg_size == ROCE_PG_SIZE_4K ?
362 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
363 pbl->pg_size == ROCE_PG_SIZE_8K ?
364 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
365 pbl->pg_size == ROCE_PG_SIZE_64K ?
366 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
367 pbl->pg_size == ROCE_PG_SIZE_2M ?
368 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
369 pbl->pg_size == ROCE_PG_SIZE_8M ?
370 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
371 pbl->pg_size == ROCE_PG_SIZE_1G ?
372 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
373 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
375 req.rcq_cid = cpu_to_le32(qp->rcq->id);
378 /* Header buffer - allow hdr_buf pass in */
379 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
384 req.qp_flags = cpu_to_le32(qp_flags);
385 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
386 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
389 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
390 CMDQ_CREATE_QP1_SQ_SGE_SFT);
392 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
393 CMDQ_CREATE_QP1_RQ_SGE_SFT);
395 req.pd_id = cpu_to_le32(qp->pd->id);
397 resp = (struct creq_create_qp1_resp *)
398 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
401 dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed");
405 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
407 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out");
412 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
413 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed ");
414 dev_err(&rcfw->pdev->dev,
415 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
416 resp->status, le16_to_cpu(req.cookie),
417 le16_to_cpu(resp->cookie));
421 qp->id = le32_to_cpu(resp->xid);
422 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
423 sq->flush_in_progress = false;
424 rq->flush_in_progress = false;
429 bnxt_qplib_free_qp_hdr_buf(res, qp);
431 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
434 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
440 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
442 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
443 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
444 struct cmdq_create_qp req;
445 struct creq_create_qp_resp *resp;
446 struct bnxt_qplib_pbl *pbl;
447 struct sq_psn_search **psn_search_ptr;
448 unsigned long int psn_search, poff = 0;
449 struct bnxt_qplib_q *sq = &qp->sq;
450 struct bnxt_qplib_q *rq = &qp->rq;
451 struct bnxt_qplib_hwq *xrrq;
452 int i, rc, req_size, psn_sz;
453 u16 cmd_flags = 0, max_ssge;
454 u32 sw_prod, qp_flags = 0;
456 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
460 req.dpi = cpu_to_le32(qp->dpi->dpi);
461 req.qp_handle = cpu_to_le64(qp->qp_handle);
464 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
465 sizeof(struct sq_psn_search) : 0;
466 sq->hwq.max_elements = sq->max_wqe;
467 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
468 sq->nmap, &sq->hwq.max_elements,
469 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
471 PAGE_SIZE, HWQ_TYPE_QUEUE);
475 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
480 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
482 psn_search_ptr = (struct sq_psn_search **)
483 &hw_sq_send_ptr[get_sqe_pg
484 (sq->hwq.max_elements)];
485 psn_search = (unsigned long int)
486 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
487 [get_sqe_idx(sq->hwq.max_elements)];
488 if (psn_search & ~PAGE_MASK) {
489 /* If the psn_search does not start on a page boundary,
490 * then calculate the offset
492 poff = (psn_search & ~PAGE_MASK) /
493 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
495 for (i = 0; i < sq->hwq.max_elements; i++)
496 sq->swq[i].psn_search =
497 &psn_search_ptr[get_psne_pg(i + poff)]
498 [get_psne_idx(i + poff)];
500 pbl = &sq->hwq.pbl[PBL_LVL_0];
501 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
502 req.sq_pg_size_sq_lvl =
503 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
504 << CMDQ_CREATE_QP_SQ_LVL_SFT) |
505 (pbl->pg_size == ROCE_PG_SIZE_4K ?
506 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
507 pbl->pg_size == ROCE_PG_SIZE_8K ?
508 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
509 pbl->pg_size == ROCE_PG_SIZE_64K ?
510 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
511 pbl->pg_size == ROCE_PG_SIZE_2M ?
512 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
513 pbl->pg_size == ROCE_PG_SIZE_8M ?
514 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
515 pbl->pg_size == ROCE_PG_SIZE_1G ?
516 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
517 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
519 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
520 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
521 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
522 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
523 [get_sqe_idx(sw_prod)];
524 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
528 req.scq_cid = cpu_to_le32(qp->scq->id);
530 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
531 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
533 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
537 rq->hwq.max_elements = rq->max_wqe;
538 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
539 rq->nmap, &rq->hwq.max_elements,
540 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
541 PAGE_SIZE, HWQ_TYPE_QUEUE);
545 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
551 pbl = &rq->hwq.pbl[PBL_LVL_0];
552 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
553 req.rq_pg_size_rq_lvl =
554 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
555 CMDQ_CREATE_QP_RQ_LVL_SFT) |
556 (pbl->pg_size == ROCE_PG_SIZE_4K ?
557 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
558 pbl->pg_size == ROCE_PG_SIZE_8K ?
559 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
560 pbl->pg_size == ROCE_PG_SIZE_64K ?
561 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
562 pbl->pg_size == ROCE_PG_SIZE_2M ?
563 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
564 pbl->pg_size == ROCE_PG_SIZE_8M ?
565 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
566 pbl->pg_size == ROCE_PG_SIZE_1G ?
567 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
568 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
572 req.rcq_cid = cpu_to_le32(qp->rcq->id);
573 req.qp_flags = cpu_to_le32(qp_flags);
574 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
575 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
576 qp->sq_hdr_buf = NULL;
577 qp->rq_hdr_buf = NULL;
579 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
583 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
584 * always create the QP with max send sges possible if the requested
585 * inline size is greater than 0.
587 max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
588 req.sq_fwo_sq_sge = cpu_to_le16(
589 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
590 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
591 req.rq_fwo_rq_sge = cpu_to_le16(
592 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
593 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
598 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
599 req_size = xrrq->max_elements *
600 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
601 req_size &= ~(PAGE_SIZE - 1);
602 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
604 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
605 0, req_size, HWQ_TYPE_CTX);
608 pbl = &xrrq->pbl[PBL_LVL_0];
609 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
612 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
613 qp->max_dest_rd_atomic);
614 req_size = xrrq->max_elements *
615 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
616 req_size &= ~(PAGE_SIZE - 1);
618 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
620 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
621 0, req_size, HWQ_TYPE_CTX);
625 pbl = &xrrq->pbl[PBL_LVL_0];
626 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
628 req.pd_id = cpu_to_le32(qp->pd->id);
630 resp = (struct creq_create_qp_resp *)
631 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
634 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed");
638 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
640 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out");
645 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
646 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed ");
647 dev_err(&rcfw->pdev->dev,
648 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
649 resp->status, le16_to_cpu(req.cookie),
650 le16_to_cpu(resp->cookie));
654 qp->id = le32_to_cpu(resp->xid);
655 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
656 sq->flush_in_progress = false;
657 rq->flush_in_progress = false;
662 if (qp->irrq.max_elements)
663 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
665 if (qp->orrq.max_elements)
666 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
668 bnxt_qplib_free_qp_hdr_buf(res, qp);
670 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
673 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
679 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
682 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
683 /* INIT->RTR, configure the path_mtu to the default
684 * 2048 if not being requested
686 if (!(qp->modify_flags &
687 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
689 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
691 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
694 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
695 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
696 if (qp->max_dest_rd_atomic < 1)
697 qp->max_dest_rd_atomic = 1;
698 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
699 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
700 if (!(qp->modify_flags &
701 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
703 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
704 qp->ah.sgid_index = 0;
712 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
715 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
716 /* Bono FW requires the max_rd_atomic to be >= 1 */
717 if (qp->max_rd_atomic < 1)
718 qp->max_rd_atomic = 1;
719 /* Bono FW does not allow PKEY_INDEX,
720 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
721 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
722 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
726 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
727 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
728 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
729 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
730 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
731 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
732 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
733 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
734 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
735 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
736 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
737 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
744 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
746 switch (qp->cur_qp_state) {
747 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
749 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
750 __modify_flags_from_init_state(qp);
752 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
753 __modify_flags_from_rtr_state(qp);
755 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
757 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
759 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
761 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
768 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
770 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
771 struct cmdq_modify_qp req;
772 struct creq_modify_qp_resp *resp;
773 u16 cmd_flags = 0, pkey;
777 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
779 /* Filter out the qp_attr_mask based on the state->new transition */
780 __filter_modify_flags(qp);
781 bmask = qp->modify_flags;
782 req.modify_mask = cpu_to_le32(qp->modify_flags);
783 req.qp_cid = cpu_to_le32(qp->id);
784 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
785 req.network_type_en_sqd_async_notify_new_state =
786 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
787 (qp->en_sqd_async_notify ?
788 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
790 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
792 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
793 req.access = qp->access;
795 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
796 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
797 qp->pkey_index, &pkey))
798 req.pkey = cpu_to_le16(pkey);
800 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
801 req.qkey = cpu_to_le32(qp->qkey);
803 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
804 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
805 req.dgid[0] = cpu_to_le32(temp32[0]);
806 req.dgid[1] = cpu_to_le32(temp32[1]);
807 req.dgid[2] = cpu_to_le32(temp32[2]);
808 req.dgid[3] = cpu_to_le32(temp32[3]);
810 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
811 req.flow_label = cpu_to_le32(qp->ah.flow_label);
813 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
814 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
815 [qp->ah.sgid_index]);
817 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
818 req.hop_limit = qp->ah.hop_limit;
820 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
821 req.traffic_class = qp->ah.traffic_class;
823 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
824 memcpy(req.dest_mac, qp->ah.dmac, 6);
826 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
827 req.path_mtu = qp->path_mtu;
829 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
830 req.timeout = qp->timeout;
832 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
833 req.retry_cnt = qp->retry_cnt;
835 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
836 req.rnr_retry = qp->rnr_retry;
838 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
839 req.min_rnr_timer = qp->min_rnr_timer;
841 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
842 req.rq_psn = cpu_to_le32(qp->rq.psn);
844 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
845 req.sq_psn = cpu_to_le32(qp->sq.psn);
847 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
849 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
851 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
852 req.max_dest_rd_atomic =
853 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
855 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
856 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
857 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
858 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
859 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
860 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
861 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
863 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
865 resp = (struct creq_modify_qp_resp *)
866 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
869 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed");
872 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
874 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out");
878 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
879 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed ");
880 dev_err(&rcfw->pdev->dev,
881 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
882 resp->status, le16_to_cpu(req.cookie),
883 le16_to_cpu(resp->cookie));
886 qp->cur_qp_state = qp->state;
890 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
892 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
893 struct cmdq_query_qp req;
894 struct creq_query_qp_resp *resp;
895 struct creq_query_qp_resp_sb *sb;
900 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
902 req.qp_cid = cpu_to_le32(qp->id);
903 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
904 resp = (struct creq_query_qp_resp *)
905 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
908 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed");
911 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
913 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out");
917 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
918 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed ");
919 dev_err(&rcfw->pdev->dev,
920 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
921 resp->status, le16_to_cpu(req.cookie),
922 le16_to_cpu(resp->cookie));
925 /* Extract the context from the side buffer */
926 qp->state = sb->en_sqd_async_notify_state &
927 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
928 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
929 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
931 qp->access = sb->access;
932 qp->pkey_index = le16_to_cpu(sb->pkey);
933 qp->qkey = le32_to_cpu(sb->qkey);
935 temp32[0] = le32_to_cpu(sb->dgid[0]);
936 temp32[1] = le32_to_cpu(sb->dgid[1]);
937 temp32[2] = le32_to_cpu(sb->dgid[2]);
938 temp32[3] = le32_to_cpu(sb->dgid[3]);
939 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
941 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
943 qp->ah.sgid_index = 0;
944 for (i = 0; i < res->sgid_tbl.max; i++) {
945 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
946 qp->ah.sgid_index = i;
950 if (i == res->sgid_tbl.max)
951 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
953 qp->ah.hop_limit = sb->hop_limit;
954 qp->ah.traffic_class = sb->traffic_class;
955 memcpy(qp->ah.dmac, sb->dest_mac, 6);
956 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
957 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
958 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
959 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
960 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
961 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
962 qp->timeout = sb->timeout;
963 qp->retry_cnt = sb->retry_cnt;
964 qp->rnr_retry = sb->rnr_retry;
965 qp->min_rnr_timer = sb->min_rnr_timer;
966 qp->rq.psn = le32_to_cpu(sb->rq_psn);
967 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
968 qp->sq.psn = le32_to_cpu(sb->sq_psn);
969 qp->max_dest_rd_atomic =
970 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
971 qp->sq.max_wqe = qp->sq.hwq.max_elements;
972 qp->rq.max_wqe = qp->rq.hwq.max_elements;
973 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
974 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
975 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
976 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
977 memcpy(qp->smac, sb->src_mac, 6);
978 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
982 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
984 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
985 struct cq_base *hw_cqe, **hw_cqe_ptr;
988 for (i = 0; i < cq_hwq->max_elements; i++) {
989 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
990 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
991 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
993 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
994 case CQ_BASE_CQE_TYPE_REQ:
995 case CQ_BASE_CQE_TYPE_TERMINAL:
997 struct cq_req *cqe = (struct cq_req *)hw_cqe;
999 if (qp == le64_to_cpu(cqe->qp_handle))
1003 case CQ_BASE_CQE_TYPE_RES_RC:
1004 case CQ_BASE_CQE_TYPE_RES_UD:
1005 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1007 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1009 if (qp == le64_to_cpu(cqe->qp_handle))
1019 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1020 struct bnxt_qplib_qp *qp)
1022 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1023 struct cmdq_destroy_qp req;
1024 struct creq_destroy_qp_resp *resp;
1025 unsigned long flags;
1028 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1030 req.qp_cid = cpu_to_le32(qp->id);
1031 resp = (struct creq_destroy_qp_resp *)
1032 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1035 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed");
1038 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1040 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out");
1044 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1045 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed ");
1046 dev_err(&rcfw->pdev->dev,
1047 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1048 resp->status, le16_to_cpu(req.cookie),
1049 le16_to_cpu(resp->cookie));
1053 /* Must walk the associated CQs to nullified the QP ptr */
1054 spin_lock_irqsave(&qp->scq->hwq.lock, flags);
1056 __clean_cq(qp->scq, (u64)(unsigned long)qp);
1058 if (qp->rcq && qp->rcq != qp->scq) {
1059 spin_lock(&qp->rcq->hwq.lock);
1060 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
1061 spin_unlock(&qp->rcq->hwq.lock);
1064 spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
1066 bnxt_qplib_free_qp_hdr_buf(res, qp);
1067 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1070 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1073 if (qp->irrq.max_elements)
1074 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1075 if (qp->orrq.max_elements)
1076 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1081 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1082 struct bnxt_qplib_sge *sge)
1084 struct bnxt_qplib_q *sq = &qp->sq;
1087 memset(sge, 0, sizeof(*sge));
1089 if (qp->sq_hdr_buf) {
1090 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1091 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1092 sw_prod * qp->sq_hdr_buf_size);
1093 sge->lkey = 0xFFFFFFFF;
1094 sge->size = qp->sq_hdr_buf_size;
1095 return qp->sq_hdr_buf + sw_prod * sge->size;
1100 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1102 struct bnxt_qplib_q *rq = &qp->rq;
1104 return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1107 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1109 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1112 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1113 struct bnxt_qplib_sge *sge)
1115 struct bnxt_qplib_q *rq = &qp->rq;
1118 memset(sge, 0, sizeof(*sge));
1120 if (qp->rq_hdr_buf) {
1121 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1122 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1123 sw_prod * qp->rq_hdr_buf_size);
1124 sge->lkey = 0xFFFFFFFF;
1125 sge->size = qp->rq_hdr_buf_size;
1126 return qp->rq_hdr_buf + sw_prod * sge->size;
1131 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1133 struct bnxt_qplib_q *sq = &qp->sq;
1134 struct dbr_dbr db_msg = { 0 };
1137 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1139 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1140 DBR_DBR_INDEX_MASK);
1142 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1144 /* Flush all the WQE writes to HW */
1146 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1149 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1150 struct bnxt_qplib_swqe *wqe)
1152 struct bnxt_qplib_q *sq = &qp->sq;
1153 struct bnxt_qplib_swq *swq;
1154 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1155 struct sq_sge *hw_sge;
1158 int i, rc = 0, data_len = 0, pkt_num = 0;
1161 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1165 if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) ==
1166 HWQ_CMP(sq->hwq.cons, &sq->hwq)) {
1170 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1171 swq = &sq->swq[sw_prod];
1172 swq->wr_id = wqe->wr_id;
1173 swq->type = wqe->type;
1174 swq->flags = wqe->flags;
1176 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1177 swq->start_psn = sq->psn & BTH_PSN_MASK;
1179 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1180 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1181 [get_sqe_idx(sw_prod)];
1183 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1185 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1186 /* Copy the inline data */
1187 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1188 dev_warn(&sq->hwq.pdev->dev,
1189 "QPLIB: Inline data length > 96 detected");
1190 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1192 data_len = wqe->inline_len;
1194 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1195 wqe_size16 = (data_len + 15) >> 4;
1197 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1198 i < wqe->num_sge; i++, hw_sge++) {
1199 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1200 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1201 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1202 data_len += wqe->sg_list[i].size;
1204 /* Each SGE entry = 1 WQE size16 */
1205 wqe_size16 = wqe->num_sge;
1209 switch (wqe->type) {
1210 case BNXT_QPLIB_SWQE_TYPE_SEND:
1211 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1212 /* Assemble info for Raw Ethertype QPs */
1213 struct sq_send_raweth_qp1 *sqe =
1214 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1216 sqe->wqe_type = wqe->type;
1217 sqe->flags = wqe->flags;
1218 sqe->wqe_size = wqe_size16 +
1219 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1220 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1221 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1222 sqe->length = cpu_to_le32(data_len);
1223 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1224 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1225 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1229 /* else, just fall thru */
1230 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1231 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1233 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1235 sqe->wqe_type = wqe->type;
1236 sqe->flags = wqe->flags;
1237 sqe->wqe_size = wqe_size16 +
1238 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1239 sqe->inv_key_or_imm_data = cpu_to_le32(
1241 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1242 sqe->q_key = cpu_to_le32(wqe->send.q_key);
1243 sqe->dst_qp = cpu_to_le32(
1244 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1245 sqe->length = cpu_to_le32(data_len);
1246 sqe->avid = cpu_to_le32(wqe->send.avid &
1248 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1250 sqe->length = cpu_to_le32(data_len);
1254 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1257 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1261 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1262 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1263 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1265 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1267 sqe->wqe_type = wqe->type;
1268 sqe->flags = wqe->flags;
1269 sqe->wqe_size = wqe_size16 +
1270 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1271 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1272 sqe->length = cpu_to_le32((u32)data_len);
1273 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1274 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1276 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1279 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1282 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1283 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1285 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1287 sqe->wqe_type = wqe->type;
1288 sqe->flags = wqe->flags;
1289 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1290 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1291 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1292 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1294 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1297 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1300 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1302 struct sq_localinvalidate *sqe =
1303 (struct sq_localinvalidate *)hw_sq_send_hdr;
1305 sqe->wqe_type = wqe->type;
1306 sqe->flags = wqe->flags;
1307 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1311 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1313 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1315 sqe->wqe_type = wqe->type;
1316 sqe->flags = wqe->flags;
1317 sqe->access_cntl = wqe->frmr.access_cntl |
1318 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1319 sqe->zero_based_page_size_log =
1320 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1321 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1322 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1323 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1324 temp32 = cpu_to_le32(wqe->frmr.length);
1325 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1326 sqe->numlevels_pbl_page_size_log =
1327 ((wqe->frmr.pbl_pg_sz_log <<
1328 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1329 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1330 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1331 SQ_FR_PMR_NUMLEVELS_MASK);
1333 for (i = 0; i < wqe->frmr.page_list_len; i++)
1334 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1335 wqe->frmr.page_list[i] |
1337 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1338 sqe->va = cpu_to_le64(wqe->frmr.va);
1342 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1344 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1346 sqe->wqe_type = wqe->type;
1347 sqe->flags = wqe->flags;
1348 sqe->access_cntl = wqe->bind.access_cntl;
1349 sqe->mw_type_zero_based = wqe->bind.mw_type |
1350 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1351 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1352 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1353 sqe->va = cpu_to_le64(wqe->bind.va);
1354 temp32 = cpu_to_le32(wqe->bind.length);
1355 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1359 /* Bad wqe, return error */
1363 swq->next_psn = sq->psn & BTH_PSN_MASK;
1364 if (swq->psn_search) {
1365 swq->psn_search->opcode_start_psn = cpu_to_le32(
1366 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1367 SQ_PSN_SEARCH_START_PSN_MASK) |
1368 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1369 SQ_PSN_SEARCH_OPCODE_MASK));
1370 swq->psn_search->flags_next_psn = cpu_to_le32(
1371 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1372 SQ_PSN_SEARCH_NEXT_PSN_MASK));
1380 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1382 struct bnxt_qplib_q *rq = &qp->rq;
1383 struct dbr_dbr db_msg = { 0 };
1386 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1387 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1388 DBR_DBR_INDEX_MASK);
1390 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1393 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1395 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1398 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1399 struct bnxt_qplib_swqe *wqe)
1401 struct bnxt_qplib_q *rq = &qp->rq;
1402 struct rq_wqe *rqe, **rqe_ptr;
1403 struct sq_sge *hw_sge;
1407 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1408 dev_err(&rq->hwq.pdev->dev,
1409 "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1414 if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) ==
1415 HWQ_CMP(rq->hwq.cons, &rq->hwq)) {
1416 dev_err(&rq->hwq.pdev->dev,
1417 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1421 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1422 rq->swq[sw_prod].wr_id = wqe->wr_id;
1424 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1425 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1427 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1429 /* Calculate wqe_size16 and data_len */
1430 for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1431 i < wqe->num_sge; i++, hw_sge++) {
1432 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1433 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1434 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1436 rqe->wqe_type = wqe->type;
1437 rqe->flags = wqe->flags;
1438 rqe->wqe_size = wqe->num_sge +
1439 ((offsetof(typeof(*rqe), data) + 15) >> 4);
1441 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1442 rqe->wr_id[0] = cpu_to_le32(sw_prod);
1451 /* Spinlock must be held */
1452 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1454 struct dbr_dbr db_msg = { 0 };
1457 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1458 DBR_DBR_TYPE_CQ_ARMENA);
1459 /* Flush memory writes before enabling the CQ */
1461 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1464 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1466 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1467 struct dbr_dbr db_msg = { 0 };
1471 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1472 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1473 DBR_DBR_INDEX_MASK);
1475 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1477 /* flush memory writes before arming the CQ */
1479 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1482 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1484 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1485 struct cmdq_create_cq req;
1486 struct creq_create_cq_resp *resp;
1487 struct bnxt_qplib_pbl *pbl;
1491 cq->hwq.max_elements = cq->max_wqe;
1492 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1493 cq->nmap, &cq->hwq.max_elements,
1494 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1495 PAGE_SIZE, HWQ_TYPE_QUEUE);
1499 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1502 dev_err(&rcfw->pdev->dev,
1503 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1506 req.dpi = cpu_to_le32(cq->dpi->dpi);
1507 req.cq_handle = cpu_to_le64(cq->cq_handle);
1509 req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1510 pbl = &cq->hwq.pbl[PBL_LVL_0];
1511 req.pg_size_lvl = cpu_to_le32(
1512 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1513 CMDQ_CREATE_CQ_LVL_SFT) |
1514 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1515 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1516 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1517 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1518 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1519 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1520 CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1522 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1524 req.cq_fco_cnq_id = cpu_to_le32(
1525 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1526 CMDQ_CREATE_CQ_CNQ_ID_SFT);
1528 resp = (struct creq_create_cq_resp *)
1529 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1532 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed");
1535 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1537 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out");
1542 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1543 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed ");
1544 dev_err(&rcfw->pdev->dev,
1545 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1546 resp->status, le16_to_cpu(req.cookie),
1547 le16_to_cpu(resp->cookie));
1551 cq->id = le32_to_cpu(resp->xid);
1552 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1553 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1554 init_waitqueue_head(&cq->waitq);
1556 bnxt_qplib_arm_cq_enable(cq);
1560 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1565 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1567 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1568 struct cmdq_destroy_cq req;
1569 struct creq_destroy_cq_resp *resp;
1572 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1574 req.cq_cid = cpu_to_le32(cq->id);
1575 resp = (struct creq_destroy_cq_resp *)
1576 bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1579 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed");
1582 if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1584 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out");
1588 le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1589 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed ");
1590 dev_err(&rcfw->pdev->dev,
1591 "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1592 resp->status, le16_to_cpu(req.cookie),
1593 le16_to_cpu(resp->cookie));
1596 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1600 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1601 struct bnxt_qplib_cqe **pcqe, int *budget)
1603 u32 sw_prod, sw_cons;
1604 struct bnxt_qplib_cqe *cqe;
1607 /* Now complete all outstanding SQEs with FLUSHED_ERR */
1608 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1611 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1612 if (sw_cons == sw_prod) {
1613 sq->flush_in_progress = false;
1616 memset(cqe, 0, sizeof(*cqe));
1617 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1618 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1619 cqe->qp_handle = (u64)(unsigned long)qp;
1620 cqe->wr_id = sq->swq[sw_cons].wr_id;
1621 cqe->src_qp = qp->id;
1622 cqe->type = sq->swq[sw_cons].type;
1628 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1635 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1636 int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
1638 struct bnxt_qplib_cqe *cqe;
1639 u32 sw_prod, sw_cons;
1642 /* Flush the rest of the RQ */
1643 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1646 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1647 if (sw_cons == sw_prod)
1649 memset(cqe, 0, sizeof(*cqe));
1651 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1652 cqe->opcode = opcode;
1653 cqe->qp_handle = (unsigned long)qp;
1654 cqe->wr_id = rq->swq[sw_cons].wr_id;
1660 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1667 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1668 struct cq_req *hwcqe,
1669 struct bnxt_qplib_cqe **pcqe, int *budget)
1671 struct bnxt_qplib_qp *qp;
1672 struct bnxt_qplib_q *sq;
1673 struct bnxt_qplib_cqe *cqe;
1674 u32 sw_cons, cqe_cons;
1677 qp = (struct bnxt_qplib_qp *)((unsigned long)
1678 le64_to_cpu(hwcqe->qp_handle));
1680 dev_err(&cq->hwq.pdev->dev,
1681 "QPLIB: FP: Process Req qp is NULL");
1686 cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1687 if (cqe_cons > sq->hwq.max_elements) {
1688 dev_err(&cq->hwq.pdev->dev,
1689 "QPLIB: FP: CQ Process req reported ");
1690 dev_err(&cq->hwq.pdev->dev,
1691 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1692 cqe_cons, sq->hwq.max_elements);
1695 /* If we were in the middle of flushing the SQ, continue */
1696 if (sq->flush_in_progress)
1699 /* Require to walk the sq's swq to fabricate CQEs for all previously
1700 * signaled SWQEs due to CQE aggregation from the current sq cons
1705 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1706 if (sw_cons == cqe_cons)
1708 memset(cqe, 0, sizeof(*cqe));
1709 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1710 cqe->qp_handle = (u64)(unsigned long)qp;
1711 cqe->src_qp = qp->id;
1712 cqe->wr_id = sq->swq[sw_cons].wr_id;
1713 cqe->type = sq->swq[sw_cons].type;
1715 /* For the last CQE, check for status. For errors, regardless
1716 * of the request being signaled or not, it must complete with
1717 * the hwcqe error status
1719 if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons &&
1720 hwcqe->status != CQ_REQ_STATUS_OK) {
1721 cqe->status = hwcqe->status;
1722 dev_err(&cq->hwq.pdev->dev,
1723 "QPLIB: FP: CQ Processed Req ");
1724 dev_err(&cq->hwq.pdev->dev,
1725 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1726 sw_cons, cqe->wr_id, cqe->status);
1729 sq->flush_in_progress = true;
1730 /* Must block new posting of SQ and RQ */
1731 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1733 if (sq->swq[sw_cons].flags &
1734 SQ_SEND_FLAGS_SIGNAL_COMP) {
1735 cqe->status = CQ_REQ_STATUS_OK;
1743 if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) {
1748 if (!sq->flush_in_progress)
1751 /* Require to walk the sq's swq to fabricate CQEs for all
1752 * previously posted SWQEs due to the error CQE received
1754 rc = __flush_sq(sq, qp, pcqe, budget);
1756 sq->flush_in_progress = false;
1761 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
1762 struct cq_res_rc *hwcqe,
1763 struct bnxt_qplib_cqe **pcqe,
1766 struct bnxt_qplib_qp *qp;
1767 struct bnxt_qplib_q *rq;
1768 struct bnxt_qplib_cqe *cqe;
1772 qp = (struct bnxt_qplib_qp *)((unsigned long)
1773 le64_to_cpu(hwcqe->qp_handle));
1775 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
1779 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1780 cqe->length = le32_to_cpu(hwcqe->length);
1781 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
1782 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
1783 cqe->flags = le16_to_cpu(hwcqe->flags);
1784 cqe->status = hwcqe->status;
1785 cqe->qp_handle = (u64)(unsigned long)qp;
1787 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
1788 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
1790 if (wr_id_idx > rq->hwq.max_elements) {
1791 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
1792 dev_err(&cq->hwq.pdev->dev,
1793 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1794 wr_id_idx, rq->hwq.max_elements);
1797 if (rq->flush_in_progress)
1800 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1806 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1807 rq->flush_in_progress = true;
1809 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
1811 rq->flush_in_progress = false;
1816 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
1817 struct cq_res_ud *hwcqe,
1818 struct bnxt_qplib_cqe **pcqe,
1821 struct bnxt_qplib_qp *qp;
1822 struct bnxt_qplib_q *rq;
1823 struct bnxt_qplib_cqe *cqe;
1827 qp = (struct bnxt_qplib_qp *)((unsigned long)
1828 le64_to_cpu(hwcqe->qp_handle));
1830 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
1834 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1835 cqe->length = le32_to_cpu(hwcqe->length);
1836 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
1837 cqe->flags = le16_to_cpu(hwcqe->flags);
1838 cqe->status = hwcqe->status;
1839 cqe->qp_handle = (u64)(unsigned long)qp;
1840 memcpy(cqe->smac, hwcqe->src_mac, 6);
1841 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
1842 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
1843 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
1845 hwcqe->src_qp_high_srq_or_rq_wr_id) &
1846 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
1849 if (wr_id_idx > rq->hwq.max_elements) {
1850 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
1851 dev_err(&cq->hwq.pdev->dev,
1852 "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1853 wr_id_idx, rq->hwq.max_elements);
1856 if (rq->flush_in_progress)
1859 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1865 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1866 rq->flush_in_progress = true;
1868 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
1870 rq->flush_in_progress = false;
1875 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
1876 struct cq_res_raweth_qp1 *hwcqe,
1877 struct bnxt_qplib_cqe **pcqe,
1880 struct bnxt_qplib_qp *qp;
1881 struct bnxt_qplib_q *rq;
1882 struct bnxt_qplib_cqe *cqe;
1886 qp = (struct bnxt_qplib_qp *)((unsigned long)
1887 le64_to_cpu(hwcqe->qp_handle));
1889 dev_err(&cq->hwq.pdev->dev,
1890 "QPLIB: process_cq Raw/QP1 qp is NULL");
1894 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1895 cqe->flags = le16_to_cpu(hwcqe->flags);
1896 cqe->qp_handle = (u64)(unsigned long)qp;
1899 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
1900 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
1901 cqe->src_qp = qp->id;
1902 if (qp->id == 1 && !cqe->length) {
1903 /* Add workaround for the length misdetection */
1906 cqe->length = le16_to_cpu(hwcqe->length);
1908 cqe->pkey_index = qp->pkey_index;
1909 memcpy(cqe->smac, qp->smac, 6);
1911 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
1912 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
1915 if (wr_id_idx > rq->hwq.max_elements) {
1916 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1917 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1918 wr_id_idx, rq->hwq.max_elements);
1921 if (rq->flush_in_progress)
1924 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1930 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1931 rq->flush_in_progress = true;
1933 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
1936 rq->flush_in_progress = false;
1941 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
1942 struct cq_terminal *hwcqe,
1943 struct bnxt_qplib_cqe **pcqe,
1946 struct bnxt_qplib_qp *qp;
1947 struct bnxt_qplib_q *sq, *rq;
1948 struct bnxt_qplib_cqe *cqe;
1949 u32 sw_cons = 0, cqe_cons;
1953 /* Check the Status */
1954 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
1955 dev_warn(&cq->hwq.pdev->dev,
1956 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1959 qp = (struct bnxt_qplib_qp *)((unsigned long)
1960 le64_to_cpu(hwcqe->qp_handle));
1962 dev_err(&cq->hwq.pdev->dev,
1963 "QPLIB: FP: CQ Process terminal qp is NULL");
1966 /* Must block new posting of SQ and RQ */
1967 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1972 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
1973 if (cqe_cons == 0xFFFF)
1976 if (cqe_cons > sq->hwq.max_elements) {
1977 dev_err(&cq->hwq.pdev->dev,
1978 "QPLIB: FP: CQ Process terminal reported ");
1979 dev_err(&cq->hwq.pdev->dev,
1980 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1981 cqe_cons, sq->hwq.max_elements);
1984 /* If we were in the middle of flushing, continue */
1985 if (sq->flush_in_progress)
1988 /* Terminal CQE can also include aggregated successful CQEs prior.
1989 * So we must complete all CQEs from the current sq's cons to the
1990 * cq_cons with status OK
1994 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1995 if (sw_cons == cqe_cons)
1997 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
1998 memset(cqe, 0, sizeof(*cqe));
1999 cqe->status = CQ_REQ_STATUS_OK;
2000 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2001 cqe->qp_handle = (u64)(unsigned long)qp;
2002 cqe->src_qp = qp->id;
2003 cqe->wr_id = sq->swq[sw_cons].wr_id;
2004 cqe->type = sq->swq[sw_cons].type;
2011 if (!(*budget) && sw_cons != cqe_cons) {
2016 sq->flush_in_progress = true;
2018 rc = __flush_sq(sq, qp, pcqe, budget);
2020 sq->flush_in_progress = false;
2025 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2026 if (cqe_cons == 0xFFFF) {
2028 } else if (cqe_cons > rq->hwq.max_elements) {
2029 dev_err(&cq->hwq.pdev->dev,
2030 "QPLIB: FP: CQ Processed terminal ");
2031 dev_err(&cq->hwq.pdev->dev,
2032 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2033 cqe_cons, rq->hwq.max_elements);
2036 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2037 * from the current rq->cons to the rq->prod regardless what the
2038 * rq->cons the terminal CQE indicates
2040 rq->flush_in_progress = true;
2042 case CMDQ_CREATE_QP1_TYPE_GSI:
2043 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2045 case CMDQ_CREATE_QP_TYPE_RC:
2046 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2048 case CMDQ_CREATE_QP_TYPE_UD:
2049 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2053 rc = __flush_rq(rq, qp, opcode, pcqe, budget);
2055 rq->flush_in_progress = false;
2060 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2061 struct cq_cutoff *hwcqe)
2063 /* Check the Status */
2064 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2065 dev_err(&cq->hwq.pdev->dev,
2066 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2070 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2071 wake_up_interruptible(&cq->waitq);
2076 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2079 struct cq_base *hw_cqe, **hw_cqe_ptr;
2080 unsigned long flags;
2081 u32 sw_cons, raw_cons;
2084 spin_lock_irqsave(&cq->hwq.lock, flags);
2085 raw_cons = cq->hwq.cons;
2089 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2090 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2091 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2093 /* Check for Valid bit */
2094 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2097 /* From the device's respective CQE format to qplib_wc*/
2098 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2099 case CQ_BASE_CQE_TYPE_REQ:
2100 rc = bnxt_qplib_cq_process_req(cq,
2101 (struct cq_req *)hw_cqe,
2104 case CQ_BASE_CQE_TYPE_RES_RC:
2105 rc = bnxt_qplib_cq_process_res_rc(cq,
2106 (struct cq_res_rc *)
2110 case CQ_BASE_CQE_TYPE_RES_UD:
2111 rc = bnxt_qplib_cq_process_res_ud
2112 (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2115 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2116 rc = bnxt_qplib_cq_process_res_raweth_qp1
2117 (cq, (struct cq_res_raweth_qp1 *)
2118 hw_cqe, &cqe, &budget);
2120 case CQ_BASE_CQE_TYPE_TERMINAL:
2121 rc = bnxt_qplib_cq_process_terminal
2122 (cq, (struct cq_terminal *)hw_cqe,
2125 case CQ_BASE_CQE_TYPE_CUT_OFF:
2126 bnxt_qplib_cq_process_cutoff
2127 (cq, (struct cq_cutoff *)hw_cqe);
2128 /* Done processing this CQ */
2131 dev_err(&cq->hwq.pdev->dev,
2132 "QPLIB: process_cq unknown type 0x%lx",
2133 hw_cqe->cqe_type_toggle &
2134 CQ_BASE_CQE_TYPE_MASK);
2141 /* Error while processing the CQE, just skip to the
2144 dev_err(&cq->hwq.pdev->dev,
2145 "QPLIB: process_cqe error rc = 0x%x", rc);
2149 if (cq->hwq.cons != raw_cons) {
2150 cq->hwq.cons = raw_cons;
2151 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2154 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2155 return num_cqes - budget;
2158 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2160 unsigned long flags;
2162 spin_lock_irqsave(&cq->hwq.lock, flags);
2164 bnxt_qplib_arm_cq(cq, arm_type);
2166 spin_unlock_irqrestore(&cq->hwq.lock, flags);