drivers/infiniband/hw/bnxt_re/qplib_fp.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: Fast Path Operators
  37  */
  38
  39 #include <linux/interrupt.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/sched.h>
  42 #include <linux/slab.h>
  43 #include <linux/pci.h>
  44 #include <linux/prefetch.h>
  45
  46 #include "roce_hsi.h"
  47
  48 #include "qplib_res.h"
  49 #include "qplib_rcfw.h"
  50 #include "qplib_sp.h"
  51 #include "qplib_fp.h"
  52
  53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
  54
  55 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
  56                                        struct bnxt_qplib_qp *qp)
  57 {
  58         struct bnxt_qplib_q *rq = &qp->rq;
  59         struct bnxt_qplib_q *sq = &qp->sq;
  60
  61         if (qp->rq_hdr_buf)
  62                 dma_free_coherent(&res->pdev->dev,
  63                                   rq->hwq.max_elements * qp->rq_hdr_buf_size,
  64                                   qp->rq_hdr_buf, qp->rq_hdr_buf_map);
  65         if (qp->sq_hdr_buf)
  66                 dma_free_coherent(&res->pdev->dev,
  67                                   sq->hwq.max_elements * qp->sq_hdr_buf_size,
  68                                   qp->sq_hdr_buf, qp->sq_hdr_buf_map);
  69         qp->rq_hdr_buf = NULL;
  70         qp->sq_hdr_buf = NULL;
  71         qp->rq_hdr_buf_map = 0;
  72         qp->sq_hdr_buf_map = 0;
  73         qp->sq_hdr_buf_size = 0;
  74         qp->rq_hdr_buf_size = 0;
  75 }
  76
  77 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
  78                                        struct bnxt_qplib_qp *qp)
  79 {
  80         struct bnxt_qplib_q *rq = &qp->rq;
  81         struct bnxt_qplib_q *sq = &qp->rq;
  82         int rc = 0;
  83
  84         if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
  85                 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
  86                                         sq->hwq.max_elements *
  87                                         qp->sq_hdr_buf_size,
  88                                         &qp->sq_hdr_buf_map, GFP_KERNEL);
  89                 if (!qp->sq_hdr_buf) {
  90                         rc = -ENOMEM;
  91                         dev_err(&res->pdev->dev,
  92                                 "QPLIB: Failed to create sq_hdr_buf");
  93                         goto fail;
  94                 }
  95         }
  96
  97         if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
  98                 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
  99                                                     rq->hwq.max_elements *
 100                                                     qp->rq_hdr_buf_size,
 101                                                     &qp->rq_hdr_buf_map,
 102                                                     GFP_KERNEL);
 103                 if (!qp->rq_hdr_buf) {
 104                         rc = -ENOMEM;
 105                         dev_err(&res->pdev->dev,
 106                                 "QPLIB: Failed to create rq_hdr_buf");
 107                         goto fail;
 108                 }
 109         }
 110         return 0;
 111
 112 fail:
 113         bnxt_qplib_free_qp_hdr_buf(res, qp);
 114         return rc;
 115 }
 116
 117 static void bnxt_qplib_service_nq(unsigned long data)
 118 {
 119         struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
 120         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 121         struct nq_base *nqe, **nq_ptr;
 122         int num_cqne_processed = 0;
 123         u32 sw_cons, raw_cons;
 124         u16 type;
 125         int budget = nq->budget;
 126         u64 q_handle;
 127
 128         /* Service the NQ until empty */
 129         raw_cons = hwq->cons;
 130         while (budget--) {
 131                 sw_cons = HWQ_CMP(raw_cons, hwq);
 132                 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
 133                 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
 134                 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
 135                         break;
 136
 137                 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
 138                 switch (type) {
 139                 case NQ_BASE_TYPE_CQ_NOTIFICATION:
 140                 {
 141                         struct nq_cn *nqcne = (struct nq_cn *)nqe;
 142
 143                         q_handle = le32_to_cpu(nqcne->cq_handle_low);
 144                         q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
 145                                                      << 32;
 146                         bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
 147                                                  ((unsigned long)q_handle));
 148                         if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
 149                                                  ((unsigned long)q_handle)))
 150                                 num_cqne_processed++;
 151                         else
 152                                 dev_warn(&nq->pdev->dev,
 153                                          "QPLIB: cqn - type 0x%x not handled",
 154                                          type);
 155                         break;
 156                 }
 157                 case NQ_BASE_TYPE_DBQ_EVENT:
 158                         break;
 159                 default:
 160                         dev_warn(&nq->pdev->dev,
 161                                  "QPLIB: nqe with type = 0x%x not handled",
 162                                  type);
 163                         break;
 164                 }
 165                 raw_cons++;
 166         }
 167         if (hwq->cons != raw_cons) {
 168                 hwq->cons = raw_cons;
 169                 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
 170         }
 171 }
 172
 173 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
 174 {
 175         struct bnxt_qplib_nq *nq = dev_instance;
 176         struct bnxt_qplib_hwq *hwq = &nq->hwq;
 177         struct nq_base **nq_ptr;
 178         u32 sw_cons;
 179
 180         /* Prefetch the NQ element */
 181         sw_cons = HWQ_CMP(hwq->cons, hwq);
 182         nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
 183         prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
 184
 185         /* Fan out to CPU affinitized kthreads? */
 186         tasklet_schedule(&nq->worker);
 187
 188         return IRQ_HANDLED;
 189 }
 190
 191 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 192 {
 193         /* Make sure the HW is stopped! */
 194         synchronize_irq(nq->vector);
 195         tasklet_disable(&nq->worker);
 196         tasklet_kill(&nq->worker);
 197
 198         if (nq->requested) {
 199                 free_irq(nq->vector, nq);
 200                 nq->requested = false;
 201         }
 202         if (nq->bar_reg_iomem)
 203                 iounmap(nq->bar_reg_iomem);
 204         nq->bar_reg_iomem = NULL;
 205
 206         nq->cqn_handler = NULL;
 207         nq->srqn_handler = NULL;
 208         nq->vector = 0;
 209 }
 210
 211 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 212                          int msix_vector, int bar_reg_offset,
 213                          int (*cqn_handler)(struct bnxt_qplib_nq *nq,
 214                                             struct bnxt_qplib_cq *),
 215                          int (*srqn_handler)(struct bnxt_qplib_nq *nq,
 216                                              void *, u8 event))
 217 {
 218         resource_size_t nq_base;
 219         int rc;
 220
 221         nq->pdev = pdev;
 222         nq->vector = msix_vector;
 223
 224         nq->cqn_handler = cqn_handler;
 225
 226         nq->srqn_handler = srqn_handler;
 227
 228         tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
 229
 230         nq->requested = false;
 231         rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
 232         if (rc) {
 233                 dev_err(&nq->pdev->dev,
 234                         "Failed to request IRQ for NQ: %#x", rc);
 235                 bnxt_qplib_disable_nq(nq);
 236                 goto fail;
 237         }
 238         nq->requested = true;
 239         nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
 240         nq->bar_reg_off = bar_reg_offset;
 241         nq_base = pci_resource_start(pdev, nq->bar_reg);
 242         if (!nq_base) {
 243                 rc = -ENOMEM;
 244                 goto fail;
 245         }
 246         nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
 247         if (!nq->bar_reg_iomem) {
 248                 rc = -ENOMEM;
 249                 goto fail;
 250         }
 251         NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
 252
 253         return 0;
 254 fail:
 255         bnxt_qplib_disable_nq(nq);
 256         return rc;
 257 }
 258
 259 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
 260 {
 261         if (nq->hwq.max_elements)
 262                 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
 263 }
 264
 265 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
 266 {
 267         nq->pdev = pdev;
 268         if (!nq->hwq.max_elements ||
 269             nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
 270                 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
 271
 272         if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
 273                                       &nq->hwq.max_elements,
 274                                       BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
 275                                       PAGE_SIZE, HWQ_TYPE_L2_CMPL))
 276                 return -ENOMEM;
 277
 278         nq->budget = 8;
 279         return 0;
 280 }
 281
 282 /* QP */
 283 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 284 {
 285         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 286         struct cmdq_create_qp1 req;
 287         struct creq_create_qp1_resp *resp;
 288         struct bnxt_qplib_pbl *pbl;
 289         struct bnxt_qplib_q *sq = &qp->sq;
 290         struct bnxt_qplib_q *rq = &qp->rq;
 291         int rc;
 292         u16 cmd_flags = 0;
 293         u32 qp_flags = 0;
 294
 295         RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
 296
 297         /* General */
 298         req.type = qp->type;
 299         req.dpi = cpu_to_le32(qp->dpi->dpi);
 300         req.qp_handle = cpu_to_le64(qp->qp_handle);
 301
 302         /* SQ */
 303         sq->hwq.max_elements = sq->max_wqe;
 304         rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
 305                                        &sq->hwq.max_elements,
 306                                        BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
 307                                        PAGE_SIZE, HWQ_TYPE_QUEUE);
 308         if (rc)
 309                 goto exit;
 310
 311         sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
 312         if (!sq->swq) {
 313                 rc = -ENOMEM;
 314                 goto fail_sq;
 315         }
 316         pbl = &sq->hwq.pbl[PBL_LVL_0];
 317         req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 318         req.sq_pg_size_sq_lvl =
 319                 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
 320                                 <<  CMDQ_CREATE_QP1_SQ_LVL_SFT) |
 321                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 322                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
 323                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 324                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
 325                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 326                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
 327                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 328                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
 329                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 330                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
 331                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 332                                 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
 333                  CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
 334
 335         if (qp->scq)
 336                 req.scq_cid = cpu_to_le32(qp->scq->id);
 337
 338         qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
 339
 340         /* RQ */
 341         if (rq->max_wqe) {
 342                 rq->hwq.max_elements = qp->rq.max_wqe;
 343                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
 344                                                &rq->hwq.max_elements,
 345                                                BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 346                                                PAGE_SIZE, HWQ_TYPE_QUEUE);
 347                 if (rc)
 348                         goto fail_sq;
 349
 350                 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
 351                                   GFP_KERNEL);
 352                 if (!rq->swq) {
 353                         rc = -ENOMEM;
 354                         goto fail_rq;
 355                 }
 356                 pbl = &rq->hwq.pbl[PBL_LVL_0];
 357                 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 358                 req.rq_pg_size_rq_lvl =
 359                         ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
 360                          CMDQ_CREATE_QP1_RQ_LVL_SFT) |
 361                                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 362                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
 363                                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 364                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
 365                                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 366                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
 367                                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 368                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
 369                                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 370                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
 371                                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 372                                         CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
 373                                  CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
 374                 if (qp->rcq)
 375                         req.rcq_cid = cpu_to_le32(qp->rcq->id);
 376         }
 377
 378         /* Header buffer - allow hdr_buf pass in */
 379         rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 380         if (rc) {
 381                 rc = -ENOMEM;
 382                 goto fail;
 383         }
 384         req.qp_flags = cpu_to_le32(qp_flags);
 385         req.sq_size = cpu_to_le32(sq->hwq.max_elements);
 386         req.rq_size = cpu_to_le32(rq->hwq.max_elements);
 387
 388         req.sq_fwo_sq_sge =
 389                 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
 390                             CMDQ_CREATE_QP1_SQ_SGE_SFT);
 391         req.rq_fwo_rq_sge =
 392                 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
 393                             CMDQ_CREATE_QP1_RQ_SGE_SFT);
 394
 395         req.pd_id = cpu_to_le32(qp->pd->id);
 396
 397         resp = (struct creq_create_qp1_resp *)
 398                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 399                                                      NULL, 0);
 400         if (!resp) {
 401                 dev_err(&res->pdev->dev, "QPLIB: FP: CREATE_QP1 send failed");
 402                 rc = -EINVAL;
 403                 goto fail;
 404         }
 405         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
 406                 /* Cmd timed out */
 407                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 timed out");
 408                 rc = -ETIMEDOUT;
 409                 goto fail;
 410         }
 411         if (resp->status ||
 412             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
 413                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP1 failed ");
 414                 dev_err(&rcfw->pdev->dev,
 415                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
 416                         resp->status, le16_to_cpu(req.cookie),
 417                         le16_to_cpu(resp->cookie));
 418                 rc = -EINVAL;
 419                 goto fail;
 420         }
 421         qp->id = le32_to_cpu(resp->xid);
 422         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
 423         sq->flush_in_progress = false;
 424         rq->flush_in_progress = false;
 425
 426         return 0;
 427
 428 fail:
 429         bnxt_qplib_free_qp_hdr_buf(res, qp);
 430 fail_rq:
 431         bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
 432         kfree(rq->swq);
 433 fail_sq:
 434         bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
 435         kfree(sq->swq);
 436 exit:
 437         return rc;
 438 }
 439
 440 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 441 {
 442         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 443         struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
 444         struct cmdq_create_qp req;
 445         struct creq_create_qp_resp *resp;
 446         struct bnxt_qplib_pbl *pbl;
 447         struct sq_psn_search **psn_search_ptr;
 448         unsigned long int psn_search, poff = 0;
 449         struct bnxt_qplib_q *sq = &qp->sq;
 450         struct bnxt_qplib_q *rq = &qp->rq;
 451         struct bnxt_qplib_hwq *xrrq;
 452         int i, rc, req_size, psn_sz;
 453         u16 cmd_flags = 0, max_ssge;
 454         u32 sw_prod, qp_flags = 0;
 455
 456         RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
 457
 458         /* General */
 459         req.type = qp->type;
 460         req.dpi = cpu_to_le32(qp->dpi->dpi);
 461         req.qp_handle = cpu_to_le64(qp->qp_handle);
 462
 463         /* SQ */
 464         psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
 465                  sizeof(struct sq_psn_search) : 0;
 466         sq->hwq.max_elements = sq->max_wqe;
 467         rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
 468                                        sq->nmap, &sq->hwq.max_elements,
 469                                        BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
 470                                        psn_sz,
 471                                        PAGE_SIZE, HWQ_TYPE_QUEUE);
 472         if (rc)
 473                 goto exit;
 474
 475         sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
 476         if (!sq->swq) {
 477                 rc = -ENOMEM;
 478                 goto fail_sq;
 479         }
 480         hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
 481         if (psn_sz) {
 482                 psn_search_ptr = (struct sq_psn_search **)
 483                                   &hw_sq_send_ptr[get_sqe_pg
 484                                         (sq->hwq.max_elements)];
 485                 psn_search = (unsigned long int)
 486                               &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
 487                               [get_sqe_idx(sq->hwq.max_elements)];
 488                 if (psn_search & ~PAGE_MASK) {
 489                         /* If the psn_search does not start on a page boundary,
 490                          * then calculate the offset
 491                          */
 492                         poff = (psn_search & ~PAGE_MASK) /
 493                                 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
 494                 }
 495                 for (i = 0; i < sq->hwq.max_elements; i++)
 496                         sq->swq[i].psn_search =
 497                                 &psn_search_ptr[get_psne_pg(i + poff)]
 498                                                [get_psne_idx(i + poff)];
 499         }
 500         pbl = &sq->hwq.pbl[PBL_LVL_0];
 501         req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 502         req.sq_pg_size_sq_lvl =
 503                 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
 504                                  <<  CMDQ_CREATE_QP_SQ_LVL_SFT) |
 505                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 506                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
 507                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 508                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
 509                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 510                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
 511                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 512                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
 513                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 514                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
 515                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 516                                 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
 517                  CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
 518
 519         /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
 520         hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
 521         for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
 522                 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
 523                                                 [get_sqe_idx(sw_prod)];
 524                 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
 525         }
 526
 527         if (qp->scq)
 528                 req.scq_cid = cpu_to_le32(qp->scq->id);
 529
 530         qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
 531         qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
 532         if (qp->sig_type)
 533                 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
 534
 535         /* RQ */
 536         if (rq->max_wqe) {
 537                 rq->hwq.max_elements = rq->max_wqe;
 538                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
 539                                                rq->nmap, &rq->hwq.max_elements,
 540                                                BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 541                                                PAGE_SIZE, HWQ_TYPE_QUEUE);
 542                 if (rc)
 543                         goto fail_sq;
 544
 545                 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
 546                                   GFP_KERNEL);
 547                 if (!rq->swq) {
 548                         rc = -ENOMEM;
 549                         goto fail_rq;
 550                 }
 551                 pbl = &rq->hwq.pbl[PBL_LVL_0];
 552                 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 553                 req.rq_pg_size_rq_lvl =
 554                         ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
 555                          CMDQ_CREATE_QP_RQ_LVL_SFT) |
 556                                 (pbl->pg_size == ROCE_PG_SIZE_4K ?
 557                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
 558                                  pbl->pg_size == ROCE_PG_SIZE_8K ?
 559                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
 560                                  pbl->pg_size == ROCE_PG_SIZE_64K ?
 561                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
 562                                  pbl->pg_size == ROCE_PG_SIZE_2M ?
 563                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
 564                                  pbl->pg_size == ROCE_PG_SIZE_8M ?
 565                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
 566                                  pbl->pg_size == ROCE_PG_SIZE_1G ?
 567                                         CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
 568                                  CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
 569         }
 570
 571         if (qp->rcq)
 572                 req.rcq_cid = cpu_to_le32(qp->rcq->id);
 573         req.qp_flags = cpu_to_le32(qp_flags);
 574         req.sq_size = cpu_to_le32(sq->hwq.max_elements);
 575         req.rq_size = cpu_to_le32(rq->hwq.max_elements);
 576         qp->sq_hdr_buf = NULL;
 577         qp->rq_hdr_buf = NULL;
 578
 579         rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 580         if (rc)
 581                 goto fail_rq;
 582
 583         /* CTRL-22434: Irrespective of the requested SGE count on the SQ
 584          * always create the QP with max send sges possible if the requested
 585          * inline size is greater than 0.
 586          */
 587         max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
 588         req.sq_fwo_sq_sge = cpu_to_le16(
 589                                 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
 590                                  << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
 591         req.rq_fwo_rq_sge = cpu_to_le16(
 592                                 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
 593                                  << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
 594         /* ORRQ and IRRQ */
 595         if (psn_sz) {
 596                 xrrq = &qp->orrq;
 597                 xrrq->max_elements =
 598                         ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
 599                 req_size = xrrq->max_elements *
 600                            BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
 601                 req_size &= ~(PAGE_SIZE - 1);
 602                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
 603                                                &xrrq->max_elements,
 604                                                BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
 605                                                0, req_size, HWQ_TYPE_CTX);
 606                 if (rc)
 607                         goto fail_buf_free;
 608                 pbl = &xrrq->pbl[PBL_LVL_0];
 609                 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
 610
 611                 xrrq = &qp->irrq;
 612                 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
 613                                                 qp->max_dest_rd_atomic);
 614                 req_size = xrrq->max_elements *
 615                            BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
 616                 req_size &= ~(PAGE_SIZE - 1);
 617
 618                 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
 619                                                &xrrq->max_elements,
 620                                                BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
 621                                                0, req_size, HWQ_TYPE_CTX);
 622                 if (rc)
 623                         goto fail_orrq;
 624
 625                 pbl = &xrrq->pbl[PBL_LVL_0];
 626                 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
 627         }
 628         req.pd_id = cpu_to_le32(qp->pd->id);
 629
 630         resp = (struct creq_create_qp_resp *)
 631                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 632                                                      NULL, 0);
 633         if (!resp) {
 634                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP send failed");
 635                 rc = -EINVAL;
 636                 goto fail;
 637         }
 638         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
 639                 /* Cmd timed out */
 640                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP timed out");
 641                 rc = -ETIMEDOUT;
 642                 goto fail;
 643         }
 644         if (resp->status ||
 645             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
 646                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_QP failed ");
 647                 dev_err(&rcfw->pdev->dev,
 648                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
 649                         resp->status, le16_to_cpu(req.cookie),
 650                         le16_to_cpu(resp->cookie));
 651                 rc = -EINVAL;
 652                 goto fail;
 653         }
 654         qp->id = le32_to_cpu(resp->xid);
 655         qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
 656         sq->flush_in_progress = false;
 657         rq->flush_in_progress = false;
 658
 659         return 0;
 660
 661 fail:
 662         if (qp->irrq.max_elements)
 663                 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
 664 fail_orrq:
 665         if (qp->orrq.max_elements)
 666                 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
 667 fail_buf_free:
 668         bnxt_qplib_free_qp_hdr_buf(res, qp);
 669 fail_rq:
 670         bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
 671         kfree(rq->swq);
 672 fail_sq:
 673         bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
 674         kfree(sq->swq);
 675 exit:
 676         return rc;
 677 }
 678
 679 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
 680 {
 681         switch (qp->state) {
 682         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
 683                 /* INIT->RTR, configure the path_mtu to the default
 684                  * 2048 if not being requested
 685                  */
 686                 if (!(qp->modify_flags &
 687                     CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
 688                         qp->modify_flags |=
 689                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
 690                         qp->path_mtu =
 691                                 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
 692                 }
 693                 qp->modify_flags &=
 694                         ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
 695                 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
 696                 if (qp->max_dest_rd_atomic < 1)
 697                         qp->max_dest_rd_atomic = 1;
 698                 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
 699                 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
 700                 if (!(qp->modify_flags &
 701                     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
 702                         qp->modify_flags |=
 703                                 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
 704                         qp->ah.sgid_index = 0;
 705                 }
 706                 break;
 707         default:
 708                 break;
 709         }
 710 }
 711
 712 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
 713 {
 714         switch (qp->state) {
 715         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
 716                 /* Bono FW requires the max_rd_atomic to be >= 1 */
 717                 if (qp->max_rd_atomic < 1)
 718                         qp->max_rd_atomic = 1;
 719                 /* Bono FW does not allow PKEY_INDEX,
 720                  * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
 721                  * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
 722                  * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
 723                  * modification
 724                  */
 725                 qp->modify_flags &=
 726                         ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
 727                           CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
 728                           CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
 729                           CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
 730                           CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
 731                           CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
 732                           CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
 733                           CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
 734                           CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
 735                           CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
 736                           CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
 737                           CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
 738                 break;
 739         default:
 740                 break;
 741         }
 742 }
 743
 744 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
 745 {
 746         switch (qp->cur_qp_state) {
 747         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
 748                 break;
 749         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
 750                 __modify_flags_from_init_state(qp);
 751                 break;
 752         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
 753                 __modify_flags_from_rtr_state(qp);
 754                 break;
 755         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
 756                 break;
 757         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
 758                 break;
 759         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
 760                 break;
 761         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
 762                 break;
 763         default:
 764                 break;
 765         }
 766 }
 767
 768 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 769 {
 770         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 771         struct cmdq_modify_qp req;
 772         struct creq_modify_qp_resp *resp;
 773         u16 cmd_flags = 0, pkey;
 774         u32 temp32[4];
 775         u32 bmask;
 776
 777         RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
 778
 779         /* Filter out the qp_attr_mask based on the state->new transition */
 780         __filter_modify_flags(qp);
 781         bmask = qp->modify_flags;
 782         req.modify_mask = cpu_to_le32(qp->modify_flags);
 783         req.qp_cid = cpu_to_le32(qp->id);
 784         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
 785                 req.network_type_en_sqd_async_notify_new_state =
 786                                 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
 787                                 (qp->en_sqd_async_notify ?
 788                                         CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
 789         }
 790         req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
 791
 792         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
 793                 req.access = qp->access;
 794
 795         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
 796                 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
 797                                          qp->pkey_index, &pkey))
 798                         req.pkey = cpu_to_le16(pkey);
 799         }
 800         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
 801                 req.qkey = cpu_to_le32(qp->qkey);
 802
 803         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
 804                 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
 805                 req.dgid[0] = cpu_to_le32(temp32[0]);
 806                 req.dgid[1] = cpu_to_le32(temp32[1]);
 807                 req.dgid[2] = cpu_to_le32(temp32[2]);
 808                 req.dgid[3] = cpu_to_le32(temp32[3]);
 809         }
 810         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
 811                 req.flow_label = cpu_to_le32(qp->ah.flow_label);
 812
 813         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
 814                 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
 815                                              [qp->ah.sgid_index]);
 816
 817         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
 818                 req.hop_limit = qp->ah.hop_limit;
 819
 820         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
 821                 req.traffic_class = qp->ah.traffic_class;
 822
 823         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
 824                 memcpy(req.dest_mac, qp->ah.dmac, 6);
 825
 826         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
 827                 req.path_mtu = qp->path_mtu;
 828
 829         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
 830                 req.timeout = qp->timeout;
 831
 832         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
 833                 req.retry_cnt = qp->retry_cnt;
 834
 835         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
 836                 req.rnr_retry = qp->rnr_retry;
 837
 838         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
 839                 req.min_rnr_timer = qp->min_rnr_timer;
 840
 841         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
 842                 req.rq_psn = cpu_to_le32(qp->rq.psn);
 843
 844         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
 845                 req.sq_psn = cpu_to_le32(qp->sq.psn);
 846
 847         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
 848                 req.max_rd_atomic =
 849                         ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
 850
 851         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
 852                 req.max_dest_rd_atomic =
 853                         IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
 854
 855         req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
 856         req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
 857         req.sq_sge = cpu_to_le16(qp->sq.max_sge);
 858         req.rq_sge = cpu_to_le16(qp->rq.max_sge);
 859         req.max_inline_data = cpu_to_le32(qp->max_inline_data);
 860         if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
 861                 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
 862
 863         req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
 864
 865         resp = (struct creq_modify_qp_resp *)
 866                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 867                                                      NULL, 0);
 868         if (!resp) {
 869                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP send failed");
 870                 return -EINVAL;
 871         }
 872         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
 873                 /* Cmd timed out */
 874                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP timed out");
 875                 return -ETIMEDOUT;
 876         }
 877         if (resp->status ||
 878             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
 879                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: MODIFY_QP failed ");
 880                 dev_err(&rcfw->pdev->dev,
 881                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
 882                         resp->status, le16_to_cpu(req.cookie),
 883                         le16_to_cpu(resp->cookie));
 884                 return -EINVAL;
 885         }
 886         qp->cur_qp_state = qp->state;
 887         return 0;
 888 }
 889
 890 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 891 {
 892         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 893         struct cmdq_query_qp req;
 894         struct creq_query_qp_resp *resp;
 895         struct creq_query_qp_resp_sb *sb;
 896         u16 cmd_flags = 0;
 897         u32 temp32[4];
 898         int i;
 899
 900         RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
 901
 902         req.qp_cid = cpu_to_le32(qp->id);
 903         req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
 904         resp = (struct creq_query_qp_resp *)
 905                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 906                                                      (void **)&sb, 0);
 907         if (!resp) {
 908                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP send failed");
 909                 return -EINVAL;
 910         }
 911         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
 912                 /* Cmd timed out */
 913                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP timed out");
 914                 return -ETIMEDOUT;
 915         }
 916         if (resp->status ||
 917             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
 918                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: QUERY_QP failed ");
 919                 dev_err(&rcfw->pdev->dev,
 920                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
 921                         resp->status, le16_to_cpu(req.cookie),
 922                         le16_to_cpu(resp->cookie));
 923                 return -EINVAL;
 924         }
 925         /* Extract the context from the side buffer */
 926         qp->state = sb->en_sqd_async_notify_state &
 927                         CREQ_QUERY_QP_RESP_SB_STATE_MASK;
 928         qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
 929                                   CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
 930                                   true : false;
 931         qp->access = sb->access;
 932         qp->pkey_index = le16_to_cpu(sb->pkey);
 933         qp->qkey = le32_to_cpu(sb->qkey);
 934
 935         temp32[0] = le32_to_cpu(sb->dgid[0]);
 936         temp32[1] = le32_to_cpu(sb->dgid[1]);
 937         temp32[2] = le32_to_cpu(sb->dgid[2]);
 938         temp32[3] = le32_to_cpu(sb->dgid[3]);
 939         memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
 940
 941         qp->ah.flow_label = le32_to_cpu(sb->flow_label);
 942
 943         qp->ah.sgid_index = 0;
 944         for (i = 0; i < res->sgid_tbl.max; i++) {
 945                 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
 946                         qp->ah.sgid_index = i;
 947                         break;
 948                 }
 949         }
 950         if (i == res->sgid_tbl.max)
 951                 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
 952
 953         qp->ah.hop_limit = sb->hop_limit;
 954         qp->ah.traffic_class = sb->traffic_class;
 955         memcpy(qp->ah.dmac, sb->dest_mac, 6);
 956         qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
 957                                 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
 958                                 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
 959         qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
 960                                     CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
 961                                     CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
 962         qp->timeout = sb->timeout;
 963         qp->retry_cnt = sb->retry_cnt;
 964         qp->rnr_retry = sb->rnr_retry;
 965         qp->min_rnr_timer = sb->min_rnr_timer;
 966         qp->rq.psn = le32_to_cpu(sb->rq_psn);
 967         qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
 968         qp->sq.psn = le32_to_cpu(sb->sq_psn);
 969         qp->max_dest_rd_atomic =
 970                         IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
 971         qp->sq.max_wqe = qp->sq.hwq.max_elements;
 972         qp->rq.max_wqe = qp->rq.hwq.max_elements;
 973         qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
 974         qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
 975         qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
 976         qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
 977         memcpy(qp->smac, sb->src_mac, 6);
 978         qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
 979         return 0;
 980 }
 981
 982 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
 983 {
 984         struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
 985         struct cq_base *hw_cqe, **hw_cqe_ptr;
 986         int i;
 987
 988         for (i = 0; i < cq_hwq->max_elements; i++) {
 989                 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
 990                 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
 991                 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
 992                         continue;
 993                 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
 994                 case CQ_BASE_CQE_TYPE_REQ:
 995                 case CQ_BASE_CQE_TYPE_TERMINAL:
 996                 {
 997                         struct cq_req *cqe = (struct cq_req *)hw_cqe;
 998
 999                         if (qp == le64_to_cpu(cqe->qp_handle))
1000                                 cqe->qp_handle = 0;
1001                         break;
1002                 }
1003                 case CQ_BASE_CQE_TYPE_RES_RC:
1004                 case CQ_BASE_CQE_TYPE_RES_UD:
1005                 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1006                 {
1007                         struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1008
1009                         if (qp == le64_to_cpu(cqe->qp_handle))
1010                                 cqe->qp_handle = 0;
1011                         break;
1012                 }
1013                 default:
1014                         break;
1015                 }
1016         }
1017 }
1018
1019 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1020                           struct bnxt_qplib_qp *qp)
1021 {
1022         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1023         struct cmdq_destroy_qp req;
1024         struct creq_destroy_qp_resp *resp;
1025         unsigned long flags;
1026         u16 cmd_flags = 0;
1027
1028         RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1029
1030         req.qp_cid = cpu_to_le32(qp->id);
1031         resp = (struct creq_destroy_qp_resp *)
1032                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1033                                                      NULL, 0);
1034         if (!resp) {
1035                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP send failed");
1036                 return -EINVAL;
1037         }
1038         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1039                 /* Cmd timed out */
1040                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP timed out");
1041                 return -ETIMEDOUT;
1042         }
1043         if (resp->status ||
1044             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1045                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_QP failed ");
1046                 dev_err(&rcfw->pdev->dev,
1047                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1048                         resp->status, le16_to_cpu(req.cookie),
1049                         le16_to_cpu(resp->cookie));
1050                 return -EINVAL;
1051         }
1052
1053         /* Must walk the associated CQs to nullified the QP ptr */
1054         spin_lock_irqsave(&qp->scq->hwq.lock, flags);
1055
1056         __clean_cq(qp->scq, (u64)(unsigned long)qp);
1057
1058         if (qp->rcq && qp->rcq != qp->scq) {
1059                 spin_lock(&qp->rcq->hwq.lock);
1060                 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
1061                 spin_unlock(&qp->rcq->hwq.lock);
1062         }
1063
1064         spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
1065
1066         bnxt_qplib_free_qp_hdr_buf(res, qp);
1067         bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1068         kfree(qp->sq.swq);
1069
1070         bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1071         kfree(qp->rq.swq);
1072
1073         if (qp->irrq.max_elements)
1074                 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1075         if (qp->orrq.max_elements)
1076                 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1077
1078         return 0;
1079 }
1080
1081 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1082                                 struct bnxt_qplib_sge *sge)
1083 {
1084         struct bnxt_qplib_q *sq = &qp->sq;
1085         u32 sw_prod;
1086
1087         memset(sge, 0, sizeof(*sge));
1088
1089         if (qp->sq_hdr_buf) {
1090                 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1091                 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1092                                          sw_prod * qp->sq_hdr_buf_size);
1093                 sge->lkey = 0xFFFFFFFF;
1094                 sge->size = qp->sq_hdr_buf_size;
1095                 return qp->sq_hdr_buf + sw_prod * sge->size;
1096         }
1097         return NULL;
1098 }
1099
1100 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1101 {
1102         struct bnxt_qplib_q *rq = &qp->rq;
1103
1104         return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1105 }
1106
1107 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1108 {
1109         return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1110 }
1111
1112 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1113                                 struct bnxt_qplib_sge *sge)
1114 {
1115         struct bnxt_qplib_q *rq = &qp->rq;
1116         u32 sw_prod;
1117
1118         memset(sge, 0, sizeof(*sge));
1119
1120         if (qp->rq_hdr_buf) {
1121                 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1122                 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1123                                          sw_prod * qp->rq_hdr_buf_size);
1124                 sge->lkey = 0xFFFFFFFF;
1125                 sge->size = qp->rq_hdr_buf_size;
1126                 return qp->rq_hdr_buf + sw_prod * sge->size;
1127         }
1128         return NULL;
1129 }
1130
1131 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1132 {
1133         struct bnxt_qplib_q *sq = &qp->sq;
1134         struct dbr_dbr db_msg = { 0 };
1135         u32 sw_prod;
1136
1137         sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1138
1139         db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1140                                    DBR_DBR_INDEX_MASK);
1141         db_msg.type_xid =
1142                 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1143                             DBR_DBR_TYPE_SQ);
1144         /* Flush all the WQE writes to HW */
1145         wmb();
1146         __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1147 }
1148
1149 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1150                          struct bnxt_qplib_swqe *wqe)
1151 {
1152         struct bnxt_qplib_q *sq = &qp->sq;
1153         struct bnxt_qplib_swq *swq;
1154         struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1155         struct sq_sge *hw_sge;
1156         u32 sw_prod;
1157         u8 wqe_size16;
1158         int i, rc = 0, data_len = 0, pkt_num = 0;
1159         __le32 temp32;
1160
1161         if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1162                 rc = -EINVAL;
1163                 goto done;
1164         }
1165         if (HWQ_CMP((sq->hwq.prod + 1), &sq->hwq) ==
1166             HWQ_CMP(sq->hwq.cons, &sq->hwq)) {
1167                 rc = -ENOMEM;
1168                 goto done;
1169         }
1170         sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1171         swq = &sq->swq[sw_prod];
1172         swq->wr_id = wqe->wr_id;
1173         swq->type = wqe->type;
1174         swq->flags = wqe->flags;
1175         if (qp->sig_type)
1176                 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1177         swq->start_psn = sq->psn & BTH_PSN_MASK;
1178
1179         hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1180         hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1181                                         [get_sqe_idx(sw_prod)];
1182
1183         memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1184
1185         if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1186                 /* Copy the inline data */
1187                 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1188                         dev_warn(&sq->hwq.pdev->dev,
1189                                  "QPLIB: Inline data length > 96 detected");
1190                         data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1191                 } else {
1192                         data_len = wqe->inline_len;
1193                 }
1194                 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1195                 wqe_size16 = (data_len + 15) >> 4;
1196         } else {
1197                 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1198                      i < wqe->num_sge; i++, hw_sge++) {
1199                         hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1200                         hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1201                         hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1202                         data_len += wqe->sg_list[i].size;
1203                 }
1204                 /* Each SGE entry = 1 WQE size16 */
1205                 wqe_size16 = wqe->num_sge;
1206         }
1207
1208         /* Specifics */
1209         switch (wqe->type) {
1210         case BNXT_QPLIB_SWQE_TYPE_SEND:
1211                 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1212                         /* Assemble info for Raw Ethertype QPs */
1213                         struct sq_send_raweth_qp1 *sqe =
1214                                 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1215
1216                         sqe->wqe_type = wqe->type;
1217                         sqe->flags = wqe->flags;
1218                         sqe->wqe_size = wqe_size16 +
1219                                 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1220                         sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1221                         sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1222                         sqe->length = cpu_to_le32(data_len);
1223                         sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1224                                 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1225                                 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1226
1227                         break;
1228                 }
1229                 /* else, just fall thru */
1230         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1231         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1232         {
1233                 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1234
1235                 sqe->wqe_type = wqe->type;
1236                 sqe->flags = wqe->flags;
1237                 sqe->wqe_size = wqe_size16 +
1238                                 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1239                 sqe->inv_key_or_imm_data = cpu_to_le32(
1240                                                 wqe->send.inv_key);
1241                 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1242                         sqe->q_key = cpu_to_le32(wqe->send.q_key);
1243                         sqe->dst_qp = cpu_to_le32(
1244                                         wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1245                         sqe->length = cpu_to_le32(data_len);
1246                         sqe->avid = cpu_to_le32(wqe->send.avid &
1247                                                 SQ_SEND_AVID_MASK);
1248                         sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1249                 } else {
1250                         sqe->length = cpu_to_le32(data_len);
1251                         sqe->dst_qp = 0;
1252                         sqe->avid = 0;
1253                         if (qp->mtu)
1254                                 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1255                         if (!pkt_num)
1256                                 pkt_num = 1;
1257                         sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1258                 }
1259                 break;
1260         }
1261         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1262         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1263         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1264         {
1265                 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1266
1267                 sqe->wqe_type = wqe->type;
1268                 sqe->flags = wqe->flags;
1269                 sqe->wqe_size = wqe_size16 +
1270                                 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1271                 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1272                 sqe->length = cpu_to_le32((u32)data_len);
1273                 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1274                 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1275                 if (qp->mtu)
1276                         pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1277                 if (!pkt_num)
1278                         pkt_num = 1;
1279                 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1280                 break;
1281         }
1282         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1283         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1284         {
1285                 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1286
1287                 sqe->wqe_type = wqe->type;
1288                 sqe->flags = wqe->flags;
1289                 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1290                 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1291                 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1292                 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1293                 if (qp->mtu)
1294                         pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1295                 if (!pkt_num)
1296                         pkt_num = 1;
1297                 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1298                 break;
1299         }
1300         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1301         {
1302                 struct sq_localinvalidate *sqe =
1303                                 (struct sq_localinvalidate *)hw_sq_send_hdr;
1304
1305                 sqe->wqe_type = wqe->type;
1306                 sqe->flags = wqe->flags;
1307                 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1308
1309                 break;
1310         }
1311         case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1312         {
1313                 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1314
1315                 sqe->wqe_type = wqe->type;
1316                 sqe->flags = wqe->flags;
1317                 sqe->access_cntl = wqe->frmr.access_cntl |
1318                                    SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1319                 sqe->zero_based_page_size_log =
1320                         (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1321                         SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1322                         (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1323                 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1324                 temp32 = cpu_to_le32(wqe->frmr.length);
1325                 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1326                 sqe->numlevels_pbl_page_size_log =
1327                         ((wqe->frmr.pbl_pg_sz_log <<
1328                                         SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1329                                         SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1330                         ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1331                                         SQ_FR_PMR_NUMLEVELS_MASK);
1332
1333                 for (i = 0; i < wqe->frmr.page_list_len; i++)
1334                         wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1335                                                 wqe->frmr.page_list[i] |
1336                                                 PTU_PTE_VALID);
1337                 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1338                 sqe->va = cpu_to_le64(wqe->frmr.va);
1339
1340                 break;
1341         }
1342         case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1343         {
1344                 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1345
1346                 sqe->wqe_type = wqe->type;
1347                 sqe->flags = wqe->flags;
1348                 sqe->access_cntl = wqe->bind.access_cntl;
1349                 sqe->mw_type_zero_based = wqe->bind.mw_type |
1350                         (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1351                 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1352                 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1353                 sqe->va = cpu_to_le64(wqe->bind.va);
1354                 temp32 = cpu_to_le32(wqe->bind.length);
1355                 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1356                 break;
1357         }
1358         default:
1359                 /* Bad wqe, return error */
1360                 rc = -EINVAL;
1361                 goto done;
1362         }
1363         swq->next_psn = sq->psn & BTH_PSN_MASK;
1364         if (swq->psn_search) {
1365                 swq->psn_search->opcode_start_psn = cpu_to_le32(
1366                         ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1367                          SQ_PSN_SEARCH_START_PSN_MASK) |
1368                         ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1369                          SQ_PSN_SEARCH_OPCODE_MASK));
1370                 swq->psn_search->flags_next_psn = cpu_to_le32(
1371                         ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1372                          SQ_PSN_SEARCH_NEXT_PSN_MASK));
1373         }
1374
1375         sq->hwq.prod++;
1376 done:
1377         return rc;
1378 }
1379
1380 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1381 {
1382         struct bnxt_qplib_q *rq = &qp->rq;
1383         struct dbr_dbr db_msg = { 0 };
1384         u32 sw_prod;
1385
1386         sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1387         db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1388                                    DBR_DBR_INDEX_MASK);
1389         db_msg.type_xid =
1390                 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1391                             DBR_DBR_TYPE_RQ);
1392
1393         /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1394         wmb();
1395         __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1396 }
1397
1398 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1399                          struct bnxt_qplib_swqe *wqe)
1400 {
1401         struct bnxt_qplib_q *rq = &qp->rq;
1402         struct rq_wqe *rqe, **rqe_ptr;
1403         struct sq_sge *hw_sge;
1404         u32 sw_prod;
1405         int i, rc = 0;
1406
1407         if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1408                 dev_err(&rq->hwq.pdev->dev,
1409                         "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1410                         qp->id, qp->state);
1411                 rc = -EINVAL;
1412                 goto done;
1413         }
1414         if (HWQ_CMP((rq->hwq.prod + 1), &rq->hwq) ==
1415             HWQ_CMP(rq->hwq.cons, &rq->hwq)) {
1416                 dev_err(&rq->hwq.pdev->dev,
1417                         "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1418                 rc = -EINVAL;
1419                 goto done;
1420         }
1421         sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1422         rq->swq[sw_prod].wr_id = wqe->wr_id;
1423
1424         rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1425         rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1426
1427         memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1428
1429         /* Calculate wqe_size16 and data_len */
1430         for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1431              i < wqe->num_sge; i++, hw_sge++) {
1432                 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1433                 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1434                 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1435         }
1436         rqe->wqe_type = wqe->type;
1437         rqe->flags = wqe->flags;
1438         rqe->wqe_size = wqe->num_sge +
1439                         ((offsetof(typeof(*rqe), data) + 15) >> 4);
1440
1441         /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1442         rqe->wr_id[0] = cpu_to_le32(sw_prod);
1443
1444         rq->hwq.prod++;
1445 done:
1446         return rc;
1447 }
1448
1449 /* CQ */
1450
1451 /* Spinlock must be held */
1452 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1453 {
1454         struct dbr_dbr db_msg = { 0 };
1455
1456         db_msg.type_xid =
1457                 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1458                             DBR_DBR_TYPE_CQ_ARMENA);
1459         /* Flush memory writes before enabling the CQ */
1460         wmb();
1461         __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1462 }
1463
1464 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1465 {
1466         struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1467         struct dbr_dbr db_msg = { 0 };
1468         u32 sw_cons;
1469
1470         /* Ring DB */
1471         sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1472         db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1473                                     DBR_DBR_INDEX_MASK);
1474         db_msg.type_xid =
1475                 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1476                             arm_type);
1477         /* flush memory writes before arming the CQ */
1478         wmb();
1479         __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1480 }
1481
1482 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1483 {
1484         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1485         struct cmdq_create_cq req;
1486         struct creq_create_cq_resp *resp;
1487         struct bnxt_qplib_pbl *pbl;
1488         u16 cmd_flags = 0;
1489         int rc;
1490
1491         cq->hwq.max_elements = cq->max_wqe;
1492         rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1493                                        cq->nmap, &cq->hwq.max_elements,
1494                                        BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1495                                        PAGE_SIZE, HWQ_TYPE_QUEUE);
1496         if (rc)
1497                 goto exit;
1498
1499         RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1500
1501         if (!cq->dpi) {
1502                 dev_err(&rcfw->pdev->dev,
1503                         "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1504                 return -EINVAL;
1505         }
1506         req.dpi = cpu_to_le32(cq->dpi->dpi);
1507         req.cq_handle = cpu_to_le64(cq->cq_handle);
1508
1509         req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1510         pbl = &cq->hwq.pbl[PBL_LVL_0];
1511         req.pg_size_lvl = cpu_to_le32(
1512             ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1513                                                 CMDQ_CREATE_CQ_LVL_SFT) |
1514             (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1515              pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1516              pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1517              pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1518              pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1519              pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1520              CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1521
1522         req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1523
1524         req.cq_fco_cnq_id = cpu_to_le32(
1525                         (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1526                          CMDQ_CREATE_CQ_CNQ_ID_SFT);
1527
1528         resp = (struct creq_create_cq_resp *)
1529                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1530                                                      NULL, 0);
1531         if (!resp) {
1532                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ send failed");
1533                 return -EINVAL;
1534         }
1535         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1536                 /* Cmd timed out */
1537                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ timed out");
1538                 rc = -ETIMEDOUT;
1539                 goto fail;
1540         }
1541         if (resp->status ||
1542             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1543                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed ");
1544                 dev_err(&rcfw->pdev->dev,
1545                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1546                         resp->status, le16_to_cpu(req.cookie),
1547                         le16_to_cpu(resp->cookie));
1548                 rc = -EINVAL;
1549                 goto fail;
1550         }
1551         cq->id = le32_to_cpu(resp->xid);
1552         cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1553         cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1554         init_waitqueue_head(&cq->waitq);
1555
1556         bnxt_qplib_arm_cq_enable(cq);
1557         return 0;
1558
1559 fail:
1560         bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1561 exit:
1562         return rc;
1563 }
1564
1565 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1566 {
1567         struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1568         struct cmdq_destroy_cq req;
1569         struct creq_destroy_cq_resp *resp;
1570         u16 cmd_flags = 0;
1571
1572         RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1573
1574         req.cq_cid = cpu_to_le32(cq->id);
1575         resp = (struct creq_destroy_cq_resp *)
1576                         bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1577                                                      NULL, 0);
1578         if (!resp) {
1579                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ send failed");
1580                 return -EINVAL;
1581         }
1582         if (!bnxt_qplib_rcfw_wait_for_resp(rcfw, le16_to_cpu(req.cookie))) {
1583                 /* Cmd timed out */
1584                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ timed out");
1585                 return -ETIMEDOUT;
1586         }
1587         if (resp->status ||
1588             le16_to_cpu(resp->cookie) != le16_to_cpu(req.cookie)) {
1589                 dev_err(&rcfw->pdev->dev, "QPLIB: FP: DESTROY_CQ failed ");
1590                 dev_err(&rcfw->pdev->dev,
1591                         "QPLIB: with status 0x%x cmdq 0x%x resp 0x%x",
1592                         resp->status, le16_to_cpu(req.cookie),
1593                         le16_to_cpu(resp->cookie));
1594                 return -EINVAL;
1595         }
1596         bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1597         return 0;
1598 }
1599
1600 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1601                       struct bnxt_qplib_cqe **pcqe, int *budget)
1602 {
1603         u32 sw_prod, sw_cons;
1604         struct bnxt_qplib_cqe *cqe;
1605         int rc = 0;
1606
1607         /* Now complete all outstanding SQEs with FLUSHED_ERR */
1608         sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1609         cqe = *pcqe;
1610         while (*budget) {
1611                 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1612                 if (sw_cons == sw_prod) {
1613                         sq->flush_in_progress = false;
1614                         break;
1615                 }
1616                 memset(cqe, 0, sizeof(*cqe));
1617                 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1618                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1619                 cqe->qp_handle = (u64)(unsigned long)qp;
1620                 cqe->wr_id = sq->swq[sw_cons].wr_id;
1621                 cqe->src_qp = qp->id;
1622                 cqe->type = sq->swq[sw_cons].type;
1623                 cqe++;
1624                 (*budget)--;
1625                 sq->hwq.cons++;
1626         }
1627         *pcqe = cqe;
1628         if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1629                 /* Out of budget */
1630                 rc = -EAGAIN;
1631
1632         return rc;
1633 }
1634
1635 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1636                       int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
1637 {
1638         struct bnxt_qplib_cqe *cqe;
1639         u32 sw_prod, sw_cons;
1640         int rc = 0;
1641
1642         /* Flush the rest of the RQ */
1643         sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1644         cqe = *pcqe;
1645         while (*budget) {
1646                 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1647                 if (sw_cons == sw_prod)
1648                         break;
1649                 memset(cqe, 0, sizeof(*cqe));
1650                 cqe->status =
1651                     CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1652                 cqe->opcode = opcode;
1653                 cqe->qp_handle = (unsigned long)qp;
1654                 cqe->wr_id = rq->swq[sw_cons].wr_id;
1655                 cqe++;
1656                 (*budget)--;
1657                 rq->hwq.cons++;
1658         }
1659         *pcqe = cqe;
1660         if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1661                 /* Out of budget */
1662                 rc = -EAGAIN;
1663
1664         return rc;
1665 }
1666
1667 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1668                                      struct cq_req *hwcqe,
1669                                      struct bnxt_qplib_cqe **pcqe, int *budget)
1670 {
1671         struct bnxt_qplib_qp *qp;
1672         struct bnxt_qplib_q *sq;
1673         struct bnxt_qplib_cqe *cqe;
1674         u32 sw_cons, cqe_cons;
1675         int rc = 0;
1676
1677         qp = (struct bnxt_qplib_qp *)((unsigned long)
1678                                       le64_to_cpu(hwcqe->qp_handle));
1679         if (!qp) {
1680                 dev_err(&cq->hwq.pdev->dev,
1681                         "QPLIB: FP: Process Req qp is NULL");
1682                 return -EINVAL;
1683         }
1684         sq = &qp->sq;
1685
1686         cqe_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1687         if (cqe_cons > sq->hwq.max_elements) {
1688                 dev_err(&cq->hwq.pdev->dev,
1689                         "QPLIB: FP: CQ Process req reported ");
1690                 dev_err(&cq->hwq.pdev->dev,
1691                         "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1692                         cqe_cons, sq->hwq.max_elements);
1693                 return -EINVAL;
1694         }
1695         /* If we were in the middle of flushing the SQ, continue */
1696         if (sq->flush_in_progress)
1697                 goto flush;
1698
1699         /* Require to walk the sq's swq to fabricate CQEs for all previously
1700          * signaled SWQEs due to CQE aggregation from the current sq cons
1701          * to the cqe_cons
1702          */
1703         cqe = *pcqe;
1704         while (*budget) {
1705                 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1706                 if (sw_cons == cqe_cons)
1707                         break;
1708                 memset(cqe, 0, sizeof(*cqe));
1709                 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1710                 cqe->qp_handle = (u64)(unsigned long)qp;
1711                 cqe->src_qp = qp->id;
1712                 cqe->wr_id = sq->swq[sw_cons].wr_id;
1713                 cqe->type = sq->swq[sw_cons].type;
1714
1715                 /* For the last CQE, check for status.  For errors, regardless
1716                  * of the request being signaled or not, it must complete with
1717                  * the hwcqe error status
1718                  */
1719                 if (HWQ_CMP((sw_cons + 1), &sq->hwq) == cqe_cons &&
1720                     hwcqe->status != CQ_REQ_STATUS_OK) {
1721                         cqe->status = hwcqe->status;
1722                         dev_err(&cq->hwq.pdev->dev,
1723                                 "QPLIB: FP: CQ Processed Req ");
1724                         dev_err(&cq->hwq.pdev->dev,
1725                                 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1726                                 sw_cons, cqe->wr_id, cqe->status);
1727                         cqe++;
1728                         (*budget)--;
1729                         sq->flush_in_progress = true;
1730                         /* Must block new posting of SQ and RQ */
1731                         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1732                 } else {
1733                         if (sq->swq[sw_cons].flags &
1734                             SQ_SEND_FLAGS_SIGNAL_COMP) {
1735                                 cqe->status = CQ_REQ_STATUS_OK;
1736                                 cqe++;
1737                                 (*budget)--;
1738                         }
1739                 }
1740                 sq->hwq.cons++;
1741         }
1742         *pcqe = cqe;
1743         if (!*budget && HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_cons) {
1744                 /* Out of budget */
1745                 rc = -EAGAIN;
1746                 goto done;
1747         }
1748         if (!sq->flush_in_progress)
1749                 goto done;
1750 flush:
1751         /* Require to walk the sq's swq to fabricate CQEs for all
1752          * previously posted SWQEs due to the error CQE received
1753          */
1754         rc = __flush_sq(sq, qp, pcqe, budget);
1755         if (!rc)
1756                 sq->flush_in_progress = false;
1757 done:
1758         return rc;
1759 }
1760
1761 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
1762                                         struct cq_res_rc *hwcqe,
1763                                         struct bnxt_qplib_cqe **pcqe,
1764                                         int *budget)
1765 {
1766         struct bnxt_qplib_qp *qp;
1767         struct bnxt_qplib_q *rq;
1768         struct bnxt_qplib_cqe *cqe;
1769         u32 wr_id_idx;
1770         int rc = 0;
1771
1772         qp = (struct bnxt_qplib_qp *)((unsigned long)
1773                                       le64_to_cpu(hwcqe->qp_handle));
1774         if (!qp) {
1775                 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
1776                 return -EINVAL;
1777         }
1778         cqe = *pcqe;
1779         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1780         cqe->length = le32_to_cpu(hwcqe->length);
1781         cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
1782         cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
1783         cqe->flags = le16_to_cpu(hwcqe->flags);
1784         cqe->status = hwcqe->status;
1785         cqe->qp_handle = (u64)(unsigned long)qp;
1786
1787         wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
1788                                 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
1789         rq = &qp->rq;
1790         if (wr_id_idx > rq->hwq.max_elements) {
1791                 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
1792                 dev_err(&cq->hwq.pdev->dev,
1793                         "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1794                         wr_id_idx, rq->hwq.max_elements);
1795                 return -EINVAL;
1796         }
1797         if (rq->flush_in_progress)
1798                 goto flush_rq;
1799
1800         cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1801         cqe++;
1802         (*budget)--;
1803         rq->hwq.cons++;
1804         *pcqe = cqe;
1805
1806         if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1807                 rq->flush_in_progress = true;
1808 flush_rq:
1809                 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
1810                 if (!rc)
1811                         rq->flush_in_progress = false;
1812         }
1813         return rc;
1814 }
1815
1816 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
1817                                         struct cq_res_ud *hwcqe,
1818                                         struct bnxt_qplib_cqe **pcqe,
1819                                         int *budget)
1820 {
1821         struct bnxt_qplib_qp *qp;
1822         struct bnxt_qplib_q *rq;
1823         struct bnxt_qplib_cqe *cqe;
1824         u32 wr_id_idx;
1825         int rc = 0;
1826
1827         qp = (struct bnxt_qplib_qp *)((unsigned long)
1828                                       le64_to_cpu(hwcqe->qp_handle));
1829         if (!qp) {
1830                 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
1831                 return -EINVAL;
1832         }
1833         cqe = *pcqe;
1834         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1835         cqe->length = le32_to_cpu(hwcqe->length);
1836         cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
1837         cqe->flags = le16_to_cpu(hwcqe->flags);
1838         cqe->status = hwcqe->status;
1839         cqe->qp_handle = (u64)(unsigned long)qp;
1840         memcpy(cqe->smac, hwcqe->src_mac, 6);
1841         wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
1842                                 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
1843         cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
1844                                   ((le32_to_cpu(
1845                                   hwcqe->src_qp_high_srq_or_rq_wr_id) &
1846                                  CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
1847
1848         rq = &qp->rq;
1849         if (wr_id_idx > rq->hwq.max_elements) {
1850                 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
1851                 dev_err(&cq->hwq.pdev->dev,
1852                         "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1853                         wr_id_idx, rq->hwq.max_elements);
1854                 return -EINVAL;
1855         }
1856         if (rq->flush_in_progress)
1857                 goto flush_rq;
1858
1859         cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1860         cqe++;
1861         (*budget)--;
1862         rq->hwq.cons++;
1863         *pcqe = cqe;
1864
1865         if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1866                 rq->flush_in_progress = true;
1867 flush_rq:
1868                 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
1869                 if (!rc)
1870                         rq->flush_in_progress = false;
1871         }
1872         return rc;
1873 }
1874
1875 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
1876                                                 struct cq_res_raweth_qp1 *hwcqe,
1877                                                 struct bnxt_qplib_cqe **pcqe,
1878                                                 int *budget)
1879 {
1880         struct bnxt_qplib_qp *qp;
1881         struct bnxt_qplib_q *rq;
1882         struct bnxt_qplib_cqe *cqe;
1883         u32 wr_id_idx;
1884         int rc = 0;
1885
1886         qp = (struct bnxt_qplib_qp *)((unsigned long)
1887                                       le64_to_cpu(hwcqe->qp_handle));
1888         if (!qp) {
1889                 dev_err(&cq->hwq.pdev->dev,
1890                         "QPLIB: process_cq Raw/QP1 qp is NULL");
1891                 return -EINVAL;
1892         }
1893         cqe = *pcqe;
1894         cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1895         cqe->flags = le16_to_cpu(hwcqe->flags);
1896         cqe->qp_handle = (u64)(unsigned long)qp;
1897
1898         wr_id_idx =
1899                 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
1900                                 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
1901         cqe->src_qp = qp->id;
1902         if (qp->id == 1 && !cqe->length) {
1903                 /* Add workaround for the length misdetection */
1904                 cqe->length = 296;
1905         } else {
1906                 cqe->length = le16_to_cpu(hwcqe->length);
1907         }
1908         cqe->pkey_index = qp->pkey_index;
1909         memcpy(cqe->smac, qp->smac, 6);
1910
1911         cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
1912         cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
1913
1914         rq = &qp->rq;
1915         if (wr_id_idx > rq->hwq.max_elements) {
1916                 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1917                 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1918                         wr_id_idx, rq->hwq.max_elements);
1919                 return -EINVAL;
1920         }
1921         if (rq->flush_in_progress)
1922                 goto flush_rq;
1923
1924         cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1925         cqe++;
1926         (*budget)--;
1927         rq->hwq.cons++;
1928         *pcqe = cqe;
1929
1930         if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1931                 rq->flush_in_progress = true;
1932 flush_rq:
1933                 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
1934                                 budget);
1935                 if (!rc)
1936                         rq->flush_in_progress = false;
1937         }
1938         return rc;
1939 }
1940
1941 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
1942                                           struct cq_terminal *hwcqe,
1943                                           struct bnxt_qplib_cqe **pcqe,
1944                                           int *budget)
1945 {
1946         struct bnxt_qplib_qp *qp;
1947         struct bnxt_qplib_q *sq, *rq;
1948         struct bnxt_qplib_cqe *cqe;
1949         u32 sw_cons = 0, cqe_cons;
1950         int rc = 0;
1951         u8 opcode = 0;
1952
1953         /* Check the Status */
1954         if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
1955                 dev_warn(&cq->hwq.pdev->dev,
1956                          "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1957                          hwcqe->status);
1958
1959         qp = (struct bnxt_qplib_qp *)((unsigned long)
1960                                       le64_to_cpu(hwcqe->qp_handle));
1961         if (!qp) {
1962                 dev_err(&cq->hwq.pdev->dev,
1963                         "QPLIB: FP: CQ Process terminal qp is NULL");
1964                 return -EINVAL;
1965         }
1966         /* Must block new posting of SQ and RQ */
1967         qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1968
1969         sq = &qp->sq;
1970         rq = &qp->rq;
1971
1972         cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
1973         if (cqe_cons == 0xFFFF)
1974                 goto do_rq;
1975
1976         if (cqe_cons > sq->hwq.max_elements) {
1977                 dev_err(&cq->hwq.pdev->dev,
1978                         "QPLIB: FP: CQ Process terminal reported ");
1979                 dev_err(&cq->hwq.pdev->dev,
1980                         "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1981                         cqe_cons, sq->hwq.max_elements);
1982                 goto do_rq;
1983         }
1984         /* If we were in the middle of flushing, continue */
1985         if (sq->flush_in_progress)
1986                 goto flush_sq;
1987
1988         /* Terminal CQE can also include aggregated successful CQEs prior.
1989          * So we must complete all CQEs from the current sq's cons to the
1990          * cq_cons with status OK
1991          */
1992         cqe = *pcqe;
1993         while (*budget) {
1994                 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1995                 if (sw_cons == cqe_cons)
1996                         break;
1997                 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
1998                         memset(cqe, 0, sizeof(*cqe));
1999                         cqe->status = CQ_REQ_STATUS_OK;
2000                         cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2001                         cqe->qp_handle = (u64)(unsigned long)qp;
2002                         cqe->src_qp = qp->id;
2003                         cqe->wr_id = sq->swq[sw_cons].wr_id;
2004                         cqe->type = sq->swq[sw_cons].type;
2005                         cqe++;
2006                         (*budget)--;
2007                 }
2008                 sq->hwq.cons++;
2009         }
2010         *pcqe = cqe;
2011         if (!(*budget) && sw_cons != cqe_cons) {
2012                 /* Out of budget */
2013                 rc = -EAGAIN;
2014                 goto sq_done;
2015         }
2016         sq->flush_in_progress = true;
2017 flush_sq:
2018         rc = __flush_sq(sq, qp, pcqe, budget);
2019         if (!rc)
2020                 sq->flush_in_progress = false;
2021 sq_done:
2022         if (rc)
2023                 return rc;
2024 do_rq:
2025         cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2026         if (cqe_cons == 0xFFFF) {
2027                 goto done;
2028         } else if (cqe_cons > rq->hwq.max_elements) {
2029                 dev_err(&cq->hwq.pdev->dev,
2030                         "QPLIB: FP: CQ Processed terminal ");
2031                 dev_err(&cq->hwq.pdev->dev,
2032                         "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2033                         cqe_cons, rq->hwq.max_elements);
2034                 goto done;
2035         }
2036         /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2037          * from the current rq->cons to the rq->prod regardless what the
2038          * rq->cons the terminal CQE indicates
2039          */
2040         rq->flush_in_progress = true;
2041         switch (qp->type) {
2042         case CMDQ_CREATE_QP1_TYPE_GSI:
2043                 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2044                 break;
2045         case CMDQ_CREATE_QP_TYPE_RC:
2046                 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2047                 break;
2048         case CMDQ_CREATE_QP_TYPE_UD:
2049                 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2050                 break;
2051         }
2052
2053         rc = __flush_rq(rq, qp, opcode, pcqe, budget);
2054         if (!rc)
2055                 rq->flush_in_progress = false;
2056 done:
2057         return rc;
2058 }
2059
2060 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2061                                         struct cq_cutoff *hwcqe)
2062 {
2063         /* Check the Status */
2064         if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2065                 dev_err(&cq->hwq.pdev->dev,
2066                         "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2067                         hwcqe->status);
2068                 return -EINVAL;
2069         }
2070         clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2071         wake_up_interruptible(&cq->waitq);
2072
2073         return 0;
2074 }
2075
2076 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2077                        int num_cqes)
2078 {
2079         struct cq_base *hw_cqe, **hw_cqe_ptr;
2080         unsigned long flags;
2081         u32 sw_cons, raw_cons;
2082         int budget, rc = 0;
2083
2084         spin_lock_irqsave(&cq->hwq.lock, flags);
2085         raw_cons = cq->hwq.cons;
2086         budget = num_cqes;
2087
2088         while (budget) {
2089                 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2090                 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2091                 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2092
2093                 /* Check for Valid bit */
2094                 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2095                         break;
2096
2097                 /* From the device's respective CQE format to qplib_wc*/
2098                 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2099                 case CQ_BASE_CQE_TYPE_REQ:
2100                         rc = bnxt_qplib_cq_process_req(cq,
2101                                                        (struct cq_req *)hw_cqe,
2102                                                        &cqe, &budget);
2103                         break;
2104                 case CQ_BASE_CQE_TYPE_RES_RC:
2105                         rc = bnxt_qplib_cq_process_res_rc(cq,
2106                                                           (struct cq_res_rc *)
2107                                                           hw_cqe, &cqe,
2108                                                           &budget);
2109                         break;
2110                 case CQ_BASE_CQE_TYPE_RES_UD:
2111                         rc = bnxt_qplib_cq_process_res_ud
2112                                         (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2113                                          &budget);
2114                         break;
2115                 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2116                         rc = bnxt_qplib_cq_process_res_raweth_qp1
2117                                         (cq, (struct cq_res_raweth_qp1 *)
2118                                          hw_cqe, &cqe, &budget);
2119                         break;
2120                 case CQ_BASE_CQE_TYPE_TERMINAL:
2121                         rc = bnxt_qplib_cq_process_terminal
2122                                         (cq, (struct cq_terminal *)hw_cqe,
2123                                          &cqe, &budget);
2124                         break;
2125                 case CQ_BASE_CQE_TYPE_CUT_OFF:
2126                         bnxt_qplib_cq_process_cutoff
2127                                         (cq, (struct cq_cutoff *)hw_cqe);
2128                         /* Done processing this CQ */
2129                         goto exit;
2130                 default:
2131                         dev_err(&cq->hwq.pdev->dev,
2132                                 "QPLIB: process_cq unknown type 0x%lx",
2133                                 hw_cqe->cqe_type_toggle &
2134                                 CQ_BASE_CQE_TYPE_MASK);
2135                         rc = -EINVAL;
2136                         break;
2137                 }
2138                 if (rc < 0) {
2139                         if (rc == -EAGAIN)
2140                                 break;
2141                         /* Error while processing the CQE, just skip to the
2142                          * next one
2143                          */
2144                         dev_err(&cq->hwq.pdev->dev,
2145                                 "QPLIB: process_cqe error rc = 0x%x", rc);
2146                 }
2147                 raw_cons++;
2148         }
2149         if (cq->hwq.cons != raw_cons) {
2150                 cq->hwq.cons = raw_cons;
2151                 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2152         }
2153 exit:
2154         spin_unlock_irqrestore(&cq->hwq.lock, flags);
2155         return num_cqes - budget;
2156 }
2157
2158 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2159 {
2160         unsigned long flags;
2161
2162         spin_lock_irqsave(&cq->hwq.lock, flags);
2163         if (arm_type)
2164                 bnxt_qplib_arm_cq(cq, arm_type);
2165
2166         spin_unlock_irqrestore(&cq->hwq.lock, flags);
2167 }