drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45
  46 #include <linux/qed/common_hsi.h>
  47 #include "qedr_hsi_rdma.h"
  48 #include <linux/qed/qed_if.h>
  49 #include "qedr.h"
  50 #include "verbs.h"
  51 #include <rdma/qedr-abi.h>
  52 #include "qedr_cm.h"
  53
  54 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  55
  56 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  57 {
  58         if (index > QEDR_ROCE_PKEY_TABLE_LEN)
  59                 return -EINVAL;
  60
  61         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  62         return 0;
  63 }
  64
  65 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
  66                    union ib_gid *sgid)
  67 {
  68         struct qedr_dev *dev = get_qedr_dev(ibdev);
  69         int rc = 0;
  70
  71         if (!rdma_cap_roce_gid_table(ibdev, port))
  72                 return -ENODEV;
  73
  74         rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
  75         if (rc == -EAGAIN) {
  76                 memcpy(sgid, &zgid, sizeof(*sgid));
  77                 return 0;
  78         }
  79
  80         DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
  81                  sgid->global.interface_id, sgid->global.subnet_prefix);
  82
  83         return rc;
  84 }
  85
  86 int qedr_add_gid(struct ib_device *device, u8 port_num,
  87                  unsigned int index, const union ib_gid *gid,
  88                  const struct ib_gid_attr *attr, void **context)
  89 {
  90         if (!rdma_cap_roce_gid_table(device, port_num))
  91                 return -EINVAL;
  92
  93         if (port_num > QEDR_MAX_PORT)
  94                 return -EINVAL;
  95
  96         if (!context)
  97                 return -EINVAL;
  98
  99         return 0;
 100 }
 101
 102 int qedr_del_gid(struct ib_device *device, u8 port_num,
 103                  unsigned int index, void **context)
 104 {
 105         if (!rdma_cap_roce_gid_table(device, port_num))
 106                 return -EINVAL;
 107
 108         if (port_num > QEDR_MAX_PORT)
 109                 return -EINVAL;
 110
 111         if (!context)
 112                 return -EINVAL;
 113
 114         return 0;
 115 }
 116
 117 int qedr_query_device(struct ib_device *ibdev,
 118                       struct ib_device_attr *attr, struct ib_udata *udata)
 119 {
 120         struct qedr_dev *dev = get_qedr_dev(ibdev);
 121         struct qedr_device_attr *qattr = &dev->attr;
 122
 123         if (!dev->rdma_ctx) {
 124                 DP_ERR(dev,
 125                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 126                        dev->rdma_ctx);
 127                 return -EINVAL;
 128         }
 129
 130         memset(attr, 0, sizeof(*attr));
 131
 132         attr->fw_ver = qattr->fw_ver;
 133         attr->sys_image_guid = qattr->sys_image_guid;
 134         attr->max_mr_size = qattr->max_mr_size;
 135         attr->page_size_cap = qattr->page_size_caps;
 136         attr->vendor_id = qattr->vendor_id;
 137         attr->vendor_part_id = qattr->vendor_part_id;
 138         attr->hw_ver = qattr->hw_ver;
 139         attr->max_qp = qattr->max_qp;
 140         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 141         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 142             IB_DEVICE_RC_RNR_NAK_GEN |
 143             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 144
 145         attr->max_sge = qattr->max_sge;
 146         attr->max_sge_rd = qattr->max_sge;
 147         attr->max_cq = qattr->max_cq;
 148         attr->max_cqe = qattr->max_cqe;
 149         attr->max_mr = qattr->max_mr;
 150         attr->max_mw = qattr->max_mw;
 151         attr->max_pd = qattr->max_pd;
 152         attr->atomic_cap = dev->atomic_cap;
 153         attr->max_fmr = qattr->max_fmr;
 154         attr->max_map_per_fmr = 16;
 155         attr->max_qp_init_rd_atom =
 156             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 157         attr->max_qp_rd_atom =
 158             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 159                 attr->max_qp_init_rd_atom);
 160
 161         attr->max_srq = qattr->max_srq;
 162         attr->max_srq_sge = qattr->max_srq_sge;
 163         attr->max_srq_wr = qattr->max_srq_wr;
 164
 165         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 166         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 167         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 168         attr->max_ah = qattr->max_ah;
 169
 170         return 0;
 171 }
 172
 173 #define QEDR_SPEED_SDR          (1)
 174 #define QEDR_SPEED_DDR          (2)
 175 #define QEDR_SPEED_QDR          (4)
 176 #define QEDR_SPEED_FDR10        (8)
 177 #define QEDR_SPEED_FDR          (16)
 178 #define QEDR_SPEED_EDR          (32)
 179
 180 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 181                                             u8 *ib_width)
 182 {
 183         switch (speed) {
 184         case 1000:
 185                 *ib_speed = QEDR_SPEED_SDR;
 186                 *ib_width = IB_WIDTH_1X;
 187                 break;
 188         case 10000:
 189                 *ib_speed = QEDR_SPEED_QDR;
 190                 *ib_width = IB_WIDTH_1X;
 191                 break;
 192
 193         case 20000:
 194                 *ib_speed = QEDR_SPEED_DDR;
 195                 *ib_width = IB_WIDTH_4X;
 196                 break;
 197
 198         case 25000:
 199                 *ib_speed = QEDR_SPEED_EDR;
 200                 *ib_width = IB_WIDTH_1X;
 201                 break;
 202
 203         case 40000:
 204                 *ib_speed = QEDR_SPEED_QDR;
 205                 *ib_width = IB_WIDTH_4X;
 206                 break;
 207
 208         case 50000:
 209                 *ib_speed = QEDR_SPEED_QDR;
 210                 *ib_width = IB_WIDTH_4X;
 211                 break;
 212
 213         case 100000:
 214                 *ib_speed = QEDR_SPEED_EDR;
 215                 *ib_width = IB_WIDTH_4X;
 216                 break;
 217
 218         default:
 219                 /* Unsupported */
 220                 *ib_speed = QEDR_SPEED_SDR;
 221                 *ib_width = IB_WIDTH_1X;
 222         }
 223 }
 224
 225 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 226 {
 227         struct qedr_dev *dev;
 228         struct qed_rdma_port *rdma_port;
 229
 230         dev = get_qedr_dev(ibdev);
 231         if (port > 1) {
 232                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 233                 return -EINVAL;
 234         }
 235
 236         if (!dev->rdma_ctx) {
 237                 DP_ERR(dev, "rdma_ctx is NULL\n");
 238                 return -EINVAL;
 239         }
 240
 241         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 242
 243         /* *attr being zeroed by the caller, avoid zeroing it here */
 244         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 245                 attr->state = IB_PORT_ACTIVE;
 246                 attr->phys_state = 5;
 247         } else {
 248                 attr->state = IB_PORT_DOWN;
 249                 attr->phys_state = 3;
 250         }
 251         attr->max_mtu = IB_MTU_4096;
 252         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 253         attr->lid = 0;
 254         attr->lmc = 0;
 255         attr->sm_lid = 0;
 256         attr->sm_sl = 0;
 257         attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
 258         attr->gid_tbl_len = QEDR_MAX_SGID;
 259         attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 260         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 261         attr->qkey_viol_cntr = 0;
 262         get_link_speed_and_width(rdma_port->link_speed,
 263                                  &attr->active_speed, &attr->active_width);
 264         attr->max_msg_sz = rdma_port->max_msg_size;
 265         attr->max_vl_num = 4;
 266
 267         return 0;
 268 }
 269
 270 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
 271                      struct ib_port_modify *props)
 272 {
 273         struct qedr_dev *dev;
 274
 275         dev = get_qedr_dev(ibdev);
 276         if (port > 1) {
 277                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 278                 return -EINVAL;
 279         }
 280
 281         return 0;
 282 }
 283
 284 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 285                          unsigned long len)
 286 {
 287         struct qedr_mm *mm;
 288
 289         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 290         if (!mm)
 291                 return -ENOMEM;
 292
 293         mm->key.phy_addr = phy_addr;
 294         /* This function might be called with a length which is not a multiple
 295          * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
 296          * forces this granularity by increasing the requested size if needed.
 297          * When qedr_mmap is called, it will search the list with the updated
 298          * length as a key. To prevent search failures, the length is rounded up
 299          * in advance to PAGE_SIZE.
 300          */
 301         mm->key.len = roundup(len, PAGE_SIZE);
 302         INIT_LIST_HEAD(&mm->entry);
 303
 304         mutex_lock(&uctx->mm_list_lock);
 305         list_add(&mm->entry, &uctx->mm_head);
 306         mutex_unlock(&uctx->mm_list_lock);
 307
 308         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 309                  "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 310                  (unsigned long long)mm->key.phy_addr,
 311                  (unsigned long)mm->key.len, uctx);
 312
 313         return 0;
 314 }
 315
 316 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 317                              unsigned long len)
 318 {
 319         bool found = false;
 320         struct qedr_mm *mm;
 321
 322         mutex_lock(&uctx->mm_list_lock);
 323         list_for_each_entry(mm, &uctx->mm_head, entry) {
 324                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 325                         continue;
 326
 327                 found = true;
 328                 break;
 329         }
 330         mutex_unlock(&uctx->mm_list_lock);
 331         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 332                  "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
 333                  mm->key.phy_addr, mm->key.len, uctx, found);
 334
 335         return found;
 336 }
 337
 338 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
 339                                         struct ib_udata *udata)
 340 {
 341         int rc;
 342         struct qedr_ucontext *ctx;
 343         struct qedr_alloc_ucontext_resp uresp;
 344         struct qedr_dev *dev = get_qedr_dev(ibdev);
 345         struct qed_rdma_add_user_out_params oparams;
 346
 347         if (!udata)
 348                 return ERR_PTR(-EFAULT);
 349
 350         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 351         if (!ctx)
 352                 return ERR_PTR(-ENOMEM);
 353
 354         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 355         if (rc) {
 356                 DP_ERR(dev,
 357                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 358                        rc);
 359                 goto err;
 360         }
 361
 362         ctx->dpi = oparams.dpi;
 363         ctx->dpi_addr = oparams.dpi_addr;
 364         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 365         ctx->dpi_size = oparams.dpi_size;
 366         INIT_LIST_HEAD(&ctx->mm_head);
 367         mutex_init(&ctx->mm_list_lock);
 368
 369         memset(&uresp, 0, sizeof(uresp));
 370
 371         uresp.db_pa = ctx->dpi_phys_addr;
 372         uresp.db_size = ctx->dpi_size;
 373         uresp.max_send_wr = dev->attr.max_sqe;
 374         uresp.max_recv_wr = dev->attr.max_rqe;
 375         uresp.max_srq_wr = dev->attr.max_srq_wr;
 376         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 377         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 378         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 379         uresp.max_cqes = QEDR_MAX_CQES;
 380
 381         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 382         if (rc)
 383                 goto err;
 384
 385         ctx->dev = dev;
 386
 387         rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
 388         if (rc)
 389                 goto err;
 390
 391         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 392                  &ctx->ibucontext);
 393         return &ctx->ibucontext;
 394
 395 err:
 396         kfree(ctx);
 397         return ERR_PTR(rc);
 398 }
 399
 400 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 401 {
 402         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 403         struct qedr_mm *mm, *tmp;
 404         int status = 0;
 405
 406         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 407                  uctx);
 408         uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
 409
 410         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 411                 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 412                          "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 413                          mm->key.phy_addr, mm->key.len, uctx);
 414                 list_del(&mm->entry);
 415                 kfree(mm);
 416         }
 417
 418         kfree(uctx);
 419         return status;
 420 }
 421
 422 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 423 {
 424         struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
 425         struct qedr_dev *dev = get_qedr_dev(context->device);
 426         unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
 427         u64 unmapped_db = dev->db_phys_addr;
 428         unsigned long len = (vma->vm_end - vma->vm_start);
 429         int rc = 0;
 430         bool found;
 431
 432         DP_DEBUG(dev, QEDR_MSG_INIT,
 433                  "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
 434                  vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
 435         if (vma->vm_start & (PAGE_SIZE - 1)) {
 436                 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
 437                        vma->vm_start);
 438                 return -EINVAL;
 439         }
 440
 441         found = qedr_search_mmap(ucontext, vm_page, len);
 442         if (!found) {
 443                 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
 444                        vma->vm_pgoff);
 445                 return -EINVAL;
 446         }
 447
 448         DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
 449
 450         if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
 451                                                      dev->db_size))) {
 452                 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
 453                 if (vma->vm_flags & VM_READ) {
 454                         DP_ERR(dev, "Trying to map doorbell bar for read\n");
 455                         return -EPERM;
 456                 }
 457
 458                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 459
 460                 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 461                                         PAGE_SIZE, vma->vm_page_prot);
 462         } else {
 463                 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
 464                 rc = remap_pfn_range(vma, vma->vm_start,
 465                                      vma->vm_pgoff, len, vma->vm_page_prot);
 466         }
 467         DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
 468         return rc;
 469 }
 470
 471 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
 472                             struct ib_ucontext *context, struct ib_udata *udata)
 473 {
 474         struct qedr_dev *dev = get_qedr_dev(ibdev);
 475         struct qedr_pd *pd;
 476         u16 pd_id;
 477         int rc;
 478
 479         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 480                  (udata && context) ? "User Lib" : "Kernel");
 481
 482         if (!dev->rdma_ctx) {
 483                 DP_ERR(dev, "invlaid RDMA context\n");
 484                 return ERR_PTR(-EINVAL);
 485         }
 486
 487         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 488         if (!pd)
 489                 return ERR_PTR(-ENOMEM);
 490
 491         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 492         if (rc)
 493                 goto err;
 494
 495         pd->pd_id = pd_id;
 496
 497         if (udata && context) {
 498                 struct qedr_alloc_pd_uresp uresp;
 499
 500                 uresp.pd_id = pd_id;
 501
 502                 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 503                 if (rc) {
 504                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 505                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 506                         goto err;
 507                 }
 508
 509                 pd->uctx = get_qedr_ucontext(context);
 510                 pd->uctx->pd = pd;
 511         }
 512
 513         return &pd->ibpd;
 514
 515 err:
 516         kfree(pd);
 517         return ERR_PTR(rc);
 518 }
 519
 520 int qedr_dealloc_pd(struct ib_pd *ibpd)
 521 {
 522         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 523         struct qedr_pd *pd = get_qedr_pd(ibpd);
 524
 525         if (!pd) {
 526                 pr_err("Invalid PD received in dealloc_pd\n");
 527                 return -EINVAL;
 528         }
 529
 530         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 531         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 532
 533         kfree(pd);
 534
 535         return 0;
 536 }
 537
 538 static void qedr_free_pbl(struct qedr_dev *dev,
 539                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 540 {
 541         struct pci_dev *pdev = dev->pdev;
 542         int i;
 543
 544         for (i = 0; i < pbl_info->num_pbls; i++) {
 545                 if (!pbl[i].va)
 546                         continue;
 547                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 548                                   pbl[i].va, pbl[i].pa);
 549         }
 550
 551         kfree(pbl);
 552 }
 553
 554 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 555 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 556
 557 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 558 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 559 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 560
 561 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 562                                            struct qedr_pbl_info *pbl_info,
 563                                            gfp_t flags)
 564 {
 565         struct pci_dev *pdev = dev->pdev;
 566         struct qedr_pbl *pbl_table;
 567         dma_addr_t *pbl_main_tbl;
 568         dma_addr_t pa;
 569         void *va;
 570         int i;
 571
 572         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 573         if (!pbl_table)
 574                 return ERR_PTR(-ENOMEM);
 575
 576         for (i = 0; i < pbl_info->num_pbls; i++) {
 577                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
 578                                         &pa, flags);
 579                 if (!va)
 580                         goto err;
 581
 582                 memset(va, 0, pbl_info->pbl_size);
 583                 pbl_table[i].va = va;
 584                 pbl_table[i].pa = pa;
 585         }
 586
 587         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 588          * the first one with physical pointers to all of the rest
 589          */
 590         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 591         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 592                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 593
 594         return pbl_table;
 595
 596 err:
 597         for (i--; i >= 0; i--)
 598                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 599                                   pbl_table[i].va, pbl_table[i].pa);
 600
 601         qedr_free_pbl(dev, pbl_info, pbl_table);
 602
 603         return ERR_PTR(-ENOMEM);
 604 }
 605
 606 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 607                                 struct qedr_pbl_info *pbl_info,
 608                                 u32 num_pbes, int two_layer_capable)
 609 {
 610         u32 pbl_capacity;
 611         u32 pbl_size;
 612         u32 num_pbls;
 613
 614         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 615                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 616                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 617                                num_pbes);
 618                         return -EINVAL;
 619                 }
 620
 621                 /* calculate required pbl page size */
 622                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 623                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 624                                NUM_PBES_ON_PAGE(pbl_size);
 625
 626                 while (pbl_capacity < num_pbes) {
 627                         pbl_size *= 2;
 628                         pbl_capacity = pbl_size / sizeof(u64);
 629                         pbl_capacity = pbl_capacity * pbl_capacity;
 630                 }
 631
 632                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 633                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 634                 pbl_info->two_layered = true;
 635         } else {
 636                 /* One layered PBL */
 637                 num_pbls = 1;
 638                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 639                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 640                 pbl_info->two_layered = false;
 641         }
 642
 643         pbl_info->num_pbls = num_pbls;
 644         pbl_info->pbl_size = pbl_size;
 645         pbl_info->num_pbes = num_pbes;
 646
 647         DP_DEBUG(dev, QEDR_MSG_MR,
 648                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 649                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 650
 651         return 0;
 652 }
 653
 654 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 655                                struct qedr_pbl *pbl,
 656                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 657 {
 658         int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
 659         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 660         struct qedr_pbl *pbl_tbl;
 661         struct scatterlist *sg;
 662         struct regpair *pbe;
 663         u64 pg_addr;
 664         int entry;
 665
 666         if (!pbl_info->num_pbes)
 667                 return;
 668
 669         /* If we have a two layered pbl, the first pbl points to the rest
 670          * of the pbls and the first entry lays on the second pbl in the table
 671          */
 672         if (pbl_info->two_layered)
 673                 pbl_tbl = &pbl[1];
 674         else
 675                 pbl_tbl = pbl;
 676
 677         pbe = (struct regpair *)pbl_tbl->va;
 678         if (!pbe) {
 679                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 680                 return;
 681         }
 682
 683         pbe_cnt = 0;
 684
 685         shift = umem->page_shift;
 686
 687         fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
 688
 689         for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
 690                 pages = sg_dma_len(sg) >> shift;
 691                 pg_addr = sg_dma_address(sg);
 692                 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
 693                         for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 694                                 pbe->lo = cpu_to_le32(pg_addr);
 695                                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 696
 697                                 pg_addr += BIT(pg_shift);
 698                                 pbe_cnt++;
 699                                 total_num_pbes++;
 700                                 pbe++;
 701
 702                                 if (total_num_pbes == pbl_info->num_pbes)
 703                                         return;
 704
 705                                 /* If the given pbl is full storing the pbes,
 706                                  * move to next pbl.
 707                                  */
 708                                 if (pbe_cnt ==
 709                                     (pbl_info->pbl_size / sizeof(u64))) {
 710                                         pbl_tbl++;
 711                                         pbe = (struct regpair *)pbl_tbl->va;
 712                                         pbe_cnt = 0;
 713                                 }
 714
 715                                 fw_pg_cnt++;
 716                         }
 717                 }
 718         }
 719 }
 720
 721 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 722                               struct qedr_cq *cq, struct ib_udata *udata)
 723 {
 724         struct qedr_create_cq_uresp uresp;
 725         int rc;
 726
 727         memset(&uresp, 0, sizeof(uresp));
 728
 729         uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 730         uresp.icid = cq->icid;
 731
 732         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 733         if (rc)
 734                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 735
 736         return rc;
 737 }
 738
 739 static void consume_cqe(struct qedr_cq *cq)
 740 {
 741         if (cq->latest_cqe == cq->toggle_cqe)
 742                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 743
 744         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 745 }
 746
 747 static inline int qedr_align_cq_entries(int entries)
 748 {
 749         u64 size, aligned_size;
 750
 751         /* We allocate an extra entry that we don't report to the FW. */
 752         size = (entries + 1) * QEDR_CQE_SIZE;
 753         aligned_size = ALIGN(size, PAGE_SIZE);
 754
 755         return aligned_size / QEDR_CQE_SIZE;
 756 }
 757
 758 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
 759                                        struct qedr_dev *dev,
 760                                        struct qedr_userq *q,
 761                                        u64 buf_addr, size_t buf_len,
 762                                        int access, int dmasync)
 763 {
 764         u32 fw_pages;
 765         int rc;
 766
 767         q->buf_addr = buf_addr;
 768         q->buf_len = buf_len;
 769         q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
 770         if (IS_ERR(q->umem)) {
 771                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 772                        PTR_ERR(q->umem));
 773                 return PTR_ERR(q->umem);
 774         }
 775
 776         fw_pages = ib_umem_page_count(q->umem) <<
 777             (q->umem->page_shift - FW_PAGE_SHIFT);
 778
 779         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 780         if (rc)
 781                 goto err0;
 782
 783         q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 784         if (IS_ERR(q->pbl_tbl)) {
 785                 rc = PTR_ERR(q->pbl_tbl);
 786                 goto err0;
 787         }
 788
 789                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 790                                    FW_PAGE_SHIFT);
 791
 792         return 0;
 793
 794 err0:
 795         ib_umem_release(q->umem);
 796
 797         return rc;
 798 }
 799
 800 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 801                                        struct qedr_ucontext *ctx,
 802                                        struct qedr_dev *dev, int vector,
 803                                        int chain_entries, int page_cnt,
 804                                        u64 pbl_ptr,
 805                                        struct qed_rdma_create_cq_in_params
 806                                        *params)
 807 {
 808         memset(params, 0, sizeof(*params));
 809         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 810         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 811         params->cnq_id = vector;
 812         params->cq_size = chain_entries - 1;
 813         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 814         params->pbl_num_pages = page_cnt;
 815         params->pbl_ptr = pbl_ptr;
 816         params->pbl_two_level = 0;
 817 }
 818
 819 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 820 {
 821         /* Flush data before signalling doorbell */
 822         wmb();
 823         cq->db.data.agg_flags = flags;
 824         cq->db.data.value = cpu_to_le32(cons);
 825         writeq(cq->db.raw, cq->db_addr);
 826
 827         /* Make sure write would stick */
 828         mmiowb();
 829 }
 830
 831 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 832 {
 833         struct qedr_cq *cq = get_qedr_cq(ibcq);
 834         unsigned long sflags;
 835         struct qedr_dev *dev;
 836
 837         dev = get_qedr_dev(ibcq->device);
 838
 839         if (cq->destroyed) {
 840                 DP_ERR(dev,
 841                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 842                        cq, cq->icid);
 843                 return -EINVAL;
 844         }
 845
 846
 847         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 848                 return 0;
 849
 850         spin_lock_irqsave(&cq->cq_lock, sflags);
 851
 852         cq->arm_flags = 0;
 853
 854         if (flags & IB_CQ_SOLICITED)
 855                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 856
 857         if (flags & IB_CQ_NEXT_COMP)
 858                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 859
 860         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 861
 862         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 863
 864         return 0;
 865 }
 866
 867 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 868                              const struct ib_cq_init_attr *attr,
 869                              struct ib_ucontext *ib_ctx, struct ib_udata *udata)
 870 {
 871         struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
 872         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 873         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 874         struct qedr_dev *dev = get_qedr_dev(ibdev);
 875         struct qed_rdma_create_cq_in_params params;
 876         struct qedr_create_cq_ureq ureq;
 877         int vector = attr->comp_vector;
 878         int entries = attr->cqe;
 879         struct qedr_cq *cq;
 880         int chain_entries;
 881         int page_cnt;
 882         u64 pbl_ptr;
 883         u16 icid;
 884         int rc;
 885
 886         DP_DEBUG(dev, QEDR_MSG_INIT,
 887                  "create_cq: called from %s. entries=%d, vector=%d\n",
 888                  udata ? "User Lib" : "Kernel", entries, vector);
 889
 890         if (entries > QEDR_MAX_CQES) {
 891                 DP_ERR(dev,
 892                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 893                        entries, QEDR_MAX_CQES);
 894                 return ERR_PTR(-EINVAL);
 895         }
 896
 897         chain_entries = qedr_align_cq_entries(entries);
 898         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 899
 900         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 901         if (!cq)
 902                 return ERR_PTR(-ENOMEM);
 903
 904         if (udata) {
 905                 memset(&ureq, 0, sizeof(ureq));
 906                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
 907                         DP_ERR(dev,
 908                                "create cq: problem copying data from user space\n");
 909                         goto err0;
 910                 }
 911
 912                 if (!ureq.len) {
 913                         DP_ERR(dev,
 914                                "create cq: cannot create a cq with 0 entries\n");
 915                         goto err0;
 916                 }
 917
 918                 cq->cq_type = QEDR_CQ_TYPE_USER;
 919
 920                 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
 921                                           ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
 922                 if (rc)
 923                         goto err0;
 924
 925                 pbl_ptr = cq->q.pbl_tbl->pa;
 926                 page_cnt = cq->q.pbl_info.num_pbes;
 927
 928                 cq->ibcq.cqe = chain_entries;
 929         } else {
 930                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 931
 932                 rc = dev->ops->common->chain_alloc(dev->cdev,
 933                                                    QED_CHAIN_USE_TO_CONSUME,
 934                                                    QED_CHAIN_MODE_PBL,
 935                                                    QED_CHAIN_CNT_TYPE_U32,
 936                                                    chain_entries,
 937                                                    sizeof(union rdma_cqe),
 938                                                    &cq->pbl);
 939                 if (rc)
 940                         goto err1;
 941
 942                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 943                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 944                 cq->ibcq.cqe = cq->pbl.capacity;
 945         }
 946
 947         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 948                             pbl_ptr, &params);
 949
 950         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 951         if (rc)
 952                 goto err2;
 953
 954         cq->icid = icid;
 955         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 956         spin_lock_init(&cq->cq_lock);
 957
 958         if (ib_ctx) {
 959                 rc = qedr_copy_cq_uresp(dev, cq, udata);
 960                 if (rc)
 961                         goto err3;
 962         } else {
 963                 /* Generate doorbell address. */
 964                 cq->db_addr = dev->db_addr +
 965                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 966                 cq->db.data.icid = cq->icid;
 967                 cq->db.data.params = DB_AGG_CMD_SET <<
 968                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
 969
 970                 /* point to the very last element, passing it we will toggle */
 971                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
 972                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 973                 cq->latest_cqe = NULL;
 974                 consume_cqe(cq);
 975                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 976         }
 977
 978         DP_DEBUG(dev, QEDR_MSG_CQ,
 979                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
 980                  cq->icid, cq, params.cq_size);
 981
 982         return &cq->ibcq;
 983
 984 err3:
 985         destroy_iparams.icid = cq->icid;
 986         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
 987                                   &destroy_oparams);
 988 err2:
 989         if (udata)
 990                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 991         else
 992                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 993 err1:
 994         if (udata)
 995                 ib_umem_release(cq->q.umem);
 996 err0:
 997         kfree(cq);
 998         return ERR_PTR(-EINVAL);
 999 }
1000
1001 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1002 {
1003         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1004         struct qedr_cq *cq = get_qedr_cq(ibcq);
1005
1006         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1007
1008         return 0;
1009 }
1010
1011 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1012 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1013
1014 int qedr_destroy_cq(struct ib_cq *ibcq)
1015 {
1016         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1017         struct qed_rdma_destroy_cq_out_params oparams;
1018         struct qed_rdma_destroy_cq_in_params iparams;
1019         struct qedr_cq *cq = get_qedr_cq(ibcq);
1020         int iter;
1021         int rc;
1022
1023         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1024
1025         cq->destroyed = 1;
1026
1027         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1028         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1029                 goto done;
1030
1031         iparams.icid = cq->icid;
1032         rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1033         if (rc)
1034                 return rc;
1035
1036         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1037
1038         if (ibcq->uobject && ibcq->uobject->context) {
1039                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1040                 ib_umem_release(cq->q.umem);
1041         }
1042
1043         /* We don't want the IRQ handler to handle a non-existing CQ so we
1044          * wait until all CNQ interrupts, if any, are received. This will always
1045          * happen and will always happen very fast. If not, then a serious error
1046          * has occured. That is why we can use a long delay.
1047          * We spin for a short time so we don’t lose time on context switching
1048          * in case all the completions are handled in that span. Otherwise
1049          * we sleep for a while and check again. Since the CNQ may be
1050          * associated with (only) the current CPU we use msleep to allow the
1051          * current CPU to be freed.
1052          * The CNQ notification is increased in qedr_irq_handler().
1053          */
1054         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1055         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1056                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1057                 iter--;
1058         }
1059
1060         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1061         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1062                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1063                 iter--;
1064         }
1065
1066         if (oparams.num_cq_notif != cq->cnq_notif)
1067                 goto err;
1068
1069         /* Note that we don't need to have explicit code to wait for the
1070          * completion of the event handler because it is invoked from the EQ.
1071          * Since the destroy CQ ramrod has also been received on the EQ we can
1072          * be certain that there's no event handler in process.
1073          */
1074 done:
1075         cq->sig = ~cq->sig;
1076
1077         kfree(cq);
1078
1079         return 0;
1080
1081 err:
1082         DP_ERR(dev,
1083                "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1084                cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1085
1086         return -EINVAL;
1087 }
1088
1089 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1090                                           struct ib_qp_attr *attr,
1091                                           int attr_mask,
1092                                           struct qed_rdma_modify_qp_in_params
1093                                           *qp_params)
1094 {
1095         enum rdma_network_type nw_type;
1096         struct ib_gid_attr gid_attr;
1097         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1098         union ib_gid gid;
1099         u32 ipv4_addr;
1100         int rc = 0;
1101         int i;
1102
1103         rc = ib_get_cached_gid(ibqp->device,
1104                                rdma_ah_get_port_num(&attr->ah_attr),
1105                                grh->sgid_index, &gid, &gid_attr);
1106         if (rc)
1107                 return rc;
1108
1109         if (!memcmp(&gid, &zgid, sizeof(gid)))
1110                 return -ENOENT;
1111
1112         if (gid_attr.ndev) {
1113                 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1114
1115                 dev_put(gid_attr.ndev);
1116                 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1117                 switch (nw_type) {
1118                 case RDMA_NETWORK_IPV6:
1119                         memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1120                                sizeof(qp_params->sgid));
1121                         memcpy(&qp_params->dgid.bytes[0],
1122                                &grh->dgid,
1123                                sizeof(qp_params->dgid));
1124                         qp_params->roce_mode = ROCE_V2_IPV6;
1125                         SET_FIELD(qp_params->modify_flags,
1126                                   QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1127                         break;
1128                 case RDMA_NETWORK_IB:
1129                         memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1130                                sizeof(qp_params->sgid));
1131                         memcpy(&qp_params->dgid.bytes[0],
1132                                &grh->dgid,
1133                                sizeof(qp_params->dgid));
1134                         qp_params->roce_mode = ROCE_V1;
1135                         break;
1136                 case RDMA_NETWORK_IPV4:
1137                         memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1138                         memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1139                         ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1140                         qp_params->sgid.ipv4_addr = ipv4_addr;
1141                         ipv4_addr =
1142                             qedr_get_ipv4_from_gid(grh->dgid.raw);
1143                         qp_params->dgid.ipv4_addr = ipv4_addr;
1144                         SET_FIELD(qp_params->modify_flags,
1145                                   QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1146                         qp_params->roce_mode = ROCE_V2_IPV4;
1147                         break;
1148                 }
1149         }
1150
1151         for (i = 0; i < 4; i++) {
1152                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1153                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1154         }
1155
1156         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1157                 qp_params->vlan_id = 0;
1158
1159         return 0;
1160 }
1161
1162 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1163                                struct ib_qp_init_attr *attrs)
1164 {
1165         struct qedr_device_attr *qattr = &dev->attr;
1166
1167         /* QP0... attrs->qp_type == IB_QPT_GSI */
1168         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1169                 DP_DEBUG(dev, QEDR_MSG_QP,
1170                          "create qp: unsupported qp type=0x%x requested\n",
1171                          attrs->qp_type);
1172                 return -EINVAL;
1173         }
1174
1175         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1176                 DP_ERR(dev,
1177                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1178                        attrs->cap.max_send_wr, qattr->max_sqe);
1179                 return -EINVAL;
1180         }
1181
1182         if (attrs->cap.max_inline_data > qattr->max_inline) {
1183                 DP_ERR(dev,
1184                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1185                        attrs->cap.max_inline_data, qattr->max_inline);
1186                 return -EINVAL;
1187         }
1188
1189         if (attrs->cap.max_send_sge > qattr->max_sge) {
1190                 DP_ERR(dev,
1191                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1192                        attrs->cap.max_send_sge, qattr->max_sge);
1193                 return -EINVAL;
1194         }
1195
1196         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1197                 DP_ERR(dev,
1198                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1199                        attrs->cap.max_recv_sge, qattr->max_sge);
1200                 return -EINVAL;
1201         }
1202
1203         /* Unprivileged user space cannot create special QP */
1204         if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1205                 DP_ERR(dev,
1206                        "create qp: userspace can't create special QPs of type=0x%x\n",
1207                        attrs->qp_type);
1208                 return -EINVAL;
1209         }
1210
1211         return 0;
1212 }
1213
1214 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1215                                struct qedr_qp *qp)
1216 {
1217         uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1218         uresp->rq_icid = qp->icid;
1219 }
1220
1221 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1222                                struct qedr_qp *qp)
1223 {
1224         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1225         uresp->sq_icid = qp->icid + 1;
1226 }
1227
1228 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1229                               struct qedr_qp *qp, struct ib_udata *udata)
1230 {
1231         struct qedr_create_qp_uresp uresp;
1232         int rc;
1233
1234         memset(&uresp, 0, sizeof(uresp));
1235         qedr_copy_sq_uresp(&uresp, qp);
1236         qedr_copy_rq_uresp(&uresp, qp);
1237
1238         uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1239         uresp.qp_id = qp->qp_id;
1240
1241         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1242         if (rc)
1243                 DP_ERR(dev,
1244                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1245                        qp->icid);
1246
1247         return rc;
1248 }
1249
1250 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1251                                       struct qedr_qp *qp,
1252                                       struct qedr_pd *pd,
1253                                       struct ib_qp_init_attr *attrs)
1254 {
1255         spin_lock_init(&qp->q_lock);
1256         qp->pd = pd;
1257         qp->qp_type = attrs->qp_type;
1258         qp->max_inline_data = attrs->cap.max_inline_data;
1259         qp->sq.max_sges = attrs->cap.max_send_sge;
1260         qp->state = QED_ROCE_QP_STATE_RESET;
1261         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1262         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1263         qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1264         qp->dev = dev;
1265         qp->rq.max_sges = attrs->cap.max_recv_sge;
1266
1267         DP_DEBUG(dev, QEDR_MSG_QP,
1268                  "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1269                  qp->rq.max_sges, qp->rq_cq->icid);
1270         DP_DEBUG(dev, QEDR_MSG_QP,
1271                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1272                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1273                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1274         DP_DEBUG(dev, QEDR_MSG_QP,
1275                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1276                  qp->sq.max_sges, qp->sq_cq->icid);
1277 }
1278
1279 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1280 {
1281         qp->sq.db = dev->db_addr +
1282                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1283         qp->sq.db_data.data.icid = qp->icid + 1;
1284         qp->rq.db = dev->db_addr +
1285                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1286         qp->rq.db_data.data.icid = qp->icid;
1287 }
1288
1289 static inline void
1290 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1291                               struct qedr_pd *pd,
1292                               struct qedr_qp *qp,
1293                               struct ib_qp_init_attr *attrs,
1294                               bool fmr_and_reserved_lkey,
1295                               struct qed_rdma_create_qp_in_params *params)
1296 {
1297         /* QP handle to be written in an async event */
1298         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1299         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1300
1301         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1302         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1303         params->pd = pd->pd_id;
1304         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1305         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1306         params->stats_queue = 0;
1307         params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1308         params->srq_id = 0;
1309         params->use_srq = false;
1310 }
1311
1312 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1313 {
1314         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1315                  "qp=%p. "
1316                  "sq_addr=0x%llx, "
1317                  "sq_len=%zd, "
1318                  "rq_addr=0x%llx, "
1319                  "rq_len=%zd"
1320                  "\n",
1321                  qp,
1322                  qp->usq.buf_addr,
1323                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1324 }
1325
1326 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1327 {
1328         if (qp->usq.umem)
1329                 ib_umem_release(qp->usq.umem);
1330         qp->usq.umem = NULL;
1331
1332         if (qp->urq.umem)
1333                 ib_umem_release(qp->urq.umem);
1334         qp->urq.umem = NULL;
1335 }
1336
1337 static int qedr_create_user_qp(struct qedr_dev *dev,
1338                                struct qedr_qp *qp,
1339                                struct ib_pd *ibpd,
1340                                struct ib_udata *udata,
1341                                struct ib_qp_init_attr *attrs)
1342 {
1343         struct qed_rdma_create_qp_in_params in_params;
1344         struct qed_rdma_create_qp_out_params out_params;
1345         struct qedr_pd *pd = get_qedr_pd(ibpd);
1346         struct ib_ucontext *ib_ctx = NULL;
1347         struct qedr_ucontext *ctx = NULL;
1348         struct qedr_create_qp_ureq ureq;
1349         int rc = -EINVAL;
1350
1351         ib_ctx = ibpd->uobject->context;
1352         ctx = get_qedr_ucontext(ib_ctx);
1353
1354         memset(&ureq, 0, sizeof(ureq));
1355         rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1356         if (rc) {
1357                 DP_ERR(dev, "Problem copying data from user space\n");
1358                 return rc;
1359         }
1360
1361         /* SQ - read access only (0), dma sync not required (0) */
1362         rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1363                                   ureq.sq_len, 0, 0);
1364         if (rc)
1365                 return rc;
1366
1367         /* RQ - read access only (0), dma sync not required (0) */
1368         rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1369                                   ureq.rq_len, 0, 0);
1370
1371         if (rc)
1372                 return rc;
1373
1374         memset(&in_params, 0, sizeof(in_params));
1375         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1376         in_params.qp_handle_lo = ureq.qp_handle_lo;
1377         in_params.qp_handle_hi = ureq.qp_handle_hi;
1378         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1379         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1380         in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1381         in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1382
1383         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1384                                               &in_params, &out_params);
1385
1386         if (!qp->qed_qp) {
1387                 rc = -ENOMEM;
1388                 goto err1;
1389         }
1390
1391         qp->qp_id = out_params.qp_id;
1392         qp->icid = out_params.icid;
1393
1394         rc = qedr_copy_qp_uresp(dev, qp, udata);
1395         if (rc)
1396                 goto err;
1397
1398         qedr_qp_user_print(dev, qp);
1399
1400         return 0;
1401 err:
1402         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1403         if (rc)
1404                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1405
1406 err1:
1407         qedr_cleanup_user(dev, qp);
1408         return rc;
1409 }
1410
1411 static int
1412 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1413                            struct qedr_qp *qp,
1414                            struct qed_rdma_create_qp_in_params *in_params,
1415                            u32 n_sq_elems, u32 n_rq_elems)
1416 {
1417         struct qed_rdma_create_qp_out_params out_params;
1418         int rc;
1419
1420         rc = dev->ops->common->chain_alloc(dev->cdev,
1421                                            QED_CHAIN_USE_TO_PRODUCE,
1422                                            QED_CHAIN_MODE_PBL,
1423                                            QED_CHAIN_CNT_TYPE_U32,
1424                                            n_sq_elems,
1425                                            QEDR_SQE_ELEMENT_SIZE,
1426                                            &qp->sq.pbl);
1427
1428         if (rc)
1429                 return rc;
1430
1431         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1432         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1433
1434         rc = dev->ops->common->chain_alloc(dev->cdev,
1435                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1436                                            QED_CHAIN_MODE_PBL,
1437                                            QED_CHAIN_CNT_TYPE_U32,
1438                                            n_rq_elems,
1439                                            QEDR_RQE_ELEMENT_SIZE,
1440                                            &qp->rq.pbl);
1441         if (rc)
1442                 return rc;
1443
1444         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1445         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1446
1447         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1448                                               in_params, &out_params);
1449
1450         if (!qp->qed_qp)
1451                 return -EINVAL;
1452
1453         qp->qp_id = out_params.qp_id;
1454         qp->icid = out_params.icid;
1455
1456         qedr_set_roce_db_info(dev, qp);
1457
1458         return 0;
1459 }
1460
1461 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1462 {
1463         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1464         kfree(qp->wqe_wr_id);
1465
1466         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1467         kfree(qp->rqe_wr_id);
1468 }
1469
1470 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1471                                  struct qedr_qp *qp,
1472                                  struct ib_pd *ibpd,
1473                                  struct ib_qp_init_attr *attrs)
1474 {
1475         struct qed_rdma_create_qp_in_params in_params;
1476         struct qedr_pd *pd = get_qedr_pd(ibpd);
1477         int rc = -EINVAL;
1478         u32 n_rq_elems;
1479         u32 n_sq_elems;
1480         u32 n_sq_entries;
1481
1482         memset(&in_params, 0, sizeof(in_params));
1483
1484         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1485          * the ring. The ring should allow at least a single WR, even if the
1486          * user requested none, due to allocation issues.
1487          * We should add an extra WR since the prod and cons indices of
1488          * wqe_wr_id are managed in such a way that the WQ is considered full
1489          * when (prod+1)%max_wr==cons. We currently don't do that because we
1490          * double the number of entries due an iSER issue that pushes far more
1491          * WRs than indicated. If we decline its ib_post_send() then we get
1492          * error prints in the dmesg we'd like to avoid.
1493          */
1494         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1495                               dev->attr.max_sqe);
1496
1497         qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1498                                 GFP_KERNEL);
1499         if (!qp->wqe_wr_id) {
1500                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1501                 return -ENOMEM;
1502         }
1503
1504         /* QP handle to be written in CQE */
1505         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1506         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1507
1508         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1509          * the ring. There ring should allow at least a single WR, even if the
1510          * user requested none, due to allocation issues.
1511          */
1512         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1513
1514         /* Allocate driver internal RQ array */
1515         qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1516                                 GFP_KERNEL);
1517         if (!qp->rqe_wr_id) {
1518                 DP_ERR(dev,
1519                        "create qp: failed RQ shadow memory allocation\n");
1520                 kfree(qp->wqe_wr_id);
1521                 return -ENOMEM;
1522         }
1523
1524         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1525
1526         n_sq_entries = attrs->cap.max_send_wr;
1527         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1528         n_sq_entries = max_t(u32, n_sq_entries, 1);
1529         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1530
1531         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1532
1533         rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1534                                         n_sq_elems, n_rq_elems);
1535         if (rc)
1536                 qedr_cleanup_kernel(dev, qp);
1537
1538         return rc;
1539 }
1540
1541 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1542                              struct ib_qp_init_attr *attrs,
1543                              struct ib_udata *udata)
1544 {
1545         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1546         struct qedr_pd *pd = get_qedr_pd(ibpd);
1547         struct qedr_qp *qp;
1548         struct ib_qp *ibqp;
1549         int rc = 0;
1550
1551         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1552                  udata ? "user library" : "kernel", pd);
1553
1554         rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1555         if (rc)
1556                 return ERR_PTR(rc);
1557
1558         if (attrs->srq)
1559                 return ERR_PTR(-EINVAL);
1560
1561         DP_DEBUG(dev, QEDR_MSG_QP,
1562                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1563                  udata ? "user library" : "kernel", attrs->event_handler, pd,
1564                  get_qedr_cq(attrs->send_cq),
1565                  get_qedr_cq(attrs->send_cq)->icid,
1566                  get_qedr_cq(attrs->recv_cq),
1567                  get_qedr_cq(attrs->recv_cq)->icid);
1568
1569         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1570         if (!qp) {
1571                 DP_ERR(dev, "create qp: failed allocating memory\n");
1572                 return ERR_PTR(-ENOMEM);
1573         }
1574
1575         qedr_set_common_qp_params(dev, qp, pd, attrs);
1576
1577         if (attrs->qp_type == IB_QPT_GSI) {
1578                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1579                 if (IS_ERR(ibqp))
1580                         kfree(qp);
1581                 return ibqp;
1582         }
1583
1584         if (udata)
1585                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1586         else
1587                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1588
1589         if (rc)
1590                 goto err;
1591
1592         qp->ibqp.qp_num = qp->qp_id;
1593
1594         return &qp->ibqp;
1595
1596 err:
1597         kfree(qp);
1598
1599         return ERR_PTR(-EFAULT);
1600 }
1601
1602 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1603 {
1604         switch (qp_state) {
1605         case QED_ROCE_QP_STATE_RESET:
1606                 return IB_QPS_RESET;
1607         case QED_ROCE_QP_STATE_INIT:
1608                 return IB_QPS_INIT;
1609         case QED_ROCE_QP_STATE_RTR:
1610                 return IB_QPS_RTR;
1611         case QED_ROCE_QP_STATE_RTS:
1612                 return IB_QPS_RTS;
1613         case QED_ROCE_QP_STATE_SQD:
1614                 return IB_QPS_SQD;
1615         case QED_ROCE_QP_STATE_ERR:
1616                 return IB_QPS_ERR;
1617         case QED_ROCE_QP_STATE_SQE:
1618                 return IB_QPS_SQE;
1619         }
1620         return IB_QPS_ERR;
1621 }
1622
1623 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1624                                         enum ib_qp_state qp_state)
1625 {
1626         switch (qp_state) {
1627         case IB_QPS_RESET:
1628                 return QED_ROCE_QP_STATE_RESET;
1629         case IB_QPS_INIT:
1630                 return QED_ROCE_QP_STATE_INIT;
1631         case IB_QPS_RTR:
1632                 return QED_ROCE_QP_STATE_RTR;
1633         case IB_QPS_RTS:
1634                 return QED_ROCE_QP_STATE_RTS;
1635         case IB_QPS_SQD:
1636                 return QED_ROCE_QP_STATE_SQD;
1637         case IB_QPS_ERR:
1638                 return QED_ROCE_QP_STATE_ERR;
1639         default:
1640                 return QED_ROCE_QP_STATE_ERR;
1641         }
1642 }
1643
1644 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1645 {
1646         qed_chain_reset(&qph->pbl);
1647         qph->prod = 0;
1648         qph->cons = 0;
1649         qph->wqe_cons = 0;
1650         qph->db_data.data.value = cpu_to_le16(0);
1651 }
1652
1653 static int qedr_update_qp_state(struct qedr_dev *dev,
1654                                 struct qedr_qp *qp,
1655                                 enum qed_roce_qp_state new_state)
1656 {
1657         int status = 0;
1658
1659         if (new_state == qp->state)
1660                 return 0;
1661
1662         switch (qp->state) {
1663         case QED_ROCE_QP_STATE_RESET:
1664                 switch (new_state) {
1665                 case QED_ROCE_QP_STATE_INIT:
1666                         qp->prev_wqe_size = 0;
1667                         qedr_reset_qp_hwq_info(&qp->sq);
1668                         qedr_reset_qp_hwq_info(&qp->rq);
1669                         break;
1670                 default:
1671                         status = -EINVAL;
1672                         break;
1673                 };
1674                 break;
1675         case QED_ROCE_QP_STATE_INIT:
1676                 switch (new_state) {
1677                 case QED_ROCE_QP_STATE_RTR:
1678                         /* Update doorbell (in case post_recv was
1679                          * done before move to RTR)
1680                          */
1681                         wmb();
1682                         writel(qp->rq.db_data.raw, qp->rq.db);
1683                         /* Make sure write takes effect */
1684                         mmiowb();
1685                         break;
1686                 case QED_ROCE_QP_STATE_ERR:
1687                         break;
1688                 default:
1689                         /* Invalid state change. */
1690                         status = -EINVAL;
1691                         break;
1692                 };
1693                 break;
1694         case QED_ROCE_QP_STATE_RTR:
1695                 /* RTR->XXX */
1696                 switch (new_state) {
1697                 case QED_ROCE_QP_STATE_RTS:
1698                         break;
1699                 case QED_ROCE_QP_STATE_ERR:
1700                         break;
1701                 default:
1702                         /* Invalid state change. */
1703                         status = -EINVAL;
1704                         break;
1705                 };
1706                 break;
1707         case QED_ROCE_QP_STATE_RTS:
1708                 /* RTS->XXX */
1709                 switch (new_state) {
1710                 case QED_ROCE_QP_STATE_SQD:
1711                         break;
1712                 case QED_ROCE_QP_STATE_ERR:
1713                         break;
1714                 default:
1715                         /* Invalid state change. */
1716                         status = -EINVAL;
1717                         break;
1718                 };
1719                 break;
1720         case QED_ROCE_QP_STATE_SQD:
1721                 /* SQD->XXX */
1722                 switch (new_state) {
1723                 case QED_ROCE_QP_STATE_RTS:
1724                 case QED_ROCE_QP_STATE_ERR:
1725                         break;
1726                 default:
1727                         /* Invalid state change. */
1728                         status = -EINVAL;
1729                         break;
1730                 };
1731                 break;
1732         case QED_ROCE_QP_STATE_ERR:
1733                 /* ERR->XXX */
1734                 switch (new_state) {
1735                 case QED_ROCE_QP_STATE_RESET:
1736                         if ((qp->rq.prod != qp->rq.cons) ||
1737                             (qp->sq.prod != qp->sq.cons)) {
1738                                 DP_NOTICE(dev,
1739                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1740                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
1741                                           qp->sq.cons);
1742                                 status = -EINVAL;
1743                         }
1744                         break;
1745                 default:
1746                         status = -EINVAL;
1747                         break;
1748                 };
1749                 break;
1750         default:
1751                 status = -EINVAL;
1752                 break;
1753         };
1754
1755         return status;
1756 }
1757
1758 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1759                    int attr_mask, struct ib_udata *udata)
1760 {
1761         struct qedr_qp *qp = get_qedr_qp(ibqp);
1762         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1763         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1764         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1765         enum ib_qp_state old_qp_state, new_qp_state;
1766         int rc = 0;
1767
1768         DP_DEBUG(dev, QEDR_MSG_QP,
1769                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1770                  attr->qp_state);
1771
1772         old_qp_state = qedr_get_ibqp_state(qp->state);
1773         if (attr_mask & IB_QP_STATE)
1774                 new_qp_state = attr->qp_state;
1775         else
1776                 new_qp_state = old_qp_state;
1777
1778         if (!ib_modify_qp_is_ok
1779             (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1780              IB_LINK_LAYER_ETHERNET)) {
1781                 DP_ERR(dev,
1782                        "modify qp: invalid attribute mask=0x%x specified for\n"
1783                        "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1784                        attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1785                        new_qp_state);
1786                 rc = -EINVAL;
1787                 goto err;
1788         }
1789
1790         /* Translate the masks... */
1791         if (attr_mask & IB_QP_STATE) {
1792                 SET_FIELD(qp_params.modify_flags,
1793                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1794                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1795         }
1796
1797         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1798                 qp_params.sqd_async = true;
1799
1800         if (attr_mask & IB_QP_PKEY_INDEX) {
1801                 SET_FIELD(qp_params.modify_flags,
1802                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1803                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1804                         rc = -EINVAL;
1805                         goto err;
1806                 }
1807
1808                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1809         }
1810
1811         if (attr_mask & IB_QP_QKEY)
1812                 qp->qkey = attr->qkey;
1813
1814         if (attr_mask & IB_QP_ACCESS_FLAGS) {
1815                 SET_FIELD(qp_params.modify_flags,
1816                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1817                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1818                                                   IB_ACCESS_REMOTE_READ;
1819                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1820                                                    IB_ACCESS_REMOTE_WRITE;
1821                 qp_params.incoming_atomic_en = attr->qp_access_flags &
1822                                                IB_ACCESS_REMOTE_ATOMIC;
1823         }
1824
1825         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1826                 if (attr_mask & IB_QP_PATH_MTU) {
1827                         if (attr->path_mtu < IB_MTU_256 ||
1828                             attr->path_mtu > IB_MTU_4096) {
1829                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1830                                 rc = -EINVAL;
1831                                 goto err;
1832                         }
1833                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1834                                       ib_mtu_enum_to_int(iboe_get_mtu
1835                                                          (dev->ndev->mtu)));
1836                 }
1837
1838                 if (!qp->mtu) {
1839                         qp->mtu =
1840                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1841                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1842                 }
1843
1844                 SET_FIELD(qp_params.modify_flags,
1845                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1846
1847                 qp_params.traffic_class_tos = grh->traffic_class;
1848                 qp_params.flow_label = grh->flow_label;
1849                 qp_params.hop_limit_ttl = grh->hop_limit;
1850
1851                 qp->sgid_idx = grh->sgid_index;
1852
1853                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1854                 if (rc) {
1855                         DP_ERR(dev,
1856                                "modify qp: problems with GID index %d (rc=%d)\n",
1857                                grh->sgid_index, rc);
1858                         return rc;
1859                 }
1860
1861                 rc = qedr_get_dmac(dev, &attr->ah_attr,
1862                                    qp_params.remote_mac_addr);
1863                 if (rc)
1864                         return rc;
1865
1866                 qp_params.use_local_mac = true;
1867                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1868
1869                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1870                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1871                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1872                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1873                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1874                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1875                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1876                          qp_params.remote_mac_addr);
1877
1878                 qp_params.mtu = qp->mtu;
1879                 qp_params.lb_indication = false;
1880         }
1881
1882         if (!qp_params.mtu) {
1883                 /* Stay with current MTU */
1884                 if (qp->mtu)
1885                         qp_params.mtu = qp->mtu;
1886                 else
1887                         qp_params.mtu =
1888                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1889         }
1890
1891         if (attr_mask & IB_QP_TIMEOUT) {
1892                 SET_FIELD(qp_params.modify_flags,
1893                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1894
1895                 qp_params.ack_timeout = attr->timeout;
1896                 if (attr->timeout) {
1897                         u32 temp;
1898
1899                         temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1900                         /* FW requires [msec] */
1901                         qp_params.ack_timeout = temp;
1902                 } else {
1903                         /* Infinite */
1904                         qp_params.ack_timeout = 0;
1905                 }
1906         }
1907         if (attr_mask & IB_QP_RETRY_CNT) {
1908                 SET_FIELD(qp_params.modify_flags,
1909                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1910                 qp_params.retry_cnt = attr->retry_cnt;
1911         }
1912
1913         if (attr_mask & IB_QP_RNR_RETRY) {
1914                 SET_FIELD(qp_params.modify_flags,
1915                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1916                 qp_params.rnr_retry_cnt = attr->rnr_retry;
1917         }
1918
1919         if (attr_mask & IB_QP_RQ_PSN) {
1920                 SET_FIELD(qp_params.modify_flags,
1921                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1922                 qp_params.rq_psn = attr->rq_psn;
1923                 qp->rq_psn = attr->rq_psn;
1924         }
1925
1926         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1927                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1928                         rc = -EINVAL;
1929                         DP_ERR(dev,
1930                                "unsupported max_rd_atomic=%d, supported=%d\n",
1931                                attr->max_rd_atomic,
1932                                dev->attr.max_qp_req_rd_atomic_resc);
1933                         goto err;
1934                 }
1935
1936                 SET_FIELD(qp_params.modify_flags,
1937                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1938                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1939         }
1940
1941         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1942                 SET_FIELD(qp_params.modify_flags,
1943                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1944                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1945         }
1946
1947         if (attr_mask & IB_QP_SQ_PSN) {
1948                 SET_FIELD(qp_params.modify_flags,
1949                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1950                 qp_params.sq_psn = attr->sq_psn;
1951                 qp->sq_psn = attr->sq_psn;
1952         }
1953
1954         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1955                 if (attr->max_dest_rd_atomic >
1956                     dev->attr.max_qp_resp_rd_atomic_resc) {
1957                         DP_ERR(dev,
1958                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1959                                attr->max_dest_rd_atomic,
1960                                dev->attr.max_qp_resp_rd_atomic_resc);
1961
1962                         rc = -EINVAL;
1963                         goto err;
1964                 }
1965
1966                 SET_FIELD(qp_params.modify_flags,
1967                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1968                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1969         }
1970
1971         if (attr_mask & IB_QP_DEST_QPN) {
1972                 SET_FIELD(qp_params.modify_flags,
1973                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1974
1975                 qp_params.dest_qp = attr->dest_qp_num;
1976                 qp->dest_qp_num = attr->dest_qp_num;
1977         }
1978
1979         if (qp->qp_type != IB_QPT_GSI)
1980                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1981                                               qp->qed_qp, &qp_params);
1982
1983         if (attr_mask & IB_QP_STATE) {
1984                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1985                         rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
1986                 qp->state = qp_params.new_state;
1987         }
1988
1989 err:
1990         return rc;
1991 }
1992
1993 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1994 {
1995         int ib_qp_acc_flags = 0;
1996
1997         if (params->incoming_rdma_write_en)
1998                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1999         if (params->incoming_rdma_read_en)
2000                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2001         if (params->incoming_atomic_en)
2002                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2003         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2004         return ib_qp_acc_flags;
2005 }
2006
2007 int qedr_query_qp(struct ib_qp *ibqp,
2008                   struct ib_qp_attr *qp_attr,
2009                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2010 {
2011         struct qed_rdma_query_qp_out_params params;
2012         struct qedr_qp *qp = get_qedr_qp(ibqp);
2013         struct qedr_dev *dev = qp->dev;
2014         int rc = 0;
2015
2016         memset(&params, 0, sizeof(params));
2017
2018         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2019         if (rc)
2020                 goto err;
2021
2022         memset(qp_attr, 0, sizeof(*qp_attr));
2023         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2024
2025         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2026         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2027         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2028         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2029         qp_attr->rq_psn = params.rq_psn;
2030         qp_attr->sq_psn = params.sq_psn;
2031         qp_attr->dest_qp_num = params.dest_qp;
2032
2033         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2034
2035         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2036         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2037         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2038         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2039         qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2040         qp_init_attr->cap = qp_attr->cap;
2041
2042         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2043         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2044                         params.flow_label, qp->sgid_idx,
2045                         params.hop_limit_ttl, params.traffic_class_tos);
2046         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2047         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2048         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2049         qp_attr->timeout = params.timeout;
2050         qp_attr->rnr_retry = params.rnr_retry;
2051         qp_attr->retry_cnt = params.retry_cnt;
2052         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2053         qp_attr->pkey_index = params.pkey_index;
2054         qp_attr->port_num = 1;
2055         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2056         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2057         qp_attr->alt_pkey_index = 0;
2058         qp_attr->alt_port_num = 0;
2059         qp_attr->alt_timeout = 0;
2060         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2061
2062         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2063         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2064         qp_attr->max_rd_atomic = params.max_rd_atomic;
2065         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2066
2067         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2068                  qp_attr->cap.max_inline_data);
2069
2070 err:
2071         return rc;
2072 }
2073
2074 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2075 {
2076         int rc = 0;
2077
2078         if (qp->qp_type != IB_QPT_GSI) {
2079                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2080                 if (rc)
2081                         return rc;
2082         }
2083
2084         if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2085                 qedr_cleanup_user(dev, qp);
2086         else
2087                 qedr_cleanup_kernel(dev, qp);
2088
2089         return 0;
2090 }
2091
2092 int qedr_destroy_qp(struct ib_qp *ibqp)
2093 {
2094         struct qedr_qp *qp = get_qedr_qp(ibqp);
2095         struct qedr_dev *dev = qp->dev;
2096         struct ib_qp_attr attr;
2097         int attr_mask = 0;
2098         int rc = 0;
2099
2100         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2101                  qp, qp->qp_type);
2102
2103         if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2104             (qp->state != QED_ROCE_QP_STATE_ERR) &&
2105             (qp->state != QED_ROCE_QP_STATE_INIT)) {
2106
2107                 attr.qp_state = IB_QPS_ERR;
2108                 attr_mask |= IB_QP_STATE;
2109
2110                 /* Change the QP state to ERROR */
2111                 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2112         }
2113
2114         if (qp->qp_type == IB_QPT_GSI)
2115                 qedr_destroy_gsi_qp(dev);
2116
2117         qedr_free_qp_resources(dev, qp);
2118
2119         kfree(qp);
2120
2121         return rc;
2122 }
2123
2124 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2125                              struct ib_udata *udata)
2126 {
2127         struct qedr_ah *ah;
2128
2129         ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2130         if (!ah)
2131                 return ERR_PTR(-ENOMEM);
2132
2133         ah->attr = *attr;
2134
2135         return &ah->ibah;
2136 }
2137
2138 int qedr_destroy_ah(struct ib_ah *ibah)
2139 {
2140         struct qedr_ah *ah = get_qedr_ah(ibah);
2141
2142         kfree(ah);
2143         return 0;
2144 }
2145
2146 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2147 {
2148         struct qedr_pbl *pbl, *tmp;
2149
2150         if (info->pbl_table)
2151                 list_add_tail(&info->pbl_table->list_entry,
2152                               &info->free_pbl_list);
2153
2154         if (!list_empty(&info->inuse_pbl_list))
2155                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2156
2157         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2158                 list_del(&pbl->list_entry);
2159                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2160         }
2161 }
2162
2163 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2164                         size_t page_list_len, bool two_layered)
2165 {
2166         struct qedr_pbl *tmp;
2167         int rc;
2168
2169         INIT_LIST_HEAD(&info->free_pbl_list);
2170         INIT_LIST_HEAD(&info->inuse_pbl_list);
2171
2172         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2173                                   page_list_len, two_layered);
2174         if (rc)
2175                 goto done;
2176
2177         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2178         if (IS_ERR(info->pbl_table)) {
2179                 rc = PTR_ERR(info->pbl_table);
2180                 goto done;
2181         }
2182
2183         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2184                  &info->pbl_table->pa);
2185
2186         /* in usual case we use 2 PBLs, so we add one to free
2187          * list and allocating another one
2188          */
2189         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2190         if (IS_ERR(tmp)) {
2191                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2192                 goto done;
2193         }
2194
2195         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2196
2197         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2198
2199 done:
2200         if (rc)
2201                 free_mr_info(dev, info);
2202
2203         return rc;
2204 }
2205
2206 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2207                                u64 usr_addr, int acc, struct ib_udata *udata)
2208 {
2209         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2210         struct qedr_mr *mr;
2211         struct qedr_pd *pd;
2212         int rc = -ENOMEM;
2213
2214         pd = get_qedr_pd(ibpd);
2215         DP_DEBUG(dev, QEDR_MSG_MR,
2216                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2217                  pd->pd_id, start, len, usr_addr, acc);
2218
2219         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2220                 return ERR_PTR(-EINVAL);
2221
2222         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2223         if (!mr)
2224                 return ERR_PTR(rc);
2225
2226         mr->type = QEDR_MR_USER;
2227
2228         mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2229         if (IS_ERR(mr->umem)) {
2230                 rc = -EFAULT;
2231                 goto err0;
2232         }
2233
2234         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2235         if (rc)
2236                 goto err1;
2237
2238         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2239                            &mr->info.pbl_info, mr->umem->page_shift);
2240
2241         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2242         if (rc) {
2243                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2244                 goto err1;
2245         }
2246
2247         /* Index only, 18 bit long, lkey = itid << 8 | key */
2248         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2249         mr->hw_mr.key = 0;
2250         mr->hw_mr.pd = pd->pd_id;
2251         mr->hw_mr.local_read = 1;
2252         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2253         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2254         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2255         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2256         mr->hw_mr.mw_bind = false;
2257         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2258         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2259         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2260         mr->hw_mr.page_size_log = mr->umem->page_shift;
2261         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2262         mr->hw_mr.length = len;
2263         mr->hw_mr.vaddr = usr_addr;
2264         mr->hw_mr.zbva = false;
2265         mr->hw_mr.phy_mr = false;
2266         mr->hw_mr.dma_mr = false;
2267
2268         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2269         if (rc) {
2270                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2271                 goto err2;
2272         }
2273
2274         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2275         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2276             mr->hw_mr.remote_atomic)
2277                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2278
2279         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2280                  mr->ibmr.lkey);
2281         return &mr->ibmr;
2282
2283 err2:
2284         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2285 err1:
2286         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2287 err0:
2288         kfree(mr);
2289         return ERR_PTR(rc);
2290 }
2291
2292 int qedr_dereg_mr(struct ib_mr *ib_mr)
2293 {
2294         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2295         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2296         int rc = 0;
2297
2298         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2299         if (rc)
2300                 return rc;
2301
2302         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2303
2304         if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2305                 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2306
2307         /* it could be user registered memory. */
2308         if (mr->umem)
2309                 ib_umem_release(mr->umem);
2310
2311         kfree(mr);
2312
2313         return rc;
2314 }
2315
2316 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2317                                        int max_page_list_len)
2318 {
2319         struct qedr_pd *pd = get_qedr_pd(ibpd);
2320         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2321         struct qedr_mr *mr;
2322         int rc = -ENOMEM;
2323
2324         DP_DEBUG(dev, QEDR_MSG_MR,
2325                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2326                  max_page_list_len);
2327
2328         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2329         if (!mr)
2330                 return ERR_PTR(rc);
2331
2332         mr->dev = dev;
2333         mr->type = QEDR_MR_FRMR;
2334
2335         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2336         if (rc)
2337                 goto err0;
2338
2339         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2340         if (rc) {
2341                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2342                 goto err0;
2343         }
2344
2345         /* Index only, 18 bit long, lkey = itid << 8 | key */
2346         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2347         mr->hw_mr.key = 0;
2348         mr->hw_mr.pd = pd->pd_id;
2349         mr->hw_mr.local_read = 1;
2350         mr->hw_mr.local_write = 0;
2351         mr->hw_mr.remote_read = 0;
2352         mr->hw_mr.remote_write = 0;
2353         mr->hw_mr.remote_atomic = 0;
2354         mr->hw_mr.mw_bind = false;
2355         mr->hw_mr.pbl_ptr = 0;
2356         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2357         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2358         mr->hw_mr.fbo = 0;
2359         mr->hw_mr.length = 0;
2360         mr->hw_mr.vaddr = 0;
2361         mr->hw_mr.zbva = false;
2362         mr->hw_mr.phy_mr = true;
2363         mr->hw_mr.dma_mr = false;
2364
2365         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2366         if (rc) {
2367                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2368                 goto err1;
2369         }
2370
2371         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2372         mr->ibmr.rkey = mr->ibmr.lkey;
2373
2374         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2375         return mr;
2376
2377 err1:
2378         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2379 err0:
2380         kfree(mr);
2381         return ERR_PTR(rc);
2382 }
2383
2384 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2385                             enum ib_mr_type mr_type, u32 max_num_sg)
2386 {
2387         struct qedr_dev *dev;
2388         struct qedr_mr *mr;
2389
2390         if (mr_type != IB_MR_TYPE_MEM_REG)
2391                 return ERR_PTR(-EINVAL);
2392
2393         mr = __qedr_alloc_mr(ibpd, max_num_sg);
2394
2395         if (IS_ERR(mr))
2396                 return ERR_PTR(-EINVAL);
2397
2398         dev = mr->dev;
2399
2400         return &mr->ibmr;
2401 }
2402
2403 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2404 {
2405         struct qedr_mr *mr = get_qedr_mr(ibmr);
2406         struct qedr_pbl *pbl_table;
2407         struct regpair *pbe;
2408         u32 pbes_in_page;
2409
2410         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2411                 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2412                 return -ENOMEM;
2413         }
2414
2415         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2416                  mr->npages, addr);
2417
2418         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2419         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2420         pbe = (struct regpair *)pbl_table->va;
2421         pbe +=  mr->npages % pbes_in_page;
2422         pbe->lo = cpu_to_le32((u32)addr);
2423         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2424
2425         mr->npages++;
2426
2427         return 0;
2428 }
2429
2430 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2431 {
2432         int work = info->completed - info->completed_handled - 1;
2433
2434         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2435         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2436                 struct qedr_pbl *pbl;
2437
2438                 /* Free all the page list that are possible to be freed
2439                  * (all the ones that were invalidated), under the assumption
2440                  * that if an FMR was completed successfully that means that
2441                  * if there was an invalidate operation before it also ended
2442                  */
2443                 pbl = list_first_entry(&info->inuse_pbl_list,
2444                                        struct qedr_pbl, list_entry);
2445                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2446                 info->completed_handled++;
2447         }
2448 }
2449
2450 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2451                    int sg_nents, unsigned int *sg_offset)
2452 {
2453         struct qedr_mr *mr = get_qedr_mr(ibmr);
2454
2455         mr->npages = 0;
2456
2457         handle_completed_mrs(mr->dev, &mr->info);
2458         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2459 }
2460
2461 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2462 {
2463         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2464         struct qedr_pd *pd = get_qedr_pd(ibpd);
2465         struct qedr_mr *mr;
2466         int rc;
2467
2468         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2469         if (!mr)
2470                 return ERR_PTR(-ENOMEM);
2471
2472         mr->type = QEDR_MR_DMA;
2473
2474         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2475         if (rc) {
2476                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2477                 goto err1;
2478         }
2479
2480         /* index only, 18 bit long, lkey = itid << 8 | key */
2481         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2482         mr->hw_mr.pd = pd->pd_id;
2483         mr->hw_mr.local_read = 1;
2484         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2485         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2486         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2487         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2488         mr->hw_mr.dma_mr = true;
2489
2490         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2491         if (rc) {
2492                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2493                 goto err2;
2494         }
2495
2496         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2497         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2498             mr->hw_mr.remote_atomic)
2499                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2500
2501         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2502         return &mr->ibmr;
2503
2504 err2:
2505         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2506 err1:
2507         kfree(mr);
2508         return ERR_PTR(rc);
2509 }
2510
2511 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2512 {
2513         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2514 }
2515
2516 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2517 {
2518         int i, len = 0;
2519
2520         for (i = 0; i < num_sge; i++)
2521                 len += sg_list[i].length;
2522
2523         return len;
2524 }
2525
2526 static void swap_wqe_data64(u64 *p)
2527 {
2528         int i;
2529
2530         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2531                 *p = cpu_to_be64(cpu_to_le64(*p));
2532 }
2533
2534 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2535                                        struct qedr_qp *qp, u8 *wqe_size,
2536                                        struct ib_send_wr *wr,
2537                                        struct ib_send_wr **bad_wr, u8 *bits,
2538                                        u8 bit)
2539 {
2540         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2541         char *seg_prt, *wqe;
2542         int i, seg_siz;
2543
2544         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2545                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2546                 *bad_wr = wr;
2547                 return 0;
2548         }
2549
2550         if (!data_size)
2551                 return data_size;
2552
2553         *bits |= bit;
2554
2555         seg_prt = NULL;
2556         wqe = NULL;
2557         seg_siz = 0;
2558
2559         /* Copy data inline */
2560         for (i = 0; i < wr->num_sge; i++) {
2561                 u32 len = wr->sg_list[i].length;
2562                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2563
2564                 while (len > 0) {
2565                         u32 cur;
2566
2567                         /* New segment required */
2568                         if (!seg_siz) {
2569                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2570                                 seg_prt = wqe;
2571                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
2572                                 (*wqe_size)++;
2573                         }
2574
2575                         /* Calculate currently allowed length */
2576                         cur = min_t(u32, len, seg_siz);
2577                         memcpy(seg_prt, src, cur);
2578
2579                         /* Update segment variables */
2580                         seg_prt += cur;
2581                         seg_siz -= cur;
2582
2583                         /* Update sge variables */
2584                         src += cur;
2585                         len -= cur;
2586
2587                         /* Swap fully-completed segments */
2588                         if (!seg_siz)
2589                                 swap_wqe_data64((u64 *)wqe);
2590                 }
2591         }
2592
2593         /* swap last not completed segment */
2594         if (seg_siz)
2595                 swap_wqe_data64((u64 *)wqe);
2596
2597         return data_size;
2598 }
2599
2600 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
2601         do {                                                    \
2602                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
2603                 (sge)->length = cpu_to_le32(vlength);           \
2604                 (sge)->flags = cpu_to_le32(vflags);             \
2605         } while (0)
2606
2607 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
2608         do {                                                    \
2609                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
2610                 (hdr)->num_sges = num_sge;                      \
2611         } while (0)
2612
2613 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
2614         do {                                                    \
2615                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
2616                 (sge)->length = cpu_to_le32(vlength);           \
2617                 (sge)->l_key = cpu_to_le32(vlkey);              \
2618         } while (0)
2619
2620 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2621                                 struct ib_send_wr *wr)
2622 {
2623         u32 data_size = 0;
2624         int i;
2625
2626         for (i = 0; i < wr->num_sge; i++) {
2627                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2628
2629                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2630                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2631                 sge->length = cpu_to_le32(wr->sg_list[i].length);
2632                 data_size += wr->sg_list[i].length;
2633         }
2634
2635         if (wqe_size)
2636                 *wqe_size += wr->num_sge;
2637
2638         return data_size;
2639 }
2640
2641 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2642                                      struct qedr_qp *qp,
2643                                      struct rdma_sq_rdma_wqe_1st *rwqe,
2644                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
2645                                      struct ib_send_wr *wr,
2646                                      struct ib_send_wr **bad_wr)
2647 {
2648         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2649         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2650
2651         if (wr->send_flags & IB_SEND_INLINE &&
2652             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2653              wr->opcode == IB_WR_RDMA_WRITE)) {
2654                 u8 flags = 0;
2655
2656                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2657                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2658                                                    bad_wr, &rwqe->flags, flags);
2659         }
2660
2661         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2662 }
2663
2664 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2665                                      struct qedr_qp *qp,
2666                                      struct rdma_sq_send_wqe_1st *swqe,
2667                                      struct rdma_sq_send_wqe_2st *swqe2,
2668                                      struct ib_send_wr *wr,
2669                                      struct ib_send_wr **bad_wr)
2670 {
2671         memset(swqe2, 0, sizeof(*swqe2));
2672         if (wr->send_flags & IB_SEND_INLINE) {
2673                 u8 flags = 0;
2674
2675                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2676                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2677                                                    bad_wr, &swqe->flags, flags);
2678         }
2679
2680         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2681 }
2682
2683 static int qedr_prepare_reg(struct qedr_qp *qp,
2684                             struct rdma_sq_fmr_wqe_1st *fwqe1,
2685                             struct ib_reg_wr *wr)
2686 {
2687         struct qedr_mr *mr = get_qedr_mr(wr->mr);
2688         struct rdma_sq_fmr_wqe_2nd *fwqe2;
2689
2690         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2691         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2692         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2693         fwqe1->l_key = wr->key;
2694
2695         fwqe2->access_ctrl = 0;
2696
2697         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2698                    !!(wr->access & IB_ACCESS_REMOTE_READ));
2699         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2700                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2701         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2702                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2703         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2704         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2705                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2706         fwqe2->fmr_ctrl = 0;
2707
2708         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2709                    ilog2(mr->ibmr.page_size) - 12);
2710
2711         fwqe2->length_hi = 0;
2712         fwqe2->length_lo = mr->ibmr.length;
2713         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2714         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2715
2716         qp->wqe_wr_id[qp->sq.prod].mr = mr;
2717
2718         return 0;
2719 }
2720
2721 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2722 {
2723         switch (opcode) {
2724         case IB_WR_RDMA_WRITE:
2725         case IB_WR_RDMA_WRITE_WITH_IMM:
2726                 return IB_WC_RDMA_WRITE;
2727         case IB_WR_SEND_WITH_IMM:
2728         case IB_WR_SEND:
2729         case IB_WR_SEND_WITH_INV:
2730                 return IB_WC_SEND;
2731         case IB_WR_RDMA_READ:
2732                 return IB_WC_RDMA_READ;
2733         case IB_WR_ATOMIC_CMP_AND_SWP:
2734                 return IB_WC_COMP_SWAP;
2735         case IB_WR_ATOMIC_FETCH_AND_ADD:
2736                 return IB_WC_FETCH_ADD;
2737         case IB_WR_REG_MR:
2738                 return IB_WC_REG_MR;
2739         case IB_WR_LOCAL_INV:
2740                 return IB_WC_LOCAL_INV;
2741         default:
2742                 return IB_WC_SEND;
2743         }
2744 }
2745
2746 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2747 {
2748         int wq_is_full, err_wr, pbl_is_full;
2749         struct qedr_dev *dev = qp->dev;
2750
2751         /* prevent SQ overflow and/or processing of a bad WR */
2752         err_wr = wr->num_sge > qp->sq.max_sges;
2753         wq_is_full = qedr_wq_is_full(&qp->sq);
2754         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2755                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2756         if (wq_is_full || err_wr || pbl_is_full) {
2757                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2758                         DP_ERR(dev,
2759                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2760                                qp);
2761                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2762                 }
2763
2764                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2765                         DP_ERR(dev,
2766                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2767                                qp);
2768                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2769                 }
2770
2771                 if (pbl_is_full &&
2772                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2773                         DP_ERR(dev,
2774                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2775                                qp);
2776                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2777                 }
2778                 return false;
2779         }
2780         return true;
2781 }
2782
2783 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2784                      struct ib_send_wr **bad_wr)
2785 {
2786         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2787         struct qedr_qp *qp = get_qedr_qp(ibqp);
2788         struct rdma_sq_atomic_wqe_1st *awqe1;
2789         struct rdma_sq_atomic_wqe_2nd *awqe2;
2790         struct rdma_sq_atomic_wqe_3rd *awqe3;
2791         struct rdma_sq_send_wqe_2st *swqe2;
2792         struct rdma_sq_local_inv_wqe *iwqe;
2793         struct rdma_sq_rdma_wqe_2nd *rwqe2;
2794         struct rdma_sq_send_wqe_1st *swqe;
2795         struct rdma_sq_rdma_wqe_1st *rwqe;
2796         struct rdma_sq_fmr_wqe_1st *fwqe1;
2797         struct rdma_sq_common_wqe *wqe;
2798         u32 length;
2799         int rc = 0;
2800         bool comp;
2801
2802         if (!qedr_can_post_send(qp, wr)) {
2803                 *bad_wr = wr;
2804                 return -ENOMEM;
2805         }
2806
2807         wqe = qed_chain_produce(&qp->sq.pbl);
2808         qp->wqe_wr_id[qp->sq.prod].signaled =
2809                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2810
2811         wqe->flags = 0;
2812         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2813                    !!(wr->send_flags & IB_SEND_SOLICITED));
2814         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2815         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2816         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2817                    !!(wr->send_flags & IB_SEND_FENCE));
2818         wqe->prev_wqe_size = qp->prev_wqe_size;
2819
2820         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2821
2822         switch (wr->opcode) {
2823         case IB_WR_SEND_WITH_IMM:
2824                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2825                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2826                 swqe->wqe_size = 2;
2827                 swqe2 = qed_chain_produce(&qp->sq.pbl);
2828
2829                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2830                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2831                                                    wr, bad_wr);
2832                 swqe->length = cpu_to_le32(length);
2833                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2834                 qp->prev_wqe_size = swqe->wqe_size;
2835                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2836                 break;
2837         case IB_WR_SEND:
2838                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2839                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2840
2841                 swqe->wqe_size = 2;
2842                 swqe2 = qed_chain_produce(&qp->sq.pbl);
2843                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2844                                                    wr, bad_wr);
2845                 swqe->length = cpu_to_le32(length);
2846                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2847                 qp->prev_wqe_size = swqe->wqe_size;
2848                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2849                 break;
2850         case IB_WR_SEND_WITH_INV:
2851                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2852                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2853                 swqe2 = qed_chain_produce(&qp->sq.pbl);
2854                 swqe->wqe_size = 2;
2855                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2856                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2857                                                    wr, bad_wr);
2858                 swqe->length = cpu_to_le32(length);
2859                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2860                 qp->prev_wqe_size = swqe->wqe_size;
2861                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2862                 break;
2863
2864         case IB_WR_RDMA_WRITE_WITH_IMM:
2865                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2866                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2867
2868                 rwqe->wqe_size = 2;
2869                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2870                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2871                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2872                                                    wr, bad_wr);
2873                 rwqe->length = cpu_to_le32(length);
2874                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2875                 qp->prev_wqe_size = rwqe->wqe_size;
2876                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2877                 break;
2878         case IB_WR_RDMA_WRITE:
2879                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2880                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2881
2882                 rwqe->wqe_size = 2;
2883                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2884                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2885                                                    wr, bad_wr);
2886                 rwqe->length = cpu_to_le32(length);
2887                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2888                 qp->prev_wqe_size = rwqe->wqe_size;
2889                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2890                 break;
2891         case IB_WR_RDMA_READ_WITH_INV:
2892                 DP_ERR(dev,
2893                        "RDMA READ WITH INVALIDATE not supported\n");
2894                 *bad_wr = wr;
2895                 rc = -EINVAL;
2896                 break;
2897
2898         case IB_WR_RDMA_READ:
2899                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2900                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2901
2902                 rwqe->wqe_size = 2;
2903                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2904                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2905                                                    wr, bad_wr);
2906                 rwqe->length = cpu_to_le32(length);
2907                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2908                 qp->prev_wqe_size = rwqe->wqe_size;
2909                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2910                 break;
2911
2912         case IB_WR_ATOMIC_CMP_AND_SWP:
2913         case IB_WR_ATOMIC_FETCH_AND_ADD:
2914                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2915                 awqe1->wqe_size = 4;
2916
2917                 awqe2 = qed_chain_produce(&qp->sq.pbl);
2918                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2919                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2920
2921                 awqe3 = qed_chain_produce(&qp->sq.pbl);
2922
2923                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2924                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2925                         DMA_REGPAIR_LE(awqe3->swap_data,
2926                                        atomic_wr(wr)->compare_add);
2927                 } else {
2928                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2929                         DMA_REGPAIR_LE(awqe3->swap_data,
2930                                        atomic_wr(wr)->swap);
2931                         DMA_REGPAIR_LE(awqe3->cmp_data,
2932                                        atomic_wr(wr)->compare_add);
2933                 }
2934
2935                 qedr_prepare_sq_sges(qp, NULL, wr);
2936
2937                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2938                 qp->prev_wqe_size = awqe1->wqe_size;
2939                 break;
2940
2941         case IB_WR_LOCAL_INV:
2942                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2943                 iwqe->wqe_size = 1;
2944
2945                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2946                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2947                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2948                 qp->prev_wqe_size = iwqe->wqe_size;
2949                 break;
2950         case IB_WR_REG_MR:
2951                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2952                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2953                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2954                 fwqe1->wqe_size = 2;
2955
2956                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2957                 if (rc) {
2958                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2959                         *bad_wr = wr;
2960                         break;
2961                 }
2962
2963                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2964                 qp->prev_wqe_size = fwqe1->wqe_size;
2965                 break;
2966         default:
2967                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2968                 rc = -EINVAL;
2969                 *bad_wr = wr;
2970                 break;
2971         }
2972
2973         if (*bad_wr) {
2974                 u16 value;
2975
2976                 /* Restore prod to its position before
2977                  * this WR was processed
2978                  */
2979                 value = le16_to_cpu(qp->sq.db_data.data.value);
2980                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2981
2982                 /* Restore prev_wqe_size */
2983                 qp->prev_wqe_size = wqe->prev_wqe_size;
2984                 rc = -EINVAL;
2985                 DP_ERR(dev, "POST SEND FAILED\n");
2986         }
2987
2988         return rc;
2989 }
2990
2991 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2992                    struct ib_send_wr **bad_wr)
2993 {
2994         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2995         struct qedr_qp *qp = get_qedr_qp(ibqp);
2996         unsigned long flags;
2997         int rc = 0;
2998
2999         *bad_wr = NULL;
3000
3001         if (qp->qp_type == IB_QPT_GSI)
3002                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3003
3004         spin_lock_irqsave(&qp->q_lock, flags);
3005
3006         if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3007             (qp->state != QED_ROCE_QP_STATE_ERR) &&
3008             (qp->state != QED_ROCE_QP_STATE_SQD)) {
3009                 spin_unlock_irqrestore(&qp->q_lock, flags);
3010                 *bad_wr = wr;
3011                 DP_DEBUG(dev, QEDR_MSG_CQ,
3012                          "QP in wrong state! QP icid=0x%x state %d\n",
3013                          qp->icid, qp->state);
3014                 return -EINVAL;
3015         }
3016
3017         while (wr) {
3018                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3019                 if (rc)
3020                         break;
3021
3022                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3023
3024                 qedr_inc_sw_prod(&qp->sq);
3025
3026                 qp->sq.db_data.data.value++;
3027
3028                 wr = wr->next;
3029         }
3030
3031         /* Trigger doorbell
3032          * If there was a failure in the first WR then it will be triggered in
3033          * vane. However this is not harmful (as long as the producer value is
3034          * unchanged). For performance reasons we avoid checking for this
3035          * redundant doorbell.
3036          */
3037         wmb();
3038         writel(qp->sq.db_data.raw, qp->sq.db);
3039
3040         /* Make sure write sticks */
3041         mmiowb();
3042
3043         spin_unlock_irqrestore(&qp->q_lock, flags);
3044
3045         return rc;
3046 }
3047
3048 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3049                    struct ib_recv_wr **bad_wr)
3050 {
3051         struct qedr_qp *qp = get_qedr_qp(ibqp);
3052         struct qedr_dev *dev = qp->dev;
3053         unsigned long flags;
3054         int status = 0;
3055
3056         if (qp->qp_type == IB_QPT_GSI)
3057                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3058
3059         spin_lock_irqsave(&qp->q_lock, flags);
3060
3061         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3062                 spin_unlock_irqrestore(&qp->q_lock, flags);
3063                 *bad_wr = wr;
3064                 return -EINVAL;
3065         }
3066
3067         while (wr) {
3068                 int i;
3069
3070                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3071                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3072                     wr->num_sge > qp->rq.max_sges) {
3073                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3074                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3075                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3076                                qp->rq.max_sges);
3077                         status = -ENOMEM;
3078                         *bad_wr = wr;
3079                         break;
3080                 }
3081                 for (i = 0; i < wr->num_sge; i++) {
3082                         u32 flags = 0;
3083                         struct rdma_rq_sge *rqe =
3084                             qed_chain_produce(&qp->rq.pbl);
3085
3086                         /* First one must include the number
3087                          * of SGE in the list
3088                          */
3089                         if (!i)
3090                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3091                                           wr->num_sge);
3092
3093                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3094                                   wr->sg_list[i].lkey);
3095
3096                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3097                                    wr->sg_list[i].length, flags);
3098                 }
3099
3100                 /* Special case of no sges. FW requires between 1-4 sges...
3101                  * in this case we need to post 1 sge with length zero. this is
3102                  * because rdma write with immediate consumes an RQ.
3103                  */
3104                 if (!wr->num_sge) {
3105                         u32 flags = 0;
3106                         struct rdma_rq_sge *rqe =
3107                             qed_chain_produce(&qp->rq.pbl);
3108
3109                         /* First one must include the number
3110                          * of SGE in the list
3111                          */
3112                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3113                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3114
3115                         RQ_SGE_SET(rqe, 0, 0, flags);
3116                         i = 1;
3117                 }
3118
3119                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3120                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3121
3122                 qedr_inc_sw_prod(&qp->rq);
3123
3124                 /* Flush all the writes before signalling doorbell */
3125                 wmb();
3126
3127                 qp->rq.db_data.data.value++;
3128
3129                 writel(qp->rq.db_data.raw, qp->rq.db);
3130
3131                 /* Make sure write sticks */
3132                 mmiowb();
3133
3134                 wr = wr->next;
3135         }
3136
3137         spin_unlock_irqrestore(&qp->q_lock, flags);
3138
3139         return status;
3140 }
3141
3142 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3143 {
3144         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3145
3146         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3147                 cq->pbl_toggle;
3148 }
3149
3150 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3151 {
3152         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3153         struct qedr_qp *qp;
3154
3155         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3156                                                    resp_cqe->qp_handle.lo,
3157                                                    u64);
3158         return qp;
3159 }
3160
3161 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3162 {
3163         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3164
3165         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3166 }
3167
3168 /* Return latest CQE (needs processing) */
3169 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3170 {
3171         return cq->latest_cqe;
3172 }
3173
3174 /* In fmr we need to increase the number of fmr completed counter for the fmr
3175  * algorithm determining whether we can free a pbl or not.
3176  * we need to perform this whether the work request was signaled or not. for
3177  * this purpose we call this function from the condition that checks if a wr
3178  * should be skipped, to make sure we don't miss it ( possibly this fmr
3179  * operation was not signalted)
3180  */
3181 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3182 {
3183         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3184                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3185 }
3186
3187 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3188                        struct qedr_cq *cq, int num_entries,
3189                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3190                        int force)
3191 {
3192         u16 cnt = 0;
3193
3194         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3195                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3196                         qedr_chk_if_fmr(qp);
3197                         /* skip WC */
3198                         goto next_cqe;
3199                 }
3200
3201                 /* fill WC */
3202                 wc->status = status;
3203                 wc->vendor_err = 0;
3204                 wc->wc_flags = 0;
3205                 wc->src_qp = qp->id;
3206                 wc->qp = &qp->ibqp;
3207
3208                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3209                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3210
3211                 switch (wc->opcode) {
3212                 case IB_WC_RDMA_WRITE:
3213                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3214                         break;
3215                 case IB_WC_COMP_SWAP:
3216                 case IB_WC_FETCH_ADD:
3217                         wc->byte_len = 8;
3218                         break;
3219                 case IB_WC_REG_MR:
3220                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3221                         break;
3222                 case IB_WC_RDMA_READ:
3223                 case IB_WC_SEND:
3224                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3225                         break;
3226                 default:
3227                         break;
3228                 }
3229
3230                 num_entries--;
3231                 wc++;
3232                 cnt++;
3233 next_cqe:
3234                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3235                         qed_chain_consume(&qp->sq.pbl);
3236                 qedr_inc_sw_cons(&qp->sq);
3237         }
3238
3239         return cnt;
3240 }
3241
3242 static int qedr_poll_cq_req(struct qedr_dev *dev,
3243                             struct qedr_qp *qp, struct qedr_cq *cq,
3244                             int num_entries, struct ib_wc *wc,
3245                             struct rdma_cqe_requester *req)
3246 {
3247         int cnt = 0;
3248
3249         switch (req->status) {
3250         case RDMA_CQE_REQ_STS_OK:
3251                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3252                                   IB_WC_SUCCESS, 0);
3253                 break;
3254         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3255                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3256                         DP_ERR(dev,
3257                                "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3258                                cq->icid, qp->icid);
3259                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3260                                   IB_WC_WR_FLUSH_ERR, 1);
3261                 break;
3262         default:
3263                 /* process all WQE before the cosumer */
3264                 qp->state = QED_ROCE_QP_STATE_ERR;
3265                 cnt = process_req(dev, qp, cq, num_entries, wc,
3266                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
3267                 wc += cnt;
3268                 /* if we have extra WC fill it with actual error info */
3269                 if (cnt < num_entries) {
3270                         enum ib_wc_status wc_status;
3271
3272                         switch (req->status) {
3273                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3274                                 DP_ERR(dev,
3275                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3276                                        cq->icid, qp->icid);
3277                                 wc_status = IB_WC_BAD_RESP_ERR;
3278                                 break;
3279                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3280                                 DP_ERR(dev,
3281                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3282                                        cq->icid, qp->icid);
3283                                 wc_status = IB_WC_LOC_LEN_ERR;
3284                                 break;
3285                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3286                                 DP_ERR(dev,
3287                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3288                                        cq->icid, qp->icid);
3289                                 wc_status = IB_WC_LOC_QP_OP_ERR;
3290                                 break;
3291                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3292                                 DP_ERR(dev,
3293                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3294                                        cq->icid, qp->icid);
3295                                 wc_status = IB_WC_LOC_PROT_ERR;
3296                                 break;
3297                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3298                                 DP_ERR(dev,
3299                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3300                                        cq->icid, qp->icid);
3301                                 wc_status = IB_WC_MW_BIND_ERR;
3302                                 break;
3303                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3304                                 DP_ERR(dev,
3305                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3306                                        cq->icid, qp->icid);
3307                                 wc_status = IB_WC_REM_INV_REQ_ERR;
3308                                 break;
3309                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3310                                 DP_ERR(dev,
3311                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3312                                        cq->icid, qp->icid);
3313                                 wc_status = IB_WC_REM_ACCESS_ERR;
3314                                 break;
3315                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3316                                 DP_ERR(dev,
3317                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3318                                        cq->icid, qp->icid);
3319                                 wc_status = IB_WC_REM_OP_ERR;
3320                                 break;
3321                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3322                                 DP_ERR(dev,
3323                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3324                                        cq->icid, qp->icid);
3325                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3326                                 break;
3327                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3328                                 DP_ERR(dev,
3329                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3330                                        cq->icid, qp->icid);
3331                                 wc_status = IB_WC_RETRY_EXC_ERR;
3332                                 break;
3333                         default:
3334                                 DP_ERR(dev,
3335                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3336                                        cq->icid, qp->icid);
3337                                 wc_status = IB_WC_GENERAL_ERR;
3338                         }
3339                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3340                                            wc_status, 1);
3341                 }
3342         }
3343
3344         return cnt;
3345 }
3346
3347 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3348 {
3349         switch (status) {
3350         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3351                 return IB_WC_LOC_ACCESS_ERR;
3352         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3353                 return IB_WC_LOC_LEN_ERR;
3354         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3355                 return IB_WC_LOC_QP_OP_ERR;
3356         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3357                 return IB_WC_LOC_PROT_ERR;
3358         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3359                 return IB_WC_MW_BIND_ERR;
3360         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3361                 return IB_WC_REM_INV_RD_REQ_ERR;
3362         case RDMA_CQE_RESP_STS_OK:
3363                 return IB_WC_SUCCESS;
3364         default:
3365                 return IB_WC_GENERAL_ERR;
3366         }
3367 }
3368
3369 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3370                                           struct ib_wc *wc)
3371 {
3372         wc->status = IB_WC_SUCCESS;
3373         wc->byte_len = le32_to_cpu(resp->length);
3374
3375         if (resp->flags & QEDR_RESP_IMM) {
3376                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3377                 wc->wc_flags |= IB_WC_WITH_IMM;
3378
3379                 if (resp->flags & QEDR_RESP_RDMA)
3380                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3381
3382                 if (resp->flags & QEDR_RESP_INV)
3383                         return -EINVAL;
3384
3385         } else if (resp->flags & QEDR_RESP_INV) {
3386                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3387                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3388
3389                 if (resp->flags & QEDR_RESP_RDMA)
3390                         return -EINVAL;
3391
3392         } else if (resp->flags & QEDR_RESP_RDMA) {
3393                 return -EINVAL;
3394         }
3395
3396         return 0;
3397 }
3398
3399 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3400                                struct qedr_cq *cq, struct ib_wc *wc,
3401                                struct rdma_cqe_responder *resp, u64 wr_id)
3402 {
3403         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3404         wc->opcode = IB_WC_RECV;
3405         wc->wc_flags = 0;
3406
3407         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3408                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3409                         DP_ERR(dev,
3410                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3411                                cq, cq->icid, resp->flags);
3412
3413         } else {
3414                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3415                 if (wc->status == IB_WC_GENERAL_ERR)
3416                         DP_ERR(dev,
3417                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3418                                cq, cq->icid, resp->status);
3419         }
3420
3421         /* Fill the rest of the WC */
3422         wc->vendor_err = 0;
3423         wc->src_qp = qp->id;
3424         wc->qp = &qp->ibqp;
3425         wc->wr_id = wr_id;
3426 }
3427
3428 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3429                             struct qedr_cq *cq, struct ib_wc *wc,
3430                             struct rdma_cqe_responder *resp)
3431 {
3432         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3433
3434         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3435
3436         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3437                 qed_chain_consume(&qp->rq.pbl);
3438         qedr_inc_sw_cons(&qp->rq);
3439
3440         return 1;
3441 }
3442
3443 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3444                               int num_entries, struct ib_wc *wc, u16 hw_cons)
3445 {
3446         u16 cnt = 0;
3447
3448         while (num_entries && qp->rq.wqe_cons != hw_cons) {
3449                 /* fill WC */
3450                 wc->status = IB_WC_WR_FLUSH_ERR;
3451                 wc->vendor_err = 0;
3452                 wc->wc_flags = 0;
3453                 wc->src_qp = qp->id;
3454                 wc->byte_len = 0;
3455                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3456                 wc->qp = &qp->ibqp;
3457                 num_entries--;
3458                 wc++;
3459                 cnt++;
3460                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3461                         qed_chain_consume(&qp->rq.pbl);
3462                 qedr_inc_sw_cons(&qp->rq);
3463         }
3464
3465         return cnt;
3466 }
3467
3468 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3469                                  struct rdma_cqe_responder *resp, int *update)
3470 {
3471         if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3472                 consume_cqe(cq);
3473                 *update |= 1;
3474         }
3475 }
3476
3477 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3478                              struct qedr_cq *cq, int num_entries,
3479                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
3480                              int *update)
3481 {
3482         int cnt;
3483
3484         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3485                 cnt = process_resp_flush(qp, cq, num_entries, wc,
3486                                          resp->rq_cons);
3487                 try_consume_resp_cqe(cq, qp, resp, update);
3488         } else {
3489                 cnt = process_resp_one(dev, qp, cq, wc, resp);
3490                 consume_cqe(cq);
3491                 *update |= 1;
3492         }
3493
3494         return cnt;
3495 }
3496
3497 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3498                                 struct rdma_cqe_requester *req, int *update)
3499 {
3500         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3501                 consume_cqe(cq);
3502                 *update |= 1;
3503         }
3504 }
3505
3506 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3507 {
3508         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3509         struct qedr_cq *cq = get_qedr_cq(ibcq);
3510         union rdma_cqe *cqe = cq->latest_cqe;
3511         u32 old_cons, new_cons;
3512         unsigned long flags;
3513         int update = 0;
3514         int done = 0;
3515
3516         if (cq->destroyed) {
3517                 DP_ERR(dev,
3518                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
3519                        cq, cq->icid);
3520                 return 0;
3521         }
3522
3523         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3524                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3525
3526         spin_lock_irqsave(&cq->cq_lock, flags);
3527         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3528         while (num_entries && is_valid_cqe(cq, cqe)) {
3529                 struct qedr_qp *qp;
3530                 int cnt = 0;
3531
3532                 /* prevent speculative reads of any field of CQE */
3533                 rmb();
3534
3535                 qp = cqe_get_qp(cqe);
3536                 if (!qp) {
3537                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3538                         break;
3539                 }
3540
3541                 wc->qp = &qp->ibqp;
3542
3543                 switch (cqe_get_type(cqe)) {
3544                 case RDMA_CQE_TYPE_REQUESTER:
3545                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3546                                                &cqe->req);
3547                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
3548                         break;
3549                 case RDMA_CQE_TYPE_RESPONDER_RQ:
3550                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3551                                                 &cqe->resp, &update);
3552                         break;
3553                 case RDMA_CQE_TYPE_INVALID:
3554                 default:
3555                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
3556                                cqe_get_type(cqe));
3557                 }
3558                 num_entries -= cnt;
3559                 wc += cnt;
3560                 done += cnt;
3561
3562                 cqe = get_cqe(cq);
3563         }
3564         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3565
3566         cq->cq_cons += new_cons - old_cons;
3567
3568         if (update)
3569                 /* doorbell notifies abount latest VALID entry,
3570                  * but chain already point to the next INVALID one
3571                  */
3572                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3573
3574         spin_unlock_irqrestore(&cq->cq_lock, flags);
3575         return done;
3576 }
3577
3578 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3579                      u8 port_num,
3580                      const struct ib_wc *in_wc,
3581                      const struct ib_grh *in_grh,
3582                      const struct ib_mad_hdr *mad_hdr,
3583                      size_t in_mad_size, struct ib_mad_hdr *out_mad,
3584                      size_t *out_mad_size, u16 *out_mad_pkey_index)
3585 {
3586         struct qedr_dev *dev = get_qedr_dev(ibdev);
3587
3588         DP_DEBUG(dev, QEDR_MSG_GSI,
3589                  "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3590                  mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3591                  mad_hdr->class_specific, mad_hdr->class_version,
3592                  mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3593         return IB_MAD_RESULT_SUCCESS;
3594 }
3595
3596 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3597                         struct ib_port_immutable *immutable)
3598 {
3599         struct ib_port_attr attr;
3600         int err;
3601
3602         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3603                                     RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3604
3605         err = ib_query_port(ibdev, port_num, &attr);
3606         if (err)
3607                 return err;
3608
3609         immutable->pkey_tbl_len = attr.pkey_tbl_len;
3610         immutable->gid_tbl_len = attr.gid_tbl_len;
3611         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3612
3613         return 0;
3614 }