drivers/net/ethernet/qlogic/qed/qed_roce.c

   1 /* QLogic qed NIC Driver
   2  * Copyright (c) 2015-2017  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/types.h>
  33 #include <asm/byteorder.h>
  34 #include <linux/bitops.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/errno.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_ether.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/io.h>
  42 #include <linux/ip.h>
  43 #include <linux/ipv6.h>
  44 #include <linux/kernel.h>
  45 #include <linux/list.h>
  46 #include <linux/module.h>
  47 #include <linux/mutex.h>
  48 #include <linux/pci.h>
  49 #include <linux/slab.h>
  50 #include <linux/spinlock.h>
  51 #include <linux/string.h>
  52 #include <linux/tcp.h>
  53 #include <linux/bitops.h>
  54 #include <linux/qed/qed_roce_if.h>
  55 #include <linux/qed/qed_roce_if.h>
  56 #include "qed.h"
  57 #include "qed_cxt.h"
  58 #include "qed_hsi.h"
  59 #include "qed_hw.h"
  60 #include "qed_init_ops.h"
  61 #include "qed_int.h"
  62 #include "qed_ll2.h"
  63 #include "qed_mcp.h"
  64 #include "qed_reg_addr.h"
  65 #include "qed_sp.h"
  66 #include "qed_roce.h"
  67 #include "qed_ll2.h"
  68
  69 static void qed_roce_free_real_icid(struct qed_hwfn *p_hwfn, u16 icid);
  70
  71 void qed_roce_async_event(struct qed_hwfn *p_hwfn,
  72                           u8 fw_event_code, union rdma_eqe_data *rdma_data)
  73 {
  74         if (fw_event_code == ROCE_ASYNC_EVENT_DESTROY_QP_DONE) {
  75                 u16 icid =
  76                     (u16)le32_to_cpu(rdma_data->rdma_destroy_qp_data.cid);
  77
  78                 /* icid release in this async event can occur only if the icid
  79                  * was offloaded to the FW. In case it wasn't offloaded this is
  80                  * handled in qed_roce_sp_destroy_qp.
  81                  */
  82                 qed_roce_free_real_icid(p_hwfn, icid);
  83         } else {
  84                 struct qed_rdma_events *events = &p_hwfn->p_rdma_info->events;
  85
  86                 events->affiliated_event(p_hwfn->p_rdma_info->events.context,
  87                                          fw_event_code,
  88                                          &rdma_data->async_handle);
  89         }
  90 }
  91
  92 static int qed_rdma_bmap_alloc(struct qed_hwfn *p_hwfn,
  93                                struct qed_bmap *bmap, u32 max_count, char *name)
  94 {
  95         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "max_count = %08x\n", max_count);
  96
  97         bmap->max_count = max_count;
  98
  99         bmap->bitmap = kzalloc(BITS_TO_LONGS(max_count) * sizeof(long),
 100                                GFP_KERNEL);
 101         if (!bmap->bitmap) {
 102                 DP_NOTICE(p_hwfn,
 103                           "qed bmap alloc failed: cannot allocate memory (bitmap)\n");
 104                 return -ENOMEM;
 105         }
 106
 107         snprintf(bmap->name, QED_RDMA_MAX_BMAP_NAME, "%s", name);
 108
 109         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n");
 110         return 0;
 111 }
 112
 113 static int qed_rdma_bmap_alloc_id(struct qed_hwfn *p_hwfn,
 114                                   struct qed_bmap *bmap, u32 *id_num)
 115 {
 116         *id_num = find_first_zero_bit(bmap->bitmap, bmap->max_count);
 117         if (*id_num >= bmap->max_count)
 118                 return -EINVAL;
 119
 120         __set_bit(*id_num, bmap->bitmap);
 121
 122         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "%s bitmap: allocated id %d\n",
 123                    bmap->name, *id_num);
 124
 125         return 0;
 126 }
 127
 128 static void qed_bmap_set_id(struct qed_hwfn *p_hwfn,
 129                             struct qed_bmap *bmap, u32 id_num)
 130 {
 131         if (id_num >= bmap->max_count)
 132                 return;
 133
 134         __set_bit(id_num, bmap->bitmap);
 135 }
 136
 137 static void qed_bmap_release_id(struct qed_hwfn *p_hwfn,
 138                                 struct qed_bmap *bmap, u32 id_num)
 139 {
 140         bool b_acquired;
 141
 142         if (id_num >= bmap->max_count)
 143                 return;
 144
 145         b_acquired = test_and_clear_bit(id_num, bmap->bitmap);
 146         if (!b_acquired) {
 147                 DP_NOTICE(p_hwfn, "%s bitmap: id %d already released\n",
 148                           bmap->name, id_num);
 149                 return;
 150         }
 151
 152         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "%s bitmap: released id %d\n",
 153                    bmap->name, id_num);
 154 }
 155
 156 static int qed_bmap_test_id(struct qed_hwfn *p_hwfn,
 157                             struct qed_bmap *bmap, u32 id_num)
 158 {
 159         if (id_num >= bmap->max_count)
 160                 return -1;
 161
 162         return test_bit(id_num, bmap->bitmap);
 163 }
 164
 165 static u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id)
 166 {
 167         /* First sb id for RoCE is after all the l2 sb */
 168         return FEAT_NUM((struct qed_hwfn *)p_hwfn, QED_PF_L2_QUE) + rel_sb_id;
 169 }
 170
 171 static int qed_rdma_alloc(struct qed_hwfn *p_hwfn,
 172                           struct qed_ptt *p_ptt,
 173                           struct qed_rdma_start_in_params *params)
 174 {
 175         struct qed_rdma_info *p_rdma_info;
 176         u32 num_cons, num_tasks;
 177         int rc = -ENOMEM;
 178
 179         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocating RDMA\n");
 180
 181         /* Allocate a struct with current pf rdma info */
 182         p_rdma_info = kzalloc(sizeof(*p_rdma_info), GFP_KERNEL);
 183         if (!p_rdma_info) {
 184                 DP_NOTICE(p_hwfn,
 185                           "qed rdma alloc failed: cannot allocate memory (rdma info). rc = %d\n",
 186                           rc);
 187                 return rc;
 188         }
 189
 190         p_hwfn->p_rdma_info = p_rdma_info;
 191         p_rdma_info->proto = PROTOCOLID_ROCE;
 192
 193         num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto,
 194                                                NULL);
 195
 196         p_rdma_info->num_qps = num_cons / 2;
 197
 198         num_tasks = qed_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE);
 199
 200         /* Each MR uses a single task */
 201         p_rdma_info->num_mrs = num_tasks;
 202
 203         /* Queue zone lines are shared between RoCE and L2 in such a way that
 204          * they can be used by each without obstructing the other.
 205          */
 206         p_rdma_info->queue_zone_base = (u16)RESC_START(p_hwfn, QED_L2_QUEUE);
 207         p_rdma_info->max_queue_zones = (u16)RESC_NUM(p_hwfn, QED_L2_QUEUE);
 208
 209         /* Allocate a struct with device params and fill it */
 210         p_rdma_info->dev = kzalloc(sizeof(*p_rdma_info->dev), GFP_KERNEL);
 211         if (!p_rdma_info->dev) {
 212                 DP_NOTICE(p_hwfn,
 213                           "qed rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n",
 214                           rc);
 215                 goto free_rdma_info;
 216         }
 217
 218         /* Allocate a struct with port params and fill it */
 219         p_rdma_info->port = kzalloc(sizeof(*p_rdma_info->port), GFP_KERNEL);
 220         if (!p_rdma_info->port) {
 221                 DP_NOTICE(p_hwfn,
 222                           "qed rdma alloc failed: cannot allocate memory (rdma info port). rc = %d\n",
 223                           rc);
 224                 goto free_rdma_dev;
 225         }
 226
 227         /* Allocate bit map for pd's */
 228         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS,
 229                                  "PD");
 230         if (rc) {
 231                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 232                            "Failed to allocate pd_map, rc = %d\n",
 233                            rc);
 234                 goto free_rdma_port;
 235         }
 236
 237         /* Allocate DPI bitmap */
 238         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map,
 239                                  p_hwfn->dpi_count, "DPI");
 240         if (rc) {
 241                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 242                            "Failed to allocate DPI bitmap, rc = %d\n", rc);
 243                 goto free_pd_map;
 244         }
 245
 246         /* Allocate bitmap for cq's. The maximum number of CQs is bounded to
 247          * twice the number of QPs.
 248          */
 249         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map,
 250                                  p_rdma_info->num_qps * 2, "CQ");
 251         if (rc) {
 252                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 253                            "Failed to allocate cq bitmap, rc = %d\n", rc);
 254                 goto free_dpi_map;
 255         }
 256
 257         /* Allocate bitmap for toggle bit for cq icids
 258          * We toggle the bit every time we create or resize cq for a given icid.
 259          * The maximum number of CQs is bounded to  twice the number of QPs.
 260          */
 261         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits,
 262                                  p_rdma_info->num_qps * 2, "Toggle");
 263         if (rc) {
 264                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 265                            "Failed to allocate toogle bits, rc = %d\n", rc);
 266                 goto free_cq_map;
 267         }
 268
 269         /* Allocate bitmap for itids */
 270         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map,
 271                                  p_rdma_info->num_mrs, "MR");
 272         if (rc) {
 273                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 274                            "Failed to allocate itids bitmaps, rc = %d\n", rc);
 275                 goto free_toggle_map;
 276         }
 277
 278         /* Allocate bitmap for cids used for qps. */
 279         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons,
 280                                  "CID");
 281         if (rc) {
 282                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 283                            "Failed to allocate cid bitmap, rc = %d\n", rc);
 284                 goto free_tid_map;
 285         }
 286
 287         /* Allocate bitmap for cids used for responders/requesters. */
 288         rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->real_cid_map, num_cons,
 289                                  "REAL_CID");
 290         if (rc) {
 291                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
 292                            "Failed to allocate real cid bitmap, rc = %d\n", rc);
 293                 goto free_cid_map;
 294         }
 295         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocation successful\n");
 296         return 0;
 297
 298 free_cid_map:
 299         kfree(p_rdma_info->cid_map.bitmap);
 300 free_tid_map:
 301         kfree(p_rdma_info->tid_map.bitmap);
 302 free_toggle_map:
 303         kfree(p_rdma_info->toggle_bits.bitmap);
 304 free_cq_map:
 305         kfree(p_rdma_info->cq_map.bitmap);
 306 free_dpi_map:
 307         kfree(p_rdma_info->dpi_map.bitmap);
 308 free_pd_map:
 309         kfree(p_rdma_info->pd_map.bitmap);
 310 free_rdma_port:
 311         kfree(p_rdma_info->port);
 312 free_rdma_dev:
 313         kfree(p_rdma_info->dev);
 314 free_rdma_info:
 315         kfree(p_rdma_info);
 316
 317         return rc;
 318 }
 319
 320 static void qed_rdma_bmap_free(struct qed_hwfn *p_hwfn,
 321                                struct qed_bmap *bmap, bool check)
 322 {
 323         int weight = bitmap_weight(bmap->bitmap, bmap->max_count);
 324         int last_line = bmap->max_count / (64 * 8);
 325         int last_item = last_line * 8 +
 326             DIV_ROUND_UP(bmap->max_count % (64 * 8), 64);
 327         u64 *pmap = (u64 *)bmap->bitmap;
 328         int line, item, offset;
 329         u8 str_last_line[200] = { 0 };
 330
 331         if (!weight || !check)
 332                 goto end;
 333
 334         DP_NOTICE(p_hwfn,
 335                   "%s bitmap not free - size=%d, weight=%d, 512 bits per line\n",
 336                   bmap->name, bmap->max_count, weight);
 337
 338         /* print aligned non-zero lines, if any */
 339         for (item = 0, line = 0; line < last_line; line++, item += 8)
 340                 if (bitmap_weight((unsigned long *)&pmap[item], 64 * 8))
 341                         DP_NOTICE(p_hwfn,
 342                                   "line 0x%04x: 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n",
 343                                   line,
 344                                   pmap[item],
 345                                   pmap[item + 1],
 346                                   pmap[item + 2],
 347                                   pmap[item + 3],
 348                                   pmap[item + 4],
 349                                   pmap[item + 5],
 350                                   pmap[item + 6], pmap[item + 7]);
 351
 352         /* print last unaligned non-zero line, if any */
 353         if ((bmap->max_count % (64 * 8)) &&
 354             (bitmap_weight((unsigned long *)&pmap[item],
 355                            bmap->max_count - item * 64))) {
 356                 offset = sprintf(str_last_line, "line 0x%04x: ", line);
 357                 for (; item < last_item; item++)
 358                         offset += sprintf(str_last_line + offset,
 359                                           "0x%016llx ", pmap[item]);
 360                 DP_NOTICE(p_hwfn, "%s\n", str_last_line);
 361         }
 362
 363 end:
 364         kfree(bmap->bitmap);
 365         bmap->bitmap = NULL;
 366 }
 367
 368 static void qed_rdma_resc_free(struct qed_hwfn *p_hwfn)
 369 {
 370         struct qed_bmap *rcid_map = &p_hwfn->p_rdma_info->real_cid_map;
 371         struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
 372         int wait_count = 0;
 373
 374         /* when destroying a_RoCE QP the control is returned to the user after
 375          * the synchronous part. The asynchronous part may take a little longer.
 376          * We delay for a short while if an async destroy QP is still expected.
 377          * Beyond the added delay we clear the bitmap anyway.
 378          */
 379         while (bitmap_weight(rcid_map->bitmap, rcid_map->max_count)) {
 380                 msleep(100);
 381                 if (wait_count++ > 20) {
 382                         DP_NOTICE(p_hwfn, "cid bitmap wait timed out\n");
 383                         break;
 384                 }
 385         }
 386
 387         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cid_map, 1);
 388         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->pd_map, 1);
 389         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 1);
 390         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cq_map, 1);
 391         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->toggle_bits, 0);
 392         qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tid_map, 1);
 393
 394         kfree(p_rdma_info->port);
 395         kfree(p_rdma_info->dev);
 396
 397         kfree(p_rdma_info);
 398 }
 399
 400 static void qed_rdma_free(struct qed_hwfn *p_hwfn)
 401 {
 402         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Freeing RDMA\n");
 403
 404         qed_rdma_resc_free(p_hwfn);
 405 }
 406
 407 static void qed_rdma_get_guid(struct qed_hwfn *p_hwfn, u8 *guid)
 408 {
 409         guid[0] = p_hwfn->hw_info.hw_mac_addr[0] ^ 2;
 410         guid[1] = p_hwfn->hw_info.hw_mac_addr[1];
 411         guid[2] = p_hwfn->hw_info.hw_mac_addr[2];
 412         guid[3] = 0xff;
 413         guid[4] = 0xfe;
 414         guid[5] = p_hwfn->hw_info.hw_mac_addr[3];
 415         guid[6] = p_hwfn->hw_info.hw_mac_addr[4];
 416         guid[7] = p_hwfn->hw_info.hw_mac_addr[5];
 417 }
 418
 419 static void qed_rdma_init_events(struct qed_hwfn *p_hwfn,
 420                                  struct qed_rdma_start_in_params *params)
 421 {
 422         struct qed_rdma_events *events;
 423
 424         events = &p_hwfn->p_rdma_info->events;
 425
 426         events->unaffiliated_event = params->events->unaffiliated_event;
 427         events->affiliated_event = params->events->affiliated_event;
 428         events->context = params->events->context;
 429 }
 430
 431 static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn,
 432                                   struct qed_rdma_start_in_params *params)
 433 {
 434         struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
 435         struct qed_dev *cdev = p_hwfn->cdev;
 436         u32 pci_status_control;
 437         u32 num_qps;
 438
 439         /* Vendor specific information */
 440         dev->vendor_id = cdev->vendor_id;
 441         dev->vendor_part_id = cdev->device_id;
 442         dev->hw_ver = 0;
 443         dev->fw_ver = (FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) |
 444                       (FW_REVISION_VERSION << 8) | (FW_ENGINEERING_VERSION);
 445
 446         qed_rdma_get_guid(p_hwfn, (u8 *)&dev->sys_image_guid);
 447         dev->node_guid = dev->sys_image_guid;
 448
 449         dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE,
 450                              RDMA_MAX_SGE_PER_RQ_WQE);
 451
 452         if (cdev->rdma_max_sge)
 453                 dev->max_sge = min_t(u32, cdev->rdma_max_sge, dev->max_sge);
 454
 455         dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE;
 456
 457         dev->max_inline = (cdev->rdma_max_inline) ?
 458                           min_t(u32, cdev->rdma_max_inline, dev->max_inline) :
 459                           dev->max_inline;
 460
 461         dev->max_wqe = QED_RDMA_MAX_WQE;
 462         dev->max_cnq = (u8)FEAT_NUM(p_hwfn, QED_RDMA_CNQ);
 463
 464         /* The number of QPs may be higher than QED_ROCE_MAX_QPS, because
 465          * it is up-aligned to 16 and then to ILT page size within qed cxt.
 466          * This is OK in terms of ILT but we don't want to configure the FW
 467          * above its abilities
 468          */
 469         num_qps = ROCE_MAX_QPS;
 470         num_qps = min_t(u64, num_qps, p_hwfn->p_rdma_info->num_qps);
 471         dev->max_qp = num_qps;
 472
 473         /* CQs uses the same icids that QPs use hence they are limited by the
 474          * number of icids. There are two icids per QP.
 475          */
 476         dev->max_cq = num_qps * 2;
 477
 478         /* The number of mrs is smaller by 1 since the first is reserved */
 479         dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1;
 480         dev->max_mr_size = QED_RDMA_MAX_MR_SIZE;
 481
 482         /* The maximum CQE capacity per CQ supported.
 483          * max number of cqes will be in two layer pbl,
 484          * 8 is the pointer size in bytes
 485          * 32 is the size of cq element in bytes
 486          */
 487         if (params->cq_mode == QED_RDMA_CQ_MODE_32_BITS)
 488                 dev->max_cqe = QED_RDMA_MAX_CQE_32_BIT;
 489         else
 490                 dev->max_cqe = QED_RDMA_MAX_CQE_16_BIT;
 491
 492         dev->max_mw = 0;
 493         dev->max_fmr = QED_RDMA_MAX_FMR;
 494         dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8);
 495         dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE;
 496         dev->max_pkey = QED_RDMA_MAX_P_KEY;
 497
 498         dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE /
 499                                           (RDMA_RESP_RD_ATOMIC_ELM_SIZE * 2);
 500         dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE /
 501                                          RDMA_REQ_RD_ATOMIC_ELM_SIZE;
 502         dev->max_dev_resp_rd_atomic_resc = dev->max_qp_resp_rd_atomic_resc *
 503                                            p_hwfn->p_rdma_info->num_qps;
 504         dev->page_size_caps = QED_RDMA_PAGE_SIZE_CAPS;
 505         dev->dev_ack_delay = QED_RDMA_ACK_DELAY;
 506         dev->max_pd = RDMA_MAX_PDS;
 507         dev->max_ah = p_hwfn->p_rdma_info->num_qps;
 508         dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, QED_RDMA_STATS_QUEUE);
 509
 510         /* Set capablities */
 511         dev->dev_caps = 0;
 512         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RNR_NAK, 1);
 513         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1);
 514         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1);
 515         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RESIZE_CQ, 1);
 516         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1);
 517         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1);
 518         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ZBVA, 1);
 519         SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1);
 520
 521         /* Check atomic operations support in PCI configuration space. */
 522         pci_read_config_dword(cdev->pdev,
 523                               cdev->pdev->pcie_cap + PCI_EXP_DEVCTL2,
 524                               &pci_status_control);
 525
 526         if (pci_status_control & PCI_EXP_DEVCTL2_LTR_EN)
 527                 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ATOMIC_OP, 1);
 528 }
 529
 530 static void qed_rdma_init_port(struct qed_hwfn *p_hwfn)
 531 {
 532         struct qed_rdma_port *port = p_hwfn->p_rdma_info->port;
 533         struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
 534
 535         port->port_state = p_hwfn->mcp_info->link_output.link_up ?
 536                            QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN;
 537
 538         port->max_msg_size = min_t(u64,
 539                                    (dev->max_mr_mw_fmr_size *
 540                                     p_hwfn->cdev->rdma_max_sge),
 541                                    BIT(31));
 542
 543         port->pkey_bad_counter = 0;
 544 }
 545
 546 static int qed_rdma_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 547 {
 548         u32 ll2_ethertype_en;
 549
 550         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW\n");
 551         p_hwfn->b_rdma_enabled_in_prs = false;
 552
 553         qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0);
 554
 555         p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE;
 556
 557         /* We delay writing to this reg until first cid is allocated. See
 558          * qed_cxt_dynamic_ilt_alloc function for more details
 559          */
 560         ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN);
 561         qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN,
 562                (ll2_ethertype_en | 0x01));
 563
 564         if (qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2) {
 565                 DP_NOTICE(p_hwfn, "The first RoCE's cid should be even\n");
 566                 return -EINVAL;
 567         }
 568
 569         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW - Done\n");
 570         return 0;
 571 }
 572
 573 static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn,
 574                              struct qed_rdma_start_in_params *params,
 575                              struct qed_ptt *p_ptt)
 576 {
 577         struct rdma_init_func_ramrod_data *p_ramrod;
 578         struct qed_rdma_cnq_params *p_cnq_pbl_list;
 579         struct rdma_init_func_hdr *p_params_header;
 580         struct rdma_cnq_params *p_cnq_params;
 581         struct qed_sp_init_data init_data;
 582         struct qed_spq_entry *p_ent;
 583         u32 cnq_id, sb_id;
 584         int rc;
 585
 586         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Starting FW\n");
 587
 588         /* Save the number of cnqs for the function close ramrod */
 589         p_hwfn->p_rdma_info->num_cnqs = params->desired_cnq;
 590
 591         /* Get SPQ entry */
 592         memset(&init_data, 0, sizeof(init_data));
 593         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 594         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 595
 596         rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT,
 597                                  p_hwfn->p_rdma_info->proto, &init_data);
 598         if (rc)
 599                 return rc;
 600
 601         p_ramrod = &p_ent->ramrod.roce_init_func.rdma;
 602
 603         p_params_header = &p_ramrod->params_header;
 604         p_params_header->cnq_start_offset = (u8)RESC_START(p_hwfn,
 605                                                            QED_RDMA_CNQ_RAM);
 606         p_params_header->num_cnqs = params->desired_cnq;
 607
 608         if (params->cq_mode == QED_RDMA_CQ_MODE_16_BITS)
 609                 p_params_header->cq_ring_mode = 1;
 610         else
 611                 p_params_header->cq_ring_mode = 0;
 612
 613         for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++) {
 614                 sb_id = qed_rdma_get_sb_id(p_hwfn, cnq_id);
 615                 p_cnq_params = &p_ramrod->cnq_params[cnq_id];
 616                 p_cnq_pbl_list = &params->cnq_pbl_list[cnq_id];
 617                 p_cnq_params->sb_num =
 618                         cpu_to_le16(p_hwfn->sbs_info[sb_id]->igu_sb_id);
 619
 620                 p_cnq_params->sb_index = p_hwfn->pf_params.rdma_pf_params.gl_pi;
 621                 p_cnq_params->num_pbl_pages = p_cnq_pbl_list->num_pbl_pages;
 622
 623                 DMA_REGPAIR_LE(p_cnq_params->pbl_base_addr,
 624                                p_cnq_pbl_list->pbl_ptr);
 625
 626                 /* we assume here that cnq_id and qz_offset are the same */
 627                 p_cnq_params->queue_zone_num =
 628                         cpu_to_le16(p_hwfn->p_rdma_info->queue_zone_base +
 629                                     cnq_id);
 630         }
 631
 632         return qed_spq_post(p_hwfn, p_ent, NULL);
 633 }
 634
 635 static int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid)
 636 {
 637         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 638         int rc;
 639
 640         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n");
 641
 642         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 643         rc = qed_rdma_bmap_alloc_id(p_hwfn,
 644                                     &p_hwfn->p_rdma_info->tid_map, itid);
 645         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 646         if (rc)
 647                 goto out;
 648
 649         rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid);
 650 out:
 651         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc);
 652         return rc;
 653 }
 654
 655 static int qed_rdma_reserve_lkey(struct qed_hwfn *p_hwfn)
 656 {
 657         struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev;
 658
 659         /* The first DPI is reserved for the Kernel */
 660         __set_bit(0, p_hwfn->p_rdma_info->dpi_map.bitmap);
 661
 662         /* Tid 0 will be used as the key for "reserved MR".
 663          * The driver should allocate memory for it so it can be loaded but no
 664          * ramrod should be passed on it.
 665          */
 666         qed_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey);
 667         if (dev->reserved_lkey != RDMA_RESERVED_LKEY) {
 668                 DP_NOTICE(p_hwfn,
 669                           "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n");
 670                 return -EINVAL;
 671         }
 672
 673         return 0;
 674 }
 675
 676 static int qed_rdma_setup(struct qed_hwfn *p_hwfn,
 677                           struct qed_ptt *p_ptt,
 678                           struct qed_rdma_start_in_params *params)
 679 {
 680         int rc;
 681
 682         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA setup\n");
 683
 684         spin_lock_init(&p_hwfn->p_rdma_info->lock);
 685
 686         qed_rdma_init_devinfo(p_hwfn, params);
 687         qed_rdma_init_port(p_hwfn);
 688         qed_rdma_init_events(p_hwfn, params);
 689
 690         rc = qed_rdma_reserve_lkey(p_hwfn);
 691         if (rc)
 692                 return rc;
 693
 694         rc = qed_rdma_init_hw(p_hwfn, p_ptt);
 695         if (rc)
 696                 return rc;
 697
 698         return qed_rdma_start_fw(p_hwfn, params, p_ptt);
 699 }
 700
 701 static int qed_rdma_stop(void *rdma_cxt)
 702 {
 703         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 704         struct rdma_close_func_ramrod_data *p_ramrod;
 705         struct qed_sp_init_data init_data;
 706         struct qed_spq_entry *p_ent;
 707         struct qed_ptt *p_ptt;
 708         u32 ll2_ethertype_en;
 709         int rc = -EBUSY;
 710
 711         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop\n");
 712
 713         p_ptt = qed_ptt_acquire(p_hwfn);
 714         if (!p_ptt) {
 715                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Failed to acquire PTT\n");
 716                 return rc;
 717         }
 718
 719         /* Disable RoCE search */
 720         qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0);
 721         p_hwfn->b_rdma_enabled_in_prs = false;
 722
 723         qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0);
 724
 725         ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN);
 726
 727         qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN,
 728                (ll2_ethertype_en & 0xFFFE));
 729
 730         qed_ptt_release(p_hwfn, p_ptt);
 731
 732         /* Get SPQ entry */
 733         memset(&init_data, 0, sizeof(init_data));
 734         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
 735         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
 736
 737         /* Stop RoCE */
 738         rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE,
 739                                  p_hwfn->p_rdma_info->proto, &init_data);
 740         if (rc)
 741                 goto out;
 742
 743         p_ramrod = &p_ent->ramrod.rdma_close_func;
 744
 745         p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs;
 746         p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM);
 747
 748         rc = qed_spq_post(p_hwfn, p_ent, NULL);
 749
 750 out:
 751         qed_rdma_free(p_hwfn);
 752
 753         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop done, rc = %d\n", rc);
 754         return rc;
 755 }
 756
 757 static int qed_rdma_add_user(void *rdma_cxt,
 758                              struct qed_rdma_add_user_out_params *out_params)
 759 {
 760         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 761         u32 dpi_start_offset;
 762         u32 returned_id = 0;
 763         int rc;
 764
 765         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding User\n");
 766
 767         /* Allocate DPI */
 768         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 769         rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map,
 770                                     &returned_id);
 771         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 772
 773         out_params->dpi = (u16)returned_id;
 774
 775         /* Calculate the corresponding DPI address */
 776         dpi_start_offset = p_hwfn->dpi_start_offset;
 777
 778         out_params->dpi_addr = (u64)((u8 __iomem *)p_hwfn->doorbells +
 779                                      dpi_start_offset +
 780                                      ((out_params->dpi) * p_hwfn->dpi_size));
 781
 782         out_params->dpi_phys_addr = p_hwfn->cdev->db_phys_addr +
 783                                     dpi_start_offset +
 784                                     ((out_params->dpi) * p_hwfn->dpi_size);
 785
 786         out_params->dpi_size = p_hwfn->dpi_size;
 787         out_params->wid_count = p_hwfn->wid_count;
 788
 789         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding user - done, rc = %d\n", rc);
 790         return rc;
 791 }
 792
 793 static struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt)
 794 {
 795         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 796         struct qed_rdma_port *p_port = p_hwfn->p_rdma_info->port;
 797
 798         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA Query port\n");
 799
 800         /* Link may have changed */
 801         p_port->port_state = p_hwfn->mcp_info->link_output.link_up ?
 802                              QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN;
 803
 804         p_port->link_speed = p_hwfn->mcp_info->link_output.speed;
 805
 806         p_port->max_msg_size = RDMA_MAX_DATA_SIZE_IN_WQE;
 807
 808         return p_port;
 809 }
 810
 811 static struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt)
 812 {
 813         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 814
 815         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query device\n");
 816
 817         /* Return struct with device parameters */
 818         return p_hwfn->p_rdma_info->dev;
 819 }
 820
 821 static void qed_rdma_free_tid(void *rdma_cxt, u32 itid)
 822 {
 823         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 824
 825         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid);
 826
 827         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 828         qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid);
 829         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 830 }
 831
 832 static void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod)
 833 {
 834         struct qed_hwfn *p_hwfn;
 835         u16 qz_num;
 836         u32 addr;
 837
 838         p_hwfn = (struct qed_hwfn *)rdma_cxt;
 839
 840         if (qz_offset > p_hwfn->p_rdma_info->max_queue_zones) {
 841                 DP_NOTICE(p_hwfn,
 842                           "queue zone offset %d is too large (max is %d)\n",
 843                           qz_offset, p_hwfn->p_rdma_info->max_queue_zones);
 844                 return;
 845         }
 846
 847         qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset;
 848         addr = GTT_BAR0_MAP_REG_USDM_RAM +
 849                USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num);
 850
 851         REG_WR16(p_hwfn, addr, prod);
 852
 853         /* keep prod updates ordered */
 854         wmb();
 855 }
 856
 857 static int qed_fill_rdma_dev_info(struct qed_dev *cdev,
 858                                   struct qed_dev_rdma_info *info)
 859 {
 860         struct qed_hwfn *p_hwfn = QED_LEADING_HWFN(cdev);
 861
 862         memset(info, 0, sizeof(*info));
 863
 864         info->rdma_type = QED_RDMA_TYPE_ROCE;
 865         info->user_dpm_enabled = (p_hwfn->db_bar_no_edpm == 0);
 866
 867         qed_fill_dev_info(cdev, &info->common);
 868
 869         return 0;
 870 }
 871
 872 static int qed_rdma_get_sb_start(struct qed_dev *cdev)
 873 {
 874         int feat_num;
 875
 876         if (cdev->num_hwfns > 1)
 877                 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE);
 878         else
 879                 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE) *
 880                            cdev->num_hwfns;
 881
 882         return feat_num;
 883 }
 884
 885 static int qed_rdma_get_min_cnq_msix(struct qed_dev *cdev)
 886 {
 887         int n_cnq = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_RDMA_CNQ);
 888         int n_msix = cdev->int_params.rdma_msix_cnt;
 889
 890         return min_t(int, n_cnq, n_msix);
 891 }
 892
 893 static int qed_rdma_set_int(struct qed_dev *cdev, u16 cnt)
 894 {
 895         int limit = 0;
 896
 897         /* Mark the fastpath as free/used */
 898         cdev->int_params.fp_initialized = cnt ? true : false;
 899
 900         if (cdev->int_params.out.int_mode != QED_INT_MODE_MSIX) {
 901                 DP_ERR(cdev,
 902                        "qed roce supports only MSI-X interrupts (detected %d).\n",
 903                        cdev->int_params.out.int_mode);
 904                 return -EINVAL;
 905         } else if (cdev->int_params.fp_msix_cnt) {
 906                 limit = cdev->int_params.rdma_msix_cnt;
 907         }
 908
 909         if (!limit)
 910                 return -ENOMEM;
 911
 912         return min_t(int, cnt, limit);
 913 }
 914
 915 static int qed_rdma_get_int(struct qed_dev *cdev, struct qed_int_info *info)
 916 {
 917         memset(info, 0, sizeof(*info));
 918
 919         if (!cdev->int_params.fp_initialized) {
 920                 DP_INFO(cdev,
 921                         "Protocol driver requested interrupt information, but its support is not yet configured\n");
 922                 return -EINVAL;
 923         }
 924
 925         if (cdev->int_params.out.int_mode == QED_INT_MODE_MSIX) {
 926                 int msix_base = cdev->int_params.rdma_msix_base;
 927
 928                 info->msix_cnt = cdev->int_params.rdma_msix_cnt;
 929                 info->msix = &cdev->int_params.msix_table[msix_base];
 930
 931                 DP_VERBOSE(cdev, QED_MSG_RDMA, "msix_cnt = %d msix_base=%d\n",
 932                            info->msix_cnt, msix_base);
 933         }
 934
 935         return 0;
 936 }
 937
 938 static int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd)
 939 {
 940         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 941         u32 returned_id;
 942         int rc;
 943
 944         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD\n");
 945
 946         /* Allocates an unused protection domain */
 947         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 948         rc = qed_rdma_bmap_alloc_id(p_hwfn,
 949                                     &p_hwfn->p_rdma_info->pd_map, &returned_id);
 950         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 951
 952         *pd = (u16)returned_id;
 953
 954         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc);
 955         return rc;
 956 }
 957
 958 static void qed_rdma_free_pd(void *rdma_cxt, u16 pd)
 959 {
 960         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 961
 962         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "pd = %08x\n", pd);
 963
 964         /* Returns a previously allocated protection domain for reuse */
 965         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
 966         qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd);
 967         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 968 }
 969
 970 static enum qed_rdma_toggle_bit
 971 qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn *p_hwfn, u16 icid)
 972 {
 973         struct qed_rdma_info *p_info = p_hwfn->p_rdma_info;
 974         enum qed_rdma_toggle_bit toggle_bit;
 975         u32 bmap_id;
 976
 977         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", icid);
 978
 979         /* the function toggle the bit that is related to a given icid
 980          * and returns the new toggle bit's value
 981          */
 982         bmap_id = icid - qed_cxt_get_proto_cid_start(p_hwfn, p_info->proto);
 983
 984         spin_lock_bh(&p_info->lock);
 985         toggle_bit = !test_and_change_bit(bmap_id,
 986                                           p_info->toggle_bits.bitmap);
 987         spin_unlock_bh(&p_info->lock);
 988
 989         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QED_RDMA_TOGGLE_BIT_= %d\n",
 990                    toggle_bit);
 991
 992         return toggle_bit;
 993 }
 994
 995 static int qed_rdma_create_cq(void *rdma_cxt,
 996                               struct qed_rdma_create_cq_in_params *params,
 997                               u16 *icid)
 998 {
 999         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
1000         struct qed_rdma_info *p_info = p_hwfn->p_rdma_info;
1001         struct rdma_create_cq_ramrod_data *p_ramrod;
1002         enum qed_rdma_toggle_bit toggle_bit;
1003         struct qed_sp_init_data init_data;
1004         struct qed_spq_entry *p_ent;
1005         u32 returned_id, start_cid;
1006         int rc;
1007
1008         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "cq_handle = %08x%08x\n",
1009                    params->cq_handle_hi, params->cq_handle_lo);
1010
1011         /* Allocate icid */
1012         spin_lock_bh(&p_info->lock);
1013         rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_info->cq_map, &returned_id);
1014         spin_unlock_bh(&p_info->lock);
1015
1016         if (rc) {
1017                 DP_NOTICE(p_hwfn, "Can't create CQ, rc = %d\n", rc);
1018                 return rc;
1019         }
1020
1021         start_cid = qed_cxt_get_proto_cid_start(p_hwfn,
1022                                                 p_info->proto);
1023         *icid = returned_id + start_cid;
1024
1025         /* Check if icid requires a page allocation */
1026         rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *icid);
1027         if (rc)
1028                 goto err;
1029
1030         /* Get SPQ entry */
1031         memset(&init_data, 0, sizeof(init_data));
1032         init_data.cid = *icid;
1033         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1034         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1035
1036         /* Send create CQ ramrod */
1037         rc = qed_sp_init_request(p_hwfn, &p_ent,
1038                                  RDMA_RAMROD_CREATE_CQ,
1039                                  p_info->proto, &init_data);
1040         if (rc)
1041                 goto err;
1042
1043         p_ramrod = &p_ent->ramrod.rdma_create_cq;
1044
1045         p_ramrod->cq_handle.hi = cpu_to_le32(params->cq_handle_hi);
1046         p_ramrod->cq_handle.lo = cpu_to_le32(params->cq_handle_lo);
1047         p_ramrod->dpi = cpu_to_le16(params->dpi);
1048         p_ramrod->is_two_level_pbl = params->pbl_two_level;
1049         p_ramrod->max_cqes = cpu_to_le32(params->cq_size);
1050         DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr);
1051         p_ramrod->pbl_num_pages = cpu_to_le16(params->pbl_num_pages);
1052         p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM) +
1053                            params->cnq_id;
1054         p_ramrod->int_timeout = params->int_timeout;
1055
1056         /* toggle the bit for every resize or create cq for a given icid */
1057         toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid);
1058
1059         p_ramrod->toggle_bit = toggle_bit;
1060
1061         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1062         if (rc) {
1063                 /* restore toggle bit */
1064                 qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid);
1065                 goto err;
1066         }
1067
1068         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Created CQ, rc = %d\n", rc);
1069         return rc;
1070
1071 err:
1072         /* release allocated icid */
1073         spin_lock_bh(&p_info->lock);
1074         qed_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id);
1075         spin_unlock_bh(&p_info->lock);
1076         DP_NOTICE(p_hwfn, "Create CQ failed, rc = %d\n", rc);
1077
1078         return rc;
1079 }
1080
1081 static int
1082 qed_rdma_destroy_cq(void *rdma_cxt,
1083                     struct qed_rdma_destroy_cq_in_params *in_params,
1084                     struct qed_rdma_destroy_cq_out_params *out_params)
1085 {
1086         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
1087         struct rdma_destroy_cq_output_params *p_ramrod_res;
1088         struct rdma_destroy_cq_ramrod_data *p_ramrod;
1089         struct qed_sp_init_data init_data;
1090         struct qed_spq_entry *p_ent;
1091         dma_addr_t ramrod_res_phys;
1092         int rc = -ENOMEM;
1093
1094         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid);
1095
1096         p_ramrod_res =
1097             (struct rdma_destroy_cq_output_params *)
1098             dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1099                                sizeof(struct rdma_destroy_cq_output_params),
1100                                &ramrod_res_phys, GFP_KERNEL);
1101         if (!p_ramrod_res) {
1102                 DP_NOTICE(p_hwfn,
1103                           "qed destroy cq failed: cannot allocate memory (ramrod)\n");
1104                 return rc;
1105         }
1106
1107         /* Get SPQ entry */
1108         memset(&init_data, 0, sizeof(init_data));
1109         init_data.cid = in_params->icid;
1110         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1111         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1112
1113         /* Send destroy CQ ramrod */
1114         rc = qed_sp_init_request(p_hwfn, &p_ent,
1115                                  RDMA_RAMROD_DESTROY_CQ,
1116                                  p_hwfn->p_rdma_info->proto, &init_data);
1117         if (rc)
1118                 goto err;
1119
1120         p_ramrod = &p_ent->ramrod.rdma_destroy_cq;
1121         DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
1122
1123         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1124         if (rc)
1125                 goto err;
1126
1127         out_params->num_cq_notif = le16_to_cpu(p_ramrod_res->cnq_num);
1128
1129         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1130                           sizeof(struct rdma_destroy_cq_output_params),
1131                           p_ramrod_res, ramrod_res_phys);
1132
1133         /* Free icid */
1134         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
1135
1136         qed_bmap_release_id(p_hwfn,
1137                             &p_hwfn->p_rdma_info->cq_map,
1138                             (in_params->icid -
1139                              qed_cxt_get_proto_cid_start(p_hwfn,
1140                                                          p_hwfn->
1141                                                          p_rdma_info->proto)));
1142
1143         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
1144
1145         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc);
1146         return rc;
1147
1148 err:    dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1149                           sizeof(struct rdma_destroy_cq_output_params),
1150                           p_ramrod_res, ramrod_res_phys);
1151
1152         return rc;
1153 }
1154
1155 static void qed_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_qed_mac)
1156 {
1157         p_fw_mac[0] = cpu_to_le16((p_qed_mac[0] << 8) + p_qed_mac[1]);
1158         p_fw_mac[1] = cpu_to_le16((p_qed_mac[2] << 8) + p_qed_mac[3]);
1159         p_fw_mac[2] = cpu_to_le16((p_qed_mac[4] << 8) + p_qed_mac[5]);
1160 }
1161
1162 static void qed_rdma_copy_gids(struct qed_rdma_qp *qp, __le32 *src_gid,
1163                                __le32 *dst_gid)
1164 {
1165         u32 i;
1166
1167         if (qp->roce_mode == ROCE_V2_IPV4) {
1168                 /* The IPv4 addresses shall be aligned to the highest word.
1169                  * The lower words must be zero.
1170                  */
1171                 memset(src_gid, 0, sizeof(union qed_gid));
1172                 memset(dst_gid, 0, sizeof(union qed_gid));
1173                 src_gid[3] = cpu_to_le32(qp->sgid.ipv4_addr);
1174                 dst_gid[3] = cpu_to_le32(qp->dgid.ipv4_addr);
1175         } else {
1176                 /* GIDs and IPv6 addresses coincide in location and size */
1177                 for (i = 0; i < ARRAY_SIZE(qp->sgid.dwords); i++) {
1178                         src_gid[i] = cpu_to_le32(qp->sgid.dwords[i]);
1179                         dst_gid[i] = cpu_to_le32(qp->dgid.dwords[i]);
1180                 }
1181         }
1182 }
1183
1184 static enum roce_flavor qed_roce_mode_to_flavor(enum roce_mode roce_mode)
1185 {
1186         enum roce_flavor flavor;
1187
1188         switch (roce_mode) {
1189         case ROCE_V1:
1190                 flavor = PLAIN_ROCE;
1191                 break;
1192         case ROCE_V2_IPV4:
1193                 flavor = RROCE_IPV4;
1194                 break;
1195         case ROCE_V2_IPV6:
1196                 flavor = ROCE_V2_IPV6;
1197                 break;
1198         default:
1199                 flavor = MAX_ROCE_MODE;
1200                 break;
1201         }
1202         return flavor;
1203 }
1204
1205 void qed_roce_free_cid_pair(struct qed_hwfn *p_hwfn, u16 cid)
1206 {
1207         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
1208         qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid);
1209         qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid + 1);
1210         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
1211 }
1212
1213 static int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid)
1214 {
1215         struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
1216         u32 responder_icid;
1217         u32 requester_icid;
1218         int rc;
1219
1220         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
1221         rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map,
1222                                     &responder_icid);
1223         if (rc) {
1224                 spin_unlock_bh(&p_rdma_info->lock);
1225                 return rc;
1226         }
1227
1228         rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map,
1229                                     &requester_icid);
1230
1231         spin_unlock_bh(&p_rdma_info->lock);
1232         if (rc)
1233                 goto err;
1234
1235         /* the two icid's should be adjacent */
1236         if ((requester_icid - responder_icid) != 1) {
1237                 DP_NOTICE(p_hwfn, "Failed to allocate two adjacent qp's'\n");
1238                 rc = -EINVAL;
1239                 goto err;
1240         }
1241
1242         responder_icid += qed_cxt_get_proto_cid_start(p_hwfn,
1243                                                       p_rdma_info->proto);
1244         requester_icid += qed_cxt_get_proto_cid_start(p_hwfn,
1245                                                       p_rdma_info->proto);
1246
1247         /* If these icids require a new ILT line allocate DMA-able context for
1248          * an ILT page
1249          */
1250         rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, responder_icid);
1251         if (rc)
1252                 goto err;
1253
1254         rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, requester_icid);
1255         if (rc)
1256                 goto err;
1257
1258         *cid = (u16)responder_icid;
1259         return rc;
1260
1261 err:
1262         spin_lock_bh(&p_rdma_info->lock);
1263         qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, responder_icid);
1264         qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, requester_icid);
1265
1266         spin_unlock_bh(&p_rdma_info->lock);
1267         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
1268                    "Allocate CID - failed, rc = %d\n", rc);
1269         return rc;
1270 }
1271
1272 static void qed_roce_set_real_cid(struct qed_hwfn *p_hwfn, u32 cid)
1273 {
1274         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
1275         qed_bmap_set_id(p_hwfn, &p_hwfn->p_rdma_info->real_cid_map, cid);
1276         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
1277 }
1278
1279 static int qed_roce_sp_create_responder(struct qed_hwfn *p_hwfn,
1280                                         struct qed_rdma_qp *qp)
1281 {
1282         struct roce_create_qp_resp_ramrod_data *p_ramrod;
1283         struct qed_sp_init_data init_data;
1284         enum roce_flavor roce_flavor;
1285         struct qed_spq_entry *p_ent;
1286         u16 regular_latency_queue;
1287         enum protocol_type proto;
1288         int rc;
1289
1290         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
1291
1292         /* Allocate DMA-able memory for IRQ */
1293         qp->irq_num_pages = 1;
1294         qp->irq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1295                                      RDMA_RING_PAGE_SIZE,
1296                                      &qp->irq_phys_addr, GFP_KERNEL);
1297         if (!qp->irq) {
1298                 rc = -ENOMEM;
1299                 DP_NOTICE(p_hwfn,
1300                           "qed create responder failed: cannot allocate memory (irq). rc = %d\n",
1301                           rc);
1302                 return rc;
1303         }
1304
1305         /* Get SPQ entry */
1306         memset(&init_data, 0, sizeof(init_data));
1307         init_data.cid = qp->icid;
1308         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1309         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1310
1311         rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP,
1312                                  PROTOCOLID_ROCE, &init_data);
1313         if (rc)
1314                 goto err;
1315
1316         p_ramrod = &p_ent->ramrod.roce_create_qp_resp;
1317
1318         p_ramrod->flags = 0;
1319
1320         roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode);
1321         SET_FIELD(p_ramrod->flags,
1322                   ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR, roce_flavor);
1323
1324         SET_FIELD(p_ramrod->flags,
1325                   ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN,
1326                   qp->incoming_rdma_read_en);
1327
1328         SET_FIELD(p_ramrod->flags,
1329                   ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN,
1330                   qp->incoming_rdma_write_en);
1331
1332         SET_FIELD(p_ramrod->flags,
1333                   ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN,
1334                   qp->incoming_atomic_en);
1335
1336         SET_FIELD(p_ramrod->flags,
1337                   ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN,
1338                   qp->e2e_flow_control_en);
1339
1340         SET_FIELD(p_ramrod->flags,
1341                   ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, qp->use_srq);
1342
1343         SET_FIELD(p_ramrod->flags,
1344                   ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN,
1345                   qp->fmr_and_reserved_lkey);
1346
1347         SET_FIELD(p_ramrod->flags,
1348                   ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER,
1349                   qp->min_rnr_nak_timer);
1350
1351         p_ramrod->max_ird = qp->max_rd_atomic_resp;
1352         p_ramrod->traffic_class = qp->traffic_class_tos;
1353         p_ramrod->hop_limit = qp->hop_limit_ttl;
1354         p_ramrod->irq_num_pages = qp->irq_num_pages;
1355         p_ramrod->p_key = cpu_to_le16(qp->pkey);
1356         p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
1357         p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp);
1358         p_ramrod->mtu = cpu_to_le16(qp->mtu);
1359         p_ramrod->initial_psn = cpu_to_le32(qp->rq_psn);
1360         p_ramrod->pd = cpu_to_le16(qp->pd);
1361         p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages);
1362         DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr);
1363         DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr);
1364         qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
1365         p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi);
1366         p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo);
1367         p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi);
1368         p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo);
1369         p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) |
1370                                        qp->rq_cq_id);
1371
1372         regular_latency_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
1373
1374         p_ramrod->regular_latency_phy_queue =
1375             cpu_to_le16(regular_latency_queue);
1376         p_ramrod->low_latency_phy_queue =
1377             cpu_to_le16(regular_latency_queue);
1378
1379         p_ramrod->dpi = cpu_to_le16(qp->dpi);
1380
1381         qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr);
1382         qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr);
1383
1384         p_ramrod->udp_src_port = qp->udp_src_port;
1385         p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id);
1386         p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id);
1387         p_ramrod->srq_id.opaque_fid = cpu_to_le16(p_hwfn->hw_info.opaque_fid);
1388
1389         p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) +
1390                                      qp->stats_queue;
1391
1392         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1393
1394         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
1395                    "rc = %d regular physical queue = 0x%x\n", rc,
1396                    regular_latency_queue);
1397
1398         if (rc)
1399                 goto err;
1400
1401         qp->resp_offloaded = true;
1402         qp->cq_prod = 0;
1403
1404         proto = p_hwfn->p_rdma_info->proto;
1405         qed_roce_set_real_cid(p_hwfn, qp->icid -
1406                               qed_cxt_get_proto_cid_start(p_hwfn, proto));
1407
1408         return rc;
1409
1410 err:
1411         DP_NOTICE(p_hwfn, "create responder - failed, rc = %d\n", rc);
1412         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1413                           qp->irq_num_pages * RDMA_RING_PAGE_SIZE,
1414                           qp->irq, qp->irq_phys_addr);
1415
1416         return rc;
1417 }
1418
1419 static int qed_roce_sp_create_requester(struct qed_hwfn *p_hwfn,
1420                                         struct qed_rdma_qp *qp)
1421 {
1422         struct roce_create_qp_req_ramrod_data *p_ramrod;
1423         struct qed_sp_init_data init_data;
1424         enum roce_flavor roce_flavor;
1425         struct qed_spq_entry *p_ent;
1426         u16 regular_latency_queue;
1427         enum protocol_type proto;
1428         int rc;
1429
1430         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
1431
1432         /* Allocate DMA-able memory for ORQ */
1433         qp->orq_num_pages = 1;
1434         qp->orq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1435                                      RDMA_RING_PAGE_SIZE,
1436                                      &qp->orq_phys_addr, GFP_KERNEL);
1437         if (!qp->orq) {
1438                 rc = -ENOMEM;
1439                 DP_NOTICE(p_hwfn,
1440                           "qed create requester failed: cannot allocate memory (orq). rc = %d\n",
1441                           rc);
1442                 return rc;
1443         }
1444
1445         /* Get SPQ entry */
1446         memset(&init_data, 0, sizeof(init_data));
1447         init_data.cid = qp->icid + 1;
1448         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1449         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1450
1451         rc = qed_sp_init_request(p_hwfn, &p_ent,
1452                                  ROCE_RAMROD_CREATE_QP,
1453                                  PROTOCOLID_ROCE, &init_data);
1454         if (rc)
1455                 goto err;
1456
1457         p_ramrod = &p_ent->ramrod.roce_create_qp_req;
1458
1459         p_ramrod->flags = 0;
1460
1461         roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode);
1462         SET_FIELD(p_ramrod->flags,
1463                   ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR, roce_flavor);
1464
1465         SET_FIELD(p_ramrod->flags,
1466                   ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN,
1467                   qp->fmr_and_reserved_lkey);
1468
1469         SET_FIELD(p_ramrod->flags,
1470                   ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, qp->signal_all);
1471
1472         SET_FIELD(p_ramrod->flags,
1473                   ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt);
1474
1475         SET_FIELD(p_ramrod->flags,
1476                   ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT,
1477                   qp->rnr_retry_cnt);
1478
1479         p_ramrod->max_ord = qp->max_rd_atomic_req;
1480         p_ramrod->traffic_class = qp->traffic_class_tos;
1481         p_ramrod->hop_limit = qp->hop_limit_ttl;
1482         p_ramrod->orq_num_pages = qp->orq_num_pages;
1483         p_ramrod->p_key = cpu_to_le16(qp->pkey);
1484         p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
1485         p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp);
1486         p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout);
1487         p_ramrod->mtu = cpu_to_le16(qp->mtu);
1488         p_ramrod->initial_psn = cpu_to_le32(qp->sq_psn);
1489         p_ramrod->pd = cpu_to_le16(qp->pd);
1490         p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages);
1491         DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr);
1492         DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr);
1493         qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
1494         p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi);
1495         p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo);
1496         p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi);
1497         p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo);
1498         p_ramrod->cq_cid =
1499             cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id);
1500
1501         regular_latency_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD);
1502
1503         p_ramrod->regular_latency_phy_queue =
1504             cpu_to_le16(regular_latency_queue);
1505         p_ramrod->low_latency_phy_queue =
1506             cpu_to_le16(regular_latency_queue);
1507
1508         p_ramrod->dpi = cpu_to_le16(qp->dpi);
1509
1510         qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr);
1511         qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr);
1512
1513         p_ramrod->udp_src_port = qp->udp_src_port;
1514         p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id);
1515         p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) +
1516                                      qp->stats_queue;
1517
1518         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1519
1520         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
1521
1522         if (rc)
1523                 goto err;
1524
1525         qp->req_offloaded = true;
1526         proto = p_hwfn->p_rdma_info->proto;
1527         qed_roce_set_real_cid(p_hwfn,
1528                               qp->icid + 1 -
1529                               qed_cxt_get_proto_cid_start(p_hwfn, proto));
1530
1531         return rc;
1532
1533 err:
1534         DP_NOTICE(p_hwfn, "Create requested - failed, rc = %d\n", rc);
1535         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1536                           qp->orq_num_pages * RDMA_RING_PAGE_SIZE,
1537                           qp->orq, qp->orq_phys_addr);
1538         return rc;
1539 }
1540
1541 static int qed_roce_sp_modify_responder(struct qed_hwfn *p_hwfn,
1542                                         struct qed_rdma_qp *qp,
1543                                         bool move_to_err, u32 modify_flags)
1544 {
1545         struct roce_modify_qp_resp_ramrod_data *p_ramrod;
1546         struct qed_sp_init_data init_data;
1547         struct qed_spq_entry *p_ent;
1548         int rc;
1549
1550         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
1551
1552         if (move_to_err && !qp->resp_offloaded)
1553                 return 0;
1554
1555         /* Get SPQ entry */
1556         memset(&init_data, 0, sizeof(init_data));
1557         init_data.cid = qp->icid;
1558         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1559         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1560
1561         rc = qed_sp_init_request(p_hwfn, &p_ent,
1562                                  ROCE_EVENT_MODIFY_QP,
1563                                  PROTOCOLID_ROCE, &init_data);
1564         if (rc) {
1565                 DP_NOTICE(p_hwfn, "rc = %d\n", rc);
1566                 return rc;
1567         }
1568
1569         p_ramrod = &p_ent->ramrod.roce_modify_qp_resp;
1570
1571         p_ramrod->flags = 0;
1572
1573         SET_FIELD(p_ramrod->flags,
1574                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err);
1575
1576         SET_FIELD(p_ramrod->flags,
1577                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN,
1578                   qp->incoming_rdma_read_en);
1579
1580         SET_FIELD(p_ramrod->flags,
1581                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN,
1582                   qp->incoming_rdma_write_en);
1583
1584         SET_FIELD(p_ramrod->flags,
1585                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN,
1586                   qp->incoming_atomic_en);
1587
1588         SET_FIELD(p_ramrod->flags,
1589                   ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN,
1590                   qp->e2e_flow_control_en);
1591
1592         SET_FIELD(p_ramrod->flags,
1593                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG,
1594                   GET_FIELD(modify_flags,
1595                             QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN));
1596
1597         SET_FIELD(p_ramrod->flags,
1598                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG,
1599                   GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY));
1600
1601         SET_FIELD(p_ramrod->flags,
1602                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG,
1603                   GET_FIELD(modify_flags,
1604                             QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR));
1605
1606         SET_FIELD(p_ramrod->flags,
1607                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG,
1608                   GET_FIELD(modify_flags,
1609                             QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP));
1610
1611         SET_FIELD(p_ramrod->flags,
1612                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG,
1613                   GET_FIELD(modify_flags,
1614                             QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER));
1615
1616         p_ramrod->fields = 0;
1617         SET_FIELD(p_ramrod->fields,
1618                   ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER,
1619                   qp->min_rnr_nak_timer);
1620
1621         p_ramrod->max_ird = qp->max_rd_atomic_resp;
1622         p_ramrod->traffic_class = qp->traffic_class_tos;
1623         p_ramrod->hop_limit = qp->hop_limit_ttl;
1624         p_ramrod->p_key = cpu_to_le16(qp->pkey);
1625         p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
1626         p_ramrod->mtu = cpu_to_le16(qp->mtu);
1627         qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
1628         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1629
1630         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify responder, rc = %d\n", rc);
1631         return rc;
1632 }
1633
1634 static int qed_roce_sp_modify_requester(struct qed_hwfn *p_hwfn,
1635                                         struct qed_rdma_qp *qp,
1636                                         bool move_to_sqd,
1637                                         bool move_to_err, u32 modify_flags)
1638 {
1639         struct roce_modify_qp_req_ramrod_data *p_ramrod;
1640         struct qed_sp_init_data init_data;
1641         struct qed_spq_entry *p_ent;
1642         int rc;
1643
1644         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
1645
1646         if (move_to_err && !(qp->req_offloaded))
1647                 return 0;
1648
1649         /* Get SPQ entry */
1650         memset(&init_data, 0, sizeof(init_data));
1651         init_data.cid = qp->icid + 1;
1652         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1653         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1654
1655         rc = qed_sp_init_request(p_hwfn, &p_ent,
1656                                  ROCE_EVENT_MODIFY_QP,
1657                                  PROTOCOLID_ROCE, &init_data);
1658         if (rc) {
1659                 DP_NOTICE(p_hwfn, "rc = %d\n", rc);
1660                 return rc;
1661         }
1662
1663         p_ramrod = &p_ent->ramrod.roce_modify_qp_req;
1664
1665         p_ramrod->flags = 0;
1666
1667         SET_FIELD(p_ramrod->flags,
1668                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err);
1669
1670         SET_FIELD(p_ramrod->flags,
1671                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG, move_to_sqd);
1672
1673         SET_FIELD(p_ramrod->flags,
1674                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY,
1675                   qp->sqd_async);
1676
1677         SET_FIELD(p_ramrod->flags,
1678                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG,
1679                   GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY));
1680
1681         SET_FIELD(p_ramrod->flags,
1682                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG,
1683                   GET_FIELD(modify_flags,
1684                             QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR));
1685
1686         SET_FIELD(p_ramrod->flags,
1687                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG,
1688                   GET_FIELD(modify_flags,
1689                             QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ));
1690
1691         SET_FIELD(p_ramrod->flags,
1692                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG,
1693                   GET_FIELD(modify_flags,
1694                             QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT));
1695
1696         SET_FIELD(p_ramrod->flags,
1697                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG,
1698                   GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT));
1699
1700         SET_FIELD(p_ramrod->flags,
1701                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG,
1702                   GET_FIELD(modify_flags,
1703                             QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT));
1704
1705         p_ramrod->fields = 0;
1706         SET_FIELD(p_ramrod->fields,
1707                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt);
1708
1709         SET_FIELD(p_ramrod->fields,
1710                   ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT,
1711                   qp->rnr_retry_cnt);
1712
1713         p_ramrod->max_ord = qp->max_rd_atomic_req;
1714         p_ramrod->traffic_class = qp->traffic_class_tos;
1715         p_ramrod->hop_limit = qp->hop_limit_ttl;
1716         p_ramrod->p_key = cpu_to_le16(qp->pkey);
1717         p_ramrod->flow_label = cpu_to_le32(qp->flow_label);
1718         p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout);
1719         p_ramrod->mtu = cpu_to_le16(qp->mtu);
1720         qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid);
1721         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1722
1723         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify requester, rc = %d\n", rc);
1724         return rc;
1725 }
1726
1727 static int qed_roce_sp_destroy_qp_responder(struct qed_hwfn *p_hwfn,
1728                                             struct qed_rdma_qp *qp,
1729                                             u32 *num_invalidated_mw,
1730                                             u32 *cq_prod)
1731 {
1732         struct roce_destroy_qp_resp_output_params *p_ramrod_res;
1733         struct roce_destroy_qp_resp_ramrod_data *p_ramrod;
1734         struct qed_sp_init_data init_data;
1735         struct qed_spq_entry *p_ent;
1736         dma_addr_t ramrod_res_phys;
1737         int rc;
1738
1739         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
1740
1741         *num_invalidated_mw = 0;
1742         *cq_prod = qp->cq_prod;
1743
1744         if (!qp->resp_offloaded) {
1745                 /* If a responder was never offload, we need to free the cids
1746                  * allocated in create_qp as a FW async event will never arrive
1747                  */
1748                 u32 cid;
1749
1750                 cid = qp->icid -
1751                       qed_cxt_get_proto_cid_start(p_hwfn,
1752                                                   p_hwfn->p_rdma_info->proto);
1753                 qed_roce_free_cid_pair(p_hwfn, (u16)cid);
1754
1755                 return 0;
1756         }
1757
1758         /* Get SPQ entry */
1759         memset(&init_data, 0, sizeof(init_data));
1760         init_data.cid = qp->icid;
1761         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1762         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1763
1764         rc = qed_sp_init_request(p_hwfn, &p_ent,
1765                                  ROCE_RAMROD_DESTROY_QP,
1766                                  PROTOCOLID_ROCE, &init_data);
1767         if (rc)
1768                 return rc;
1769
1770         p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp;
1771
1772         p_ramrod_res = (struct roce_destroy_qp_resp_output_params *)
1773             dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res),
1774                                &ramrod_res_phys, GFP_KERNEL);
1775
1776         if (!p_ramrod_res) {
1777                 rc = -ENOMEM;
1778                 DP_NOTICE(p_hwfn,
1779                           "qed destroy responder failed: cannot allocate memory (ramrod). rc = %d\n",
1780                           rc);
1781                 return rc;
1782         }
1783
1784         DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
1785
1786         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1787         if (rc)
1788                 goto err;
1789
1790         *num_invalidated_mw = le32_to_cpu(p_ramrod_res->num_invalidated_mw);
1791         *cq_prod = le32_to_cpu(p_ramrod_res->cq_prod);
1792         qp->cq_prod = *cq_prod;
1793
1794         /* Free IRQ - only if ramrod succeeded, in case FW is still using it */
1795         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1796                           qp->irq_num_pages * RDMA_RING_PAGE_SIZE,
1797                           qp->irq, qp->irq_phys_addr);
1798
1799         qp->resp_offloaded = false;
1800
1801         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy responder, rc = %d\n", rc);
1802
1803 err:
1804         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1805                           sizeof(struct roce_destroy_qp_resp_output_params),
1806                           p_ramrod_res, ramrod_res_phys);
1807
1808         return rc;
1809 }
1810
1811 static int qed_roce_sp_destroy_qp_requester(struct qed_hwfn *p_hwfn,
1812                                             struct qed_rdma_qp *qp,
1813                                             u32 *num_bound_mw)
1814 {
1815         struct roce_destroy_qp_req_output_params *p_ramrod_res;
1816         struct roce_destroy_qp_req_ramrod_data *p_ramrod;
1817         struct qed_sp_init_data init_data;
1818         struct qed_spq_entry *p_ent;
1819         dma_addr_t ramrod_res_phys;
1820         int rc = -ENOMEM;
1821
1822         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
1823
1824         if (!qp->req_offloaded)
1825                 return 0;
1826
1827         p_ramrod_res = (struct roce_destroy_qp_req_output_params *)
1828                        dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1829                                           sizeof(*p_ramrod_res),
1830                                           &ramrod_res_phys, GFP_KERNEL);
1831         if (!p_ramrod_res) {
1832                 DP_NOTICE(p_hwfn,
1833                           "qed destroy requester failed: cannot allocate memory (ramrod)\n");
1834                 return rc;
1835         }
1836
1837         /* Get SPQ entry */
1838         memset(&init_data, 0, sizeof(init_data));
1839         init_data.cid = qp->icid + 1;
1840         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1841         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1842
1843         rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP,
1844                                  PROTOCOLID_ROCE, &init_data);
1845         if (rc)
1846                 goto err;
1847
1848         p_ramrod = &p_ent->ramrod.roce_destroy_qp_req;
1849         DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys);
1850
1851         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1852         if (rc)
1853                 goto err;
1854
1855         *num_bound_mw = le32_to_cpu(p_ramrod_res->num_bound_mw);
1856
1857         /* Free ORQ - only if ramrod succeeded, in case FW is still using it */
1858         dma_free_coherent(&p_hwfn->cdev->pdev->dev,
1859                           qp->orq_num_pages * RDMA_RING_PAGE_SIZE,
1860                           qp->orq, qp->orq_phys_addr);
1861
1862         qp->req_offloaded = false;
1863
1864         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy requester, rc = %d\n", rc);
1865
1866 err:
1867         dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res),
1868                           p_ramrod_res, ramrod_res_phys);
1869
1870         return rc;
1871 }
1872
1873 static int qed_roce_query_qp(struct qed_hwfn *p_hwfn,
1874                              struct qed_rdma_qp *qp,
1875                              struct qed_rdma_query_qp_out_params *out_params)
1876 {
1877         struct roce_query_qp_resp_output_params *p_resp_ramrod_res;
1878         struct roce_query_qp_req_output_params *p_req_ramrod_res;
1879         struct roce_query_qp_resp_ramrod_data *p_resp_ramrod;
1880         struct roce_query_qp_req_ramrod_data *p_req_ramrod;
1881         struct qed_sp_init_data init_data;
1882         dma_addr_t resp_ramrod_res_phys;
1883         dma_addr_t req_ramrod_res_phys;
1884         struct qed_spq_entry *p_ent;
1885         bool rq_err_state;
1886         bool sq_err_state;
1887         bool sq_draining;
1888         int rc = -ENOMEM;
1889
1890         if ((!(qp->resp_offloaded)) && (!(qp->req_offloaded))) {
1891                 /* We can't send ramrod to the fw since this qp wasn't offloaded
1892                  * to the fw yet
1893                  */
1894                 out_params->draining = false;
1895                 out_params->rq_psn = qp->rq_psn;
1896                 out_params->sq_psn = qp->sq_psn;
1897                 out_params->state = qp->cur_state;
1898
1899                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "No QPs as no offload\n");
1900                 return 0;
1901         }
1902
1903         if (!(qp->resp_offloaded)) {
1904                 DP_NOTICE(p_hwfn,
1905                           "The responder's qp should be offloded before requester's\n");
1906                 return -EINVAL;
1907         }
1908
1909         /* Send a query responder ramrod to FW to get RQ-PSN and state */
1910         p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *)
1911             dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1912                                sizeof(*p_resp_ramrod_res),
1913                                &resp_ramrod_res_phys, GFP_KERNEL);
1914         if (!p_resp_ramrod_res) {
1915                 DP_NOTICE(p_hwfn,
1916                           "qed query qp failed: cannot allocate memory (ramrod)\n");
1917                 return rc;
1918         }
1919
1920         /* Get SPQ entry */
1921         memset(&init_data, 0, sizeof(init_data));
1922         init_data.cid = qp->icid;
1923         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
1924         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
1925         rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP,
1926                                  PROTOCOLID_ROCE, &init_data);
1927         if (rc)
1928                 goto err_resp;
1929
1930         p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp;
1931         DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys);
1932
1933         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1934         if (rc)
1935                 goto err_resp;
1936
1937         out_params->rq_psn = le32_to_cpu(p_resp_ramrod_res->psn);
1938         rq_err_state = GET_FIELD(le32_to_cpu(p_resp_ramrod_res->err_flag),
1939                                  ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG);
1940
1941         dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res),
1942                           p_resp_ramrod_res, resp_ramrod_res_phys);
1943
1944         if (!(qp->req_offloaded)) {
1945                 /* Don't send query qp for the requester */
1946                 out_params->sq_psn = qp->sq_psn;
1947                 out_params->draining = false;
1948
1949                 if (rq_err_state)
1950                         qp->cur_state = QED_ROCE_QP_STATE_ERR;
1951
1952                 out_params->state = qp->cur_state;
1953
1954                 return 0;
1955         }
1956
1957         /* Send a query requester ramrod to FW to get SQ-PSN and state */
1958         p_req_ramrod_res = (struct roce_query_qp_req_output_params *)
1959                            dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
1960                                               sizeof(*p_req_ramrod_res),
1961                                               &req_ramrod_res_phys,
1962                                               GFP_KERNEL);
1963         if (!p_req_ramrod_res) {
1964                 rc = -ENOMEM;
1965                 DP_NOTICE(p_hwfn,
1966                           "qed query qp failed: cannot allocate memory (ramrod)\n");
1967                 return rc;
1968         }
1969
1970         /* Get SPQ entry */
1971         init_data.cid = qp->icid + 1;
1972         rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP,
1973                                  PROTOCOLID_ROCE, &init_data);
1974         if (rc)
1975                 goto err_req;
1976
1977         p_req_ramrod = &p_ent->ramrod.roce_query_qp_req;
1978         DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys);
1979
1980         rc = qed_spq_post(p_hwfn, p_ent, NULL);
1981         if (rc)
1982                 goto err_req;
1983
1984         out_params->sq_psn = le32_to_cpu(p_req_ramrod_res->psn);
1985         sq_err_state = GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags),
1986                                  ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG);
1987         sq_draining =
1988                 GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags),
1989                           ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG);
1990
1991         dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res),
1992                           p_req_ramrod_res, req_ramrod_res_phys);
1993
1994         out_params->draining = false;
1995
1996         if (rq_err_state || sq_err_state)
1997                 qp->cur_state = QED_ROCE_QP_STATE_ERR;
1998         else if (sq_draining)
1999                 out_params->draining = true;
2000         out_params->state = qp->cur_state;
2001
2002         return 0;
2003
2004 err_req:
2005         dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res),
2006                           p_req_ramrod_res, req_ramrod_res_phys);
2007         return rc;
2008 err_resp:
2009         dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res),
2010                           p_resp_ramrod_res, resp_ramrod_res_phys);
2011         return rc;
2012 }
2013
2014 static int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp)
2015 {
2016         u32 num_invalidated_mw = 0;
2017         u32 num_bound_mw = 0;
2018         u32 cq_prod;
2019         int rc;
2020
2021         /* Destroys the specified QP */
2022         if ((qp->cur_state != QED_ROCE_QP_STATE_RESET) &&
2023             (qp->cur_state != QED_ROCE_QP_STATE_ERR) &&
2024             (qp->cur_state != QED_ROCE_QP_STATE_INIT)) {
2025                 DP_NOTICE(p_hwfn,
2026                           "QP must be in error, reset or init state before destroying it\n");
2027                 return -EINVAL;
2028         }
2029
2030         if (qp->cur_state != QED_ROCE_QP_STATE_RESET) {
2031                 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp,
2032                                                       &num_invalidated_mw,
2033                                                       &cq_prod);
2034                 if (rc)
2035                         return rc;
2036
2037                 /* Send destroy requester ramrod */
2038                 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp,
2039                                                       &num_bound_mw);
2040                 if (rc)
2041                         return rc;
2042
2043                 if (num_invalidated_mw != num_bound_mw) {
2044                         DP_NOTICE(p_hwfn,
2045                                   "number of invalidate memory windows is different from bounded ones\n");
2046                         return -EINVAL;
2047                 }
2048         }
2049
2050         return 0;
2051 }
2052
2053 static int qed_rdma_query_qp(void *rdma_cxt,
2054                              struct qed_rdma_qp *qp,
2055                              struct qed_rdma_query_qp_out_params *out_params)
2056 {
2057         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2058         int rc;
2059
2060         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
2061
2062         /* The following fields are filled in from qp and not FW as they can't
2063          * be modified by FW
2064          */
2065         out_params->mtu = qp->mtu;
2066         out_params->dest_qp = qp->dest_qp;
2067         out_params->incoming_atomic_en = qp->incoming_atomic_en;
2068         out_params->e2e_flow_control_en = qp->e2e_flow_control_en;
2069         out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en;
2070         out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en;
2071         out_params->dgid = qp->dgid;
2072         out_params->flow_label = qp->flow_label;
2073         out_params->hop_limit_ttl = qp->hop_limit_ttl;
2074         out_params->traffic_class_tos = qp->traffic_class_tos;
2075         out_params->timeout = qp->ack_timeout;
2076         out_params->rnr_retry = qp->rnr_retry_cnt;
2077         out_params->retry_cnt = qp->retry_cnt;
2078         out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer;
2079         out_params->pkey_index = 0;
2080         out_params->max_rd_atomic = qp->max_rd_atomic_req;
2081         out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp;
2082         out_params->sqd_async = qp->sqd_async;
2083
2084         rc = qed_roce_query_qp(p_hwfn, qp, out_params);
2085
2086         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query QP, rc = %d\n", rc);
2087         return rc;
2088 }
2089
2090 static int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp)
2091 {
2092         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2093         int rc = 0;
2094
2095         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid);
2096
2097         rc = qed_roce_destroy_qp(p_hwfn, qp);
2098
2099         /* free qp params struct */
2100         kfree(qp);
2101
2102         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP destroyed\n");
2103         return rc;
2104 }
2105
2106 static struct qed_rdma_qp *
2107 qed_rdma_create_qp(void *rdma_cxt,
2108                    struct qed_rdma_create_qp_in_params *in_params,
2109                    struct qed_rdma_create_qp_out_params *out_params)
2110 {
2111         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2112         struct qed_rdma_qp *qp;
2113         u8 max_stats_queues;
2114         int rc;
2115
2116         if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) {
2117                 DP_ERR(p_hwfn->cdev,
2118                        "qed roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n",
2119                        rdma_cxt, in_params, out_params);
2120                 return NULL;
2121         }
2122
2123         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2124                    "qed rdma create qp called with qp_handle = %08x%08x\n",
2125                    in_params->qp_handle_hi, in_params->qp_handle_lo);
2126
2127         /* Some sanity checks... */
2128         max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues;
2129         if (in_params->stats_queue >= max_stats_queues) {
2130                 DP_ERR(p_hwfn->cdev,
2131                        "qed rdma create qp failed due to invalid statistics queue %d. maximum is %d\n",
2132                        in_params->stats_queue, max_stats_queues);
2133                 return NULL;
2134         }
2135
2136         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2137         if (!qp) {
2138                 DP_NOTICE(p_hwfn, "Failed to allocate qed_rdma_qp\n");
2139                 return NULL;
2140         }
2141
2142         rc = qed_roce_alloc_cid(p_hwfn, &qp->icid);
2143         qp->qpid = ((0xFF << 16) | qp->icid);
2144
2145         DP_INFO(p_hwfn, "ROCE qpid=%x\n", qp->qpid);
2146
2147         if (rc) {
2148                 kfree(qp);
2149                 return NULL;
2150         }
2151
2152         qp->cur_state = QED_ROCE_QP_STATE_RESET;
2153         qp->qp_handle.hi = cpu_to_le32(in_params->qp_handle_hi);
2154         qp->qp_handle.lo = cpu_to_le32(in_params->qp_handle_lo);
2155         qp->qp_handle_async.hi = cpu_to_le32(in_params->qp_handle_async_hi);
2156         qp->qp_handle_async.lo = cpu_to_le32(in_params->qp_handle_async_lo);
2157         qp->use_srq = in_params->use_srq;
2158         qp->signal_all = in_params->signal_all;
2159         qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey;
2160         qp->pd = in_params->pd;
2161         qp->dpi = in_params->dpi;
2162         qp->sq_cq_id = in_params->sq_cq_id;
2163         qp->sq_num_pages = in_params->sq_num_pages;
2164         qp->sq_pbl_ptr = in_params->sq_pbl_ptr;
2165         qp->rq_cq_id = in_params->rq_cq_id;
2166         qp->rq_num_pages = in_params->rq_num_pages;
2167         qp->rq_pbl_ptr = in_params->rq_pbl_ptr;
2168         qp->srq_id = in_params->srq_id;
2169         qp->req_offloaded = false;
2170         qp->resp_offloaded = false;
2171         qp->e2e_flow_control_en = qp->use_srq ? false : true;
2172         qp->stats_queue = in_params->stats_queue;
2173
2174         out_params->icid = qp->icid;
2175         out_params->qp_id = qp->qpid;
2176
2177         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Create QP, rc = %d\n", rc);
2178         return qp;
2179 }
2180
2181 static int qed_roce_modify_qp(struct qed_hwfn *p_hwfn,
2182                               struct qed_rdma_qp *qp,
2183                               enum qed_roce_qp_state prev_state,
2184                               struct qed_rdma_modify_qp_in_params *params)
2185 {
2186         u32 num_invalidated_mw = 0, num_bound_mw = 0;
2187         int rc = 0;
2188
2189         /* Perform additional operations according to the current state and the
2190          * next state
2191          */
2192         if (((prev_state == QED_ROCE_QP_STATE_INIT) ||
2193              (prev_state == QED_ROCE_QP_STATE_RESET)) &&
2194             (qp->cur_state == QED_ROCE_QP_STATE_RTR)) {
2195                 /* Init->RTR or Reset->RTR */
2196                 rc = qed_roce_sp_create_responder(p_hwfn, qp);
2197                 return rc;
2198         } else if ((prev_state == QED_ROCE_QP_STATE_RTR) &&
2199                    (qp->cur_state == QED_ROCE_QP_STATE_RTS)) {
2200                 /* RTR-> RTS */
2201                 rc = qed_roce_sp_create_requester(p_hwfn, qp);
2202                 if (rc)
2203                         return rc;
2204
2205                 /* Send modify responder ramrod */
2206                 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
2207                                                   params->modify_flags);
2208                 return rc;
2209         } else if ((prev_state == QED_ROCE_QP_STATE_RTS) &&
2210                    (qp->cur_state == QED_ROCE_QP_STATE_RTS)) {
2211                 /* RTS->RTS */
2212                 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
2213                                                   params->modify_flags);
2214                 if (rc)
2215                         return rc;
2216
2217                 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false,
2218                                                   params->modify_flags);
2219                 return rc;
2220         } else if ((prev_state == QED_ROCE_QP_STATE_RTS) &&
2221                    (qp->cur_state == QED_ROCE_QP_STATE_SQD)) {
2222                 /* RTS->SQD */
2223                 rc = qed_roce_sp_modify_requester(p_hwfn, qp, true, false,
2224                                                   params->modify_flags);
2225                 return rc;
2226         } else if ((prev_state == QED_ROCE_QP_STATE_SQD) &&
2227                    (qp->cur_state == QED_ROCE_QP_STATE_SQD)) {
2228                 /* SQD->SQD */
2229                 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
2230                                                   params->modify_flags);
2231                 if (rc)
2232                         return rc;
2233
2234                 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false,
2235                                                   params->modify_flags);
2236                 return rc;
2237         } else if ((prev_state == QED_ROCE_QP_STATE_SQD) &&
2238                    (qp->cur_state == QED_ROCE_QP_STATE_RTS)) {
2239                 /* SQD->RTS */
2240                 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false,
2241                                                   params->modify_flags);
2242                 if (rc)
2243                         return rc;
2244
2245                 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false,
2246                                                   params->modify_flags);
2247
2248                 return rc;
2249         } else if (qp->cur_state == QED_ROCE_QP_STATE_ERR) {
2250                 /* ->ERR */
2251                 rc = qed_roce_sp_modify_responder(p_hwfn, qp, true,
2252                                                   params->modify_flags);
2253                 if (rc)
2254                         return rc;
2255
2256                 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, true,
2257                                                   params->modify_flags);
2258                 return rc;
2259         } else if (qp->cur_state == QED_ROCE_QP_STATE_RESET) {
2260                 /* Any state -> RESET */
2261                 u32 cq_prod;
2262
2263                 /* Send destroy responder ramrod */
2264                 rc = qed_roce_sp_destroy_qp_responder(p_hwfn,
2265                                                       qp,
2266                                                       &num_invalidated_mw,
2267                                                       &cq_prod);
2268
2269                 if (rc)
2270                         return rc;
2271
2272                 qp->cq_prod = cq_prod;
2273
2274                 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp,
2275                                                       &num_bound_mw);
2276
2277                 if (num_invalidated_mw != num_bound_mw) {
2278                         DP_NOTICE(p_hwfn,
2279                                   "number of invalidate memory windows is different from bounded ones\n");
2280                         return -EINVAL;
2281                 }
2282         } else {
2283                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n");
2284         }
2285
2286         return rc;
2287 }
2288
2289 static int qed_rdma_modify_qp(void *rdma_cxt,
2290                               struct qed_rdma_qp *qp,
2291                               struct qed_rdma_modify_qp_in_params *params)
2292 {
2293         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2294         enum qed_roce_qp_state prev_state;
2295         int rc = 0;
2296
2297         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x params->new_state=%d\n",
2298                    qp->icid, params->new_state);
2299
2300         if (rc) {
2301                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
2302                 return rc;
2303         }
2304
2305         if (GET_FIELD(params->modify_flags,
2306                       QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)) {
2307                 qp->incoming_rdma_read_en = params->incoming_rdma_read_en;
2308                 qp->incoming_rdma_write_en = params->incoming_rdma_write_en;
2309                 qp->incoming_atomic_en = params->incoming_atomic_en;
2310         }
2311
2312         /* Update QP structure with the updated values */
2313         if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE))
2314                 qp->roce_mode = params->roce_mode;
2315         if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY))
2316                 qp->pkey = params->pkey;
2317         if (GET_FIELD(params->modify_flags,
2318                       QED_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN))
2319                 qp->e2e_flow_control_en = params->e2e_flow_control_en;
2320         if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_DEST_QP))
2321                 qp->dest_qp = params->dest_qp;
2322         if (GET_FIELD(params->modify_flags,
2323                       QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)) {
2324                 /* Indicates that the following parameters have changed:
2325                  * Traffic class, flow label, hop limit, source GID,
2326                  * destination GID, loopback indicator
2327                  */
2328                 qp->traffic_class_tos = params->traffic_class_tos;
2329                 qp->flow_label = params->flow_label;
2330                 qp->hop_limit_ttl = params->hop_limit_ttl;
2331
2332                 qp->sgid = params->sgid;
2333                 qp->dgid = params->dgid;
2334                 qp->udp_src_port = 0;
2335                 qp->vlan_id = params->vlan_id;
2336                 qp->mtu = params->mtu;
2337                 qp->lb_indication = params->lb_indication;
2338                 memcpy((u8 *)&qp->remote_mac_addr[0],
2339                        (u8 *)&params->remote_mac_addr[0], ETH_ALEN);
2340                 if (params->use_local_mac) {
2341                         memcpy((u8 *)&qp->local_mac_addr[0],
2342                                (u8 *)&params->local_mac_addr[0], ETH_ALEN);
2343                 } else {
2344                         memcpy((u8 *)&qp->local_mac_addr[0],
2345                                (u8 *)&p_hwfn->hw_info.hw_mac_addr, ETH_ALEN);
2346                 }
2347         }
2348         if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RQ_PSN))
2349                 qp->rq_psn = params->rq_psn;
2350         if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_SQ_PSN))
2351                 qp->sq_psn = params->sq_psn;
2352         if (GET_FIELD(params->modify_flags,
2353                       QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ))
2354                 qp->max_rd_atomic_req = params->max_rd_atomic_req;
2355         if (GET_FIELD(params->modify_flags,
2356                       QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP))
2357                 qp->max_rd_atomic_resp = params->max_rd_atomic_resp;
2358         if (GET_FIELD(params->modify_flags,
2359                       QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT))
2360                 qp->ack_timeout = params->ack_timeout;
2361         if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT))
2362                 qp->retry_cnt = params->retry_cnt;
2363         if (GET_FIELD(params->modify_flags,
2364                       QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT))
2365                 qp->rnr_retry_cnt = params->rnr_retry_cnt;
2366         if (GET_FIELD(params->modify_flags,
2367                       QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER))
2368                 qp->min_rnr_nak_timer = params->min_rnr_nak_timer;
2369
2370         qp->sqd_async = params->sqd_async;
2371
2372         prev_state = qp->cur_state;
2373         if (GET_FIELD(params->modify_flags,
2374                       QED_RDMA_MODIFY_QP_VALID_NEW_STATE)) {
2375                 qp->cur_state = params->new_state;
2376                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "qp->cur_state=%d\n",
2377                            qp->cur_state);
2378         }
2379
2380         rc = qed_roce_modify_qp(p_hwfn, qp, prev_state, params);
2381
2382         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify QP, rc = %d\n", rc);
2383         return rc;
2384 }
2385
2386 static int
2387 qed_rdma_register_tid(void *rdma_cxt,
2388                       struct qed_rdma_register_tid_in_params *params)
2389 {
2390         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2391         struct rdma_register_tid_ramrod_data *p_ramrod;
2392         struct qed_sp_init_data init_data;
2393         struct qed_spq_entry *p_ent;
2394         enum rdma_tid_type tid_type;
2395         u8 fw_return_code;
2396         int rc;
2397
2398         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", params->itid);
2399
2400         /* Get SPQ entry */
2401         memset(&init_data, 0, sizeof(init_data));
2402         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
2403         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
2404
2405         rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR,
2406                                  p_hwfn->p_rdma_info->proto, &init_data);
2407         if (rc) {
2408                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
2409                 return rc;
2410         }
2411
2412         if (p_hwfn->p_rdma_info->last_tid < params->itid)
2413                 p_hwfn->p_rdma_info->last_tid = params->itid;
2414
2415         p_ramrod = &p_ent->ramrod.rdma_register_tid;
2416
2417         p_ramrod->flags = 0;
2418         SET_FIELD(p_ramrod->flags,
2419                   RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL,
2420                   params->pbl_two_level);
2421
2422         SET_FIELD(p_ramrod->flags,
2423                   RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, params->zbva);
2424
2425         SET_FIELD(p_ramrod->flags,
2426                   RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, params->phy_mr);
2427
2428         /* Don't initialize D/C field, as it may override other bits. */
2429         if (!(params->tid_type == QED_RDMA_TID_FMR) && !(params->dma_mr))
2430                 SET_FIELD(p_ramrod->flags,
2431                           RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG,
2432                           params->page_size_log - 12);
2433
2434         SET_FIELD(p_ramrod->flags,
2435                   RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID,
2436                   p_hwfn->p_rdma_info->last_tid);
2437
2438         SET_FIELD(p_ramrod->flags,
2439                   RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ,
2440                   params->remote_read);
2441
2442         SET_FIELD(p_ramrod->flags,
2443                   RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE,
2444                   params->remote_write);
2445
2446         SET_FIELD(p_ramrod->flags,
2447                   RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC,
2448                   params->remote_atomic);
2449
2450         SET_FIELD(p_ramrod->flags,
2451                   RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE,
2452                   params->local_write);
2453
2454         SET_FIELD(p_ramrod->flags,
2455                   RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ, params->local_read);
2456
2457         SET_FIELD(p_ramrod->flags,
2458                   RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND,
2459                   params->mw_bind);
2460
2461         SET_FIELD(p_ramrod->flags1,
2462                   RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG,
2463                   params->pbl_page_size_log - 12);
2464
2465         SET_FIELD(p_ramrod->flags2,
2466                   RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR, params->dma_mr);
2467
2468         switch (params->tid_type) {
2469         case QED_RDMA_TID_REGISTERED_MR:
2470                 tid_type = RDMA_TID_REGISTERED_MR;
2471                 break;
2472         case QED_RDMA_TID_FMR:
2473                 tid_type = RDMA_TID_FMR;
2474                 break;
2475         case QED_RDMA_TID_MW_TYPE1:
2476                 tid_type = RDMA_TID_MW_TYPE1;
2477                 break;
2478         case QED_RDMA_TID_MW_TYPE2A:
2479                 tid_type = RDMA_TID_MW_TYPE2A;
2480                 break;
2481         default:
2482                 rc = -EINVAL;
2483                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
2484                 return rc;
2485         }
2486         SET_FIELD(p_ramrod->flags1,
2487                   RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE, tid_type);
2488
2489         p_ramrod->itid = cpu_to_le32(params->itid);
2490         p_ramrod->key = params->key;
2491         p_ramrod->pd = cpu_to_le16(params->pd);
2492         p_ramrod->length_hi = (u8)(params->length >> 32);
2493         p_ramrod->length_lo = DMA_LO_LE(params->length);
2494         if (params->zbva) {
2495                 /* Lower 32 bits of the registered MR address.
2496                  * In case of zero based MR, will hold FBO
2497                  */
2498                 p_ramrod->va.hi = 0;
2499                 p_ramrod->va.lo = cpu_to_le32(params->fbo);
2500         } else {
2501                 DMA_REGPAIR_LE(p_ramrod->va, params->vaddr);
2502         }
2503         DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr);
2504
2505         /* DIF */
2506         if (params->dif_enabled) {
2507                 SET_FIELD(p_ramrod->flags2,
2508                           RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1);
2509                 DMA_REGPAIR_LE(p_ramrod->dif_error_addr,
2510                                params->dif_error_addr);
2511                 DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr);
2512         }
2513
2514         rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
2515         if (rc)
2516                 return rc;
2517
2518         if (fw_return_code != RDMA_RETURN_OK) {
2519                 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code);
2520                 return -EINVAL;
2521         }
2522
2523         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Register TID, rc = %d\n", rc);
2524         return rc;
2525 }
2526
2527 static int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid)
2528 {
2529         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2530         struct rdma_deregister_tid_ramrod_data *p_ramrod;
2531         struct qed_sp_init_data init_data;
2532         struct qed_spq_entry *p_ent;
2533         struct qed_ptt *p_ptt;
2534         u8 fw_return_code;
2535         int rc;
2536
2537         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid);
2538
2539         /* Get SPQ entry */
2540         memset(&init_data, 0, sizeof(init_data));
2541         init_data.opaque_fid = p_hwfn->hw_info.opaque_fid;
2542         init_data.comp_mode = QED_SPQ_MODE_EBLOCK;
2543
2544         rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_DEREGISTER_MR,
2545                                  p_hwfn->p_rdma_info->proto, &init_data);
2546         if (rc) {
2547                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
2548                 return rc;
2549         }
2550
2551         p_ramrod = &p_ent->ramrod.rdma_deregister_tid;
2552         p_ramrod->itid = cpu_to_le32(itid);
2553
2554         rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
2555         if (rc) {
2556                 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc);
2557                 return rc;
2558         }
2559
2560         if (fw_return_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) {
2561                 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code);
2562                 return -EINVAL;
2563         } else if (fw_return_code == RDMA_RETURN_NIG_DRAIN_REQ) {
2564                 /* Bit indicating that the TID is in use and a nig drain is
2565                  * required before sending the ramrod again
2566                  */
2567                 p_ptt = qed_ptt_acquire(p_hwfn);
2568                 if (!p_ptt) {
2569                         rc = -EBUSY;
2570                         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2571                                    "Failed to acquire PTT\n");
2572                         return rc;
2573                 }
2574
2575                 rc = qed_mcp_drain(p_hwfn, p_ptt);
2576                 if (rc) {
2577                         qed_ptt_release(p_hwfn, p_ptt);
2578                         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2579                                    "Drain failed\n");
2580                         return rc;
2581                 }
2582
2583                 qed_ptt_release(p_hwfn, p_ptt);
2584
2585                 /* Resend the ramrod */
2586                 rc = qed_sp_init_request(p_hwfn, &p_ent,
2587                                          RDMA_RAMROD_DEREGISTER_MR,
2588                                          p_hwfn->p_rdma_info->proto,
2589                                          &init_data);
2590                 if (rc) {
2591                         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2592                                    "Failed to init sp-element\n");
2593                         return rc;
2594                 }
2595
2596                 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code);
2597                 if (rc) {
2598                         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2599                                    "Ramrod failed\n");
2600                         return rc;
2601                 }
2602
2603                 if (fw_return_code != RDMA_RETURN_OK) {
2604                         DP_NOTICE(p_hwfn, "fw_return_code = %d\n",
2605                                   fw_return_code);
2606                         return rc;
2607                 }
2608         }
2609
2610         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "De-registered TID, rc = %d\n", rc);
2611         return rc;
2612 }
2613
2614 static void qed_roce_free_real_icid(struct qed_hwfn *p_hwfn, u16 icid)
2615 {
2616         struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info;
2617         u32 start_cid, cid, xcid;
2618
2619         /* an even icid belongs to a responder while an odd icid belongs to a
2620          * requester. The 'cid' received as an input can be either. We calculate
2621          * the "partner" icid and call it xcid. Only if both are free then the
2622          * "cid" map can be cleared.
2623          */
2624         start_cid = qed_cxt_get_proto_cid_start(p_hwfn, p_rdma_info->proto);
2625         cid = icid - start_cid;
2626         xcid = cid ^ 1;
2627
2628         spin_lock_bh(&p_rdma_info->lock);
2629
2630         qed_bmap_release_id(p_hwfn, &p_rdma_info->real_cid_map, cid);
2631         if (qed_bmap_test_id(p_hwfn, &p_rdma_info->real_cid_map, xcid) == 0) {
2632                 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, cid);
2633                 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, xcid);
2634         }
2635
2636         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
2637 }
2638
2639 static void *qed_rdma_get_rdma_ctx(struct qed_dev *cdev)
2640 {
2641         return QED_LEADING_HWFN(cdev);
2642 }
2643
2644 static void qed_rdma_dpm_conf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
2645 {
2646         u32 val;
2647
2648         val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1;
2649
2650         qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val);
2651         DP_VERBOSE(p_hwfn, (QED_MSG_DCB | QED_MSG_RDMA),
2652                    "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n",
2653                    val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm);
2654 }
2655
2656 void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
2657 {
2658         p_hwfn->db_bar_no_edpm = true;
2659
2660         qed_rdma_dpm_conf(p_hwfn, p_ptt);
2661 }
2662
2663 static int qed_rdma_start(void *rdma_cxt,
2664                           struct qed_rdma_start_in_params *params)
2665 {
2666         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2667         struct qed_ptt *p_ptt;
2668         int rc = -EBUSY;
2669
2670         DP_VERBOSE(p_hwfn, QED_MSG_RDMA,
2671                    "desired_cnq = %08x\n", params->desired_cnq);
2672
2673         p_ptt = qed_ptt_acquire(p_hwfn);
2674         if (!p_ptt)
2675                 goto err;
2676
2677         rc = qed_rdma_alloc(p_hwfn, p_ptt, params);
2678         if (rc)
2679                 goto err1;
2680
2681         rc = qed_rdma_setup(p_hwfn, p_ptt, params);
2682         if (rc)
2683                 goto err2;
2684
2685         qed_ptt_release(p_hwfn, p_ptt);
2686
2687         return rc;
2688
2689 err2:
2690         qed_rdma_free(p_hwfn);
2691 err1:
2692         qed_ptt_release(p_hwfn, p_ptt);
2693 err:
2694         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA start - error, rc = %d\n", rc);
2695         return rc;
2696 }
2697
2698 static int qed_rdma_init(struct qed_dev *cdev,
2699                          struct qed_rdma_start_in_params *params)
2700 {
2701         return qed_rdma_start(QED_LEADING_HWFN(cdev), params);
2702 }
2703
2704 static void qed_rdma_remove_user(void *rdma_cxt, u16 dpi)
2705 {
2706         struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
2707
2708         DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "dpi = %08x\n", dpi);
2709
2710         spin_lock_bh(&p_hwfn->p_rdma_info->lock);
2711         qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi);
2712         spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
2713 }
2714
2715 void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn,
2716                                      u8 connection_handle,
2717                                      void *cookie,
2718                                      dma_addr_t first_frag_addr,
2719                                      bool b_last_fragment, bool b_last_packet)
2720 {
2721         struct qed_roce_ll2_packet *packet = cookie;
2722         struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2;
2723
2724         roce_ll2->cbs.tx_cb(roce_ll2->cb_cookie, packet);
2725 }
2726
2727 void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn,
2728                                     u8 connection_handle,
2729                                     void *cookie,
2730                                     dma_addr_t first_frag_addr,
2731                                     bool b_last_fragment, bool b_last_packet)
2732 {
2733         qed_ll2b_complete_tx_gsi_packet(p_hwfn, connection_handle,
2734                                         cookie, first_frag_addr,
2735                                         b_last_fragment, b_last_packet);
2736 }
2737
2738 void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn,
2739                                      u8 connection_handle,
2740                                      void *cookie,
2741                                      dma_addr_t rx_buf_addr,
2742                                      u16 data_length,
2743                                      u8 data_length_error,
2744                                      u16 parse_flags,
2745                                      u16 vlan,
2746                                      u32 src_mac_addr_hi,
2747                                      u16 src_mac_addr_lo, bool b_last_packet)
2748 {
2749         struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2;
2750         struct qed_roce_ll2_rx_params params;
2751         struct qed_dev *cdev = p_hwfn->cdev;
2752         struct qed_roce_ll2_packet pkt;
2753
2754         DP_VERBOSE(cdev,
2755                    QED_MSG_LL2,
2756                    "roce ll2 rx complete: bus_addr=%p, len=%d, data_len_err=%d\n",
2757                    (void *)(uintptr_t)rx_buf_addr,
2758                    data_length, data_length_error);
2759
2760         memset(&pkt, 0, sizeof(pkt));
2761         pkt.n_seg = 1;
2762         pkt.payload[0].baddr = rx_buf_addr;
2763         pkt.payload[0].len = data_length;
2764
2765         memset(&params, 0, sizeof(params));
2766         params.vlan_id = vlan;
2767         *((u32 *)&params.smac[0]) = ntohl(src_mac_addr_hi);
2768         *((u16 *)&params.smac[4]) = ntohs(src_mac_addr_lo);
2769
2770         if (data_length_error) {
2771                 DP_ERR(cdev,
2772                        "roce ll2 rx complete: data length error %d, length=%d\n",
2773                        data_length_error, data_length);
2774                 params.rc = -EINVAL;
2775         }
2776
2777         roce_ll2->cbs.rx_cb(roce_ll2->cb_cookie, &pkt, &params);
2778 }
2779
2780 static int qed_roce_ll2_set_mac_filter(struct qed_dev *cdev,
2781                                        u8 *old_mac_address,
2782                                        u8 *new_mac_address)
2783 {
2784         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
2785         struct qed_ptt *p_ptt;
2786         int rc = 0;
2787
2788         if (!hwfn->ll2 || hwfn->ll2->handle == QED_LL2_UNUSED_HANDLE) {
2789                 DP_ERR(cdev,
2790                        "qed roce mac filter failed - roce_info/ll2 NULL\n");
2791                 return -EINVAL;
2792         }
2793
2794         p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
2795         if (!p_ptt) {
2796                 DP_ERR(cdev,
2797                        "qed roce ll2 mac filter set: failed to acquire PTT\n");
2798                 return -EINVAL;
2799         }
2800
2801         mutex_lock(&hwfn->ll2->lock);
2802         if (old_mac_address)
2803                 qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
2804                                           old_mac_address);
2805         if (new_mac_address)
2806                 rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt,
2807                                             new_mac_address);
2808         mutex_unlock(&hwfn->ll2->lock);
2809
2810         qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
2811
2812         if (rc)
2813                 DP_ERR(cdev,
2814                        "qed roce ll2 mac filter set: failed to add mac filter\n");
2815
2816         return rc;
2817 }
2818
2819 static int qed_roce_ll2_start(struct qed_dev *cdev,
2820                               struct qed_roce_ll2_params *params)
2821 {
2822         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
2823         struct qed_roce_ll2_info *roce_ll2;
2824         struct qed_ll2_conn ll2_params;
2825         int rc;
2826
2827         if (!params) {
2828                 DP_ERR(cdev, "qed roce ll2 start: failed due to NULL params\n");
2829                 return -EINVAL;
2830         }
2831         if (!params->cbs.tx_cb || !params->cbs.rx_cb) {
2832                 DP_ERR(cdev,
2833                        "qed roce ll2 start: failed due to NULL tx/rx. tx_cb=%p, rx_cb=%p\n",
2834                        params->cbs.tx_cb, params->cbs.rx_cb);
2835                 return -EINVAL;
2836         }
2837         if (!is_valid_ether_addr(params->mac_address)) {
2838                 DP_ERR(cdev,
2839                        "qed roce ll2 start: failed due to invalid Ethernet address %pM\n",
2840                        params->mac_address);
2841                 return -EINVAL;
2842         }
2843
2844         /* Initialize */
2845         roce_ll2 = kzalloc(sizeof(*roce_ll2), GFP_ATOMIC);
2846         if (!roce_ll2) {
2847                 DP_ERR(cdev, "qed roce ll2 start: failed memory allocation\n");
2848                 return -ENOMEM;
2849         }
2850         roce_ll2->handle = QED_LL2_UNUSED_HANDLE;
2851         roce_ll2->cbs = params->cbs;
2852         roce_ll2->cb_cookie = params->cb_cookie;
2853         mutex_init(&roce_ll2->lock);
2854
2855         memset(&ll2_params, 0, sizeof(ll2_params));
2856         ll2_params.conn_type = QED_LL2_TYPE_ROCE;
2857         ll2_params.mtu = params->mtu;
2858         ll2_params.rx_drop_ttl0_flg = true;
2859         ll2_params.rx_vlan_removal_en = false;
2860         ll2_params.tx_dest = CORE_TX_DEST_NW;
2861         ll2_params.ai_err_packet_too_big = LL2_DROP_PACKET;
2862         ll2_params.ai_err_no_buf = LL2_DROP_PACKET;
2863         ll2_params.gsi_enable = true;
2864
2865         rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_params,
2866                                         params->max_rx_buffers,
2867                                         params->max_tx_buffers,
2868                                         &roce_ll2->handle);
2869         if (rc) {
2870                 DP_ERR(cdev,
2871                        "qed roce ll2 start: failed to acquire LL2 connection (rc=%d)\n",
2872                        rc);
2873                 goto err;
2874         }
2875
2876         rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev),
2877                                           roce_ll2->handle);
2878         if (rc) {
2879                 DP_ERR(cdev,
2880                        "qed roce ll2 start: failed to establish LL2 connection (rc=%d)\n",
2881                        rc);
2882                 goto err1;
2883         }
2884
2885         hwfn->ll2 = roce_ll2;
2886
2887         rc = qed_roce_ll2_set_mac_filter(cdev, NULL, params->mac_address);
2888         if (rc) {
2889                 hwfn->ll2 = NULL;
2890                 goto err2;
2891         }
2892         ether_addr_copy(roce_ll2->mac_address, params->mac_address);
2893
2894         return 0;
2895
2896 err2:
2897         qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle);
2898 err1:
2899         qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle);
2900 err:
2901         kfree(roce_ll2);
2902         return rc;
2903 }
2904
2905 static int qed_roce_ll2_stop(struct qed_dev *cdev)
2906 {
2907         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
2908         struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2;
2909         int rc;
2910
2911         if (roce_ll2->handle == QED_LL2_UNUSED_HANDLE) {
2912                 DP_ERR(cdev, "qed roce ll2 stop: cannot stop an unused LL2\n");
2913                 return -EINVAL;
2914         }
2915
2916         /* remove LL2 MAC address filter */
2917         rc = qed_roce_ll2_set_mac_filter(cdev, roce_ll2->mac_address, NULL);
2918         eth_zero_addr(roce_ll2->mac_address);
2919
2920         rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev),
2921                                           roce_ll2->handle);
2922         if (rc)
2923                 DP_ERR(cdev,
2924                        "qed roce ll2 stop: failed to terminate LL2 connection (rc=%d)\n",
2925                        rc);
2926
2927         qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle);
2928
2929         roce_ll2->handle = QED_LL2_UNUSED_HANDLE;
2930
2931         kfree(roce_ll2);
2932
2933         return rc;
2934 }
2935
2936 static int qed_roce_ll2_tx(struct qed_dev *cdev,
2937                            struct qed_roce_ll2_packet *pkt,
2938                            struct qed_roce_ll2_tx_params *params)
2939 {
2940         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
2941         struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2;
2942         enum qed_ll2_roce_flavor_type qed_roce_flavor;
2943         u8 flags = 0;
2944         int rc;
2945         int i;
2946
2947         if (!pkt || !params) {
2948                 DP_ERR(cdev,
2949                        "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n",
2950                        cdev, pkt, params);
2951                 return -EINVAL;
2952         }
2953
2954         qed_roce_flavor = (pkt->roce_mode == ROCE_V1) ? QED_LL2_ROCE
2955                                                       : QED_LL2_RROCE;
2956
2957         if (pkt->roce_mode == ROCE_V2_IPV4)
2958                 flags |= BIT(CORE_TX_BD_DATA_IP_CSUM_SHIFT);
2959
2960         /* Tx header */
2961         rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev), roce_ll2->handle,
2962                                        1 + pkt->n_seg, 0, flags, 0,
2963                                        QED_LL2_TX_DEST_NW,
2964                                        qed_roce_flavor, pkt->header.baddr,
2965                                        pkt->header.len, pkt, 1);
2966         if (rc) {
2967                 DP_ERR(cdev, "roce ll2 tx: header failed (rc=%d)\n", rc);
2968                 return QED_ROCE_TX_HEAD_FAILURE;
2969         }
2970
2971         /* Tx payload */
2972         for (i = 0; i < pkt->n_seg; i++) {
2973                 rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev),
2974                                                        roce_ll2->handle,
2975                                                        pkt->payload[i].baddr,
2976                                                        pkt->payload[i].len);
2977                 if (rc) {
2978                         /* If failed not much to do here, partial packet has
2979                          * been posted * we can't free memory, will need to wait
2980                          * for completion
2981                          */
2982                         DP_ERR(cdev,
2983                                "roce ll2 tx: payload failed (rc=%d)\n", rc);
2984                         return QED_ROCE_TX_FRAG_FAILURE;
2985                 }
2986         }
2987
2988         return 0;
2989 }
2990
2991 static int qed_roce_ll2_post_rx_buffer(struct qed_dev *cdev,
2992                                        struct qed_roce_ll2_buffer *buf,
2993                                        u64 cookie, u8 notify_fw)
2994 {
2995         return qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev),
2996                                       QED_LEADING_HWFN(cdev)->ll2->handle,
2997                                       buf->baddr, buf->len,
2998                                       (void *)(uintptr_t)cookie, notify_fw);
2999 }
3000
3001 static int qed_roce_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats)
3002 {
3003         struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev);
3004         struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2;
3005
3006         return qed_ll2_get_stats(QED_LEADING_HWFN(cdev),
3007                                  roce_ll2->handle, stats);
3008 }
3009
3010 static const struct qed_rdma_ops qed_rdma_ops_pass = {
3011         .common = &qed_common_ops_pass,
3012         .fill_dev_info = &qed_fill_rdma_dev_info,
3013         .rdma_get_rdma_ctx = &qed_rdma_get_rdma_ctx,
3014         .rdma_init = &qed_rdma_init,
3015         .rdma_add_user = &qed_rdma_add_user,
3016         .rdma_remove_user = &qed_rdma_remove_user,
3017         .rdma_stop = &qed_rdma_stop,
3018         .rdma_query_port = &qed_rdma_query_port,
3019         .rdma_query_device = &qed_rdma_query_device,
3020         .rdma_get_start_sb = &qed_rdma_get_sb_start,
3021         .rdma_get_rdma_int = &qed_rdma_get_int,
3022         .rdma_set_rdma_int = &qed_rdma_set_int,
3023         .rdma_get_min_cnq_msix = &qed_rdma_get_min_cnq_msix,
3024         .rdma_cnq_prod_update = &qed_rdma_cnq_prod_update,
3025         .rdma_alloc_pd = &qed_rdma_alloc_pd,
3026         .rdma_dealloc_pd = &qed_rdma_free_pd,
3027         .rdma_create_cq = &qed_rdma_create_cq,
3028         .rdma_destroy_cq = &qed_rdma_destroy_cq,
3029         .rdma_create_qp = &qed_rdma_create_qp,
3030         .rdma_modify_qp = &qed_rdma_modify_qp,
3031         .rdma_query_qp = &qed_rdma_query_qp,
3032         .rdma_destroy_qp = &qed_rdma_destroy_qp,
3033         .rdma_alloc_tid = &qed_rdma_alloc_tid,
3034         .rdma_free_tid = &qed_rdma_free_tid,
3035         .rdma_register_tid = &qed_rdma_register_tid,
3036         .rdma_deregister_tid = &qed_rdma_deregister_tid,
3037         .roce_ll2_start = &qed_roce_ll2_start,
3038         .roce_ll2_stop = &qed_roce_ll2_stop,
3039         .roce_ll2_tx = &qed_roce_ll2_tx,
3040         .roce_ll2_post_rx_buffer = &qed_roce_ll2_post_rx_buffer,
3041         .roce_ll2_set_mac_filter = &qed_roce_ll2_set_mac_filter,
3042         .roce_ll2_stats = &qed_roce_ll2_stats,
3043 };
3044
3045 const struct qed_rdma_ops *qed_get_rdma_ops(void)
3046 {
3047         return &qed_rdma_ops_pass;
3048 }
3049 EXPORT_SYMBOL(qed_get_rdma_ops);