drivers/net/ethernet/qlogic/qed/qed_spq.c

   1 /* QLogic qed NIC Driver
   2  * Copyright (c) 2015-2017  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/types.h>
  34 #include <asm/byteorder.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/errno.h>
  39 #include <linux/kernel.h>
  40 #include <linux/list.h>
  41 #include <linux/pci.h>
  42 #include <linux/slab.h>
  43 #include <linux/spinlock.h>
  44 #include <linux/string.h>
  45 #include "qed.h"
  46 #include "qed_cxt.h"
  47 #include "qed_dev_api.h"
  48 #include "qed_hsi.h"
  49 #include "qed_hw.h"
  50 #include "qed_int.h"
  51 #include "qed_iscsi.h"
  52 #include "qed_mcp.h"
  53 #include "qed_ooo.h"
  54 #include "qed_reg_addr.h"
  55 #include "qed_sp.h"
  56 #include "qed_sriov.h"
  57 #include "qed_roce.h"
  58
  59 /***************************************************************************
  60 * Structures & Definitions
  61 ***************************************************************************/
  62
  63 #define SPQ_HIGH_PRI_RESERVE_DEFAULT    (1)
  64
  65 #define SPQ_BLOCK_DELAY_MAX_ITER        (10)
  66 #define SPQ_BLOCK_DELAY_US              (10)
  67 #define SPQ_BLOCK_SLEEP_MAX_ITER        (1000)
  68 #define SPQ_BLOCK_SLEEP_MS              (5)
  69
  70 /***************************************************************************
  71 * Blocking Imp. (BLOCK/EBLOCK mode)
  72 ***************************************************************************/
  73 static void qed_spq_blocking_cb(struct qed_hwfn *p_hwfn,
  74                                 void *cookie,
  75                                 union event_ring_data *data, u8 fw_return_code)
  76 {
  77         struct qed_spq_comp_done *comp_done;
  78
  79         comp_done = (struct qed_spq_comp_done *)cookie;
  80
  81         comp_done->fw_return_code = fw_return_code;
  82
  83         /* Make sure completion done is visible on waiting thread */
  84         smp_store_release(&comp_done->done, 0x1);
  85 }
  86
  87 static int __qed_spq_block(struct qed_hwfn *p_hwfn,
  88                            struct qed_spq_entry *p_ent,
  89                            u8 *p_fw_ret, bool sleep_between_iter)
  90 {
  91         struct qed_spq_comp_done *comp_done;
  92         u32 iter_cnt;
  93
  94         comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie;
  95         iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER
  96                                       : SPQ_BLOCK_DELAY_MAX_ITER;
  97
  98         while (iter_cnt--) {
  99                 /* Validate we receive completion update */
 100                 if (READ_ONCE(comp_done->done) == 1) {
 101                         /* Read updated FW return value */
 102                         smp_read_barrier_depends();
 103                         if (p_fw_ret)
 104                                 *p_fw_ret = comp_done->fw_return_code;
 105                         return 0;
 106                 }
 107
 108                 if (sleep_between_iter)
 109                         msleep(SPQ_BLOCK_SLEEP_MS);
 110                 else
 111                         udelay(SPQ_BLOCK_DELAY_US);
 112         }
 113
 114         return -EBUSY;
 115 }
 116
 117 static int qed_spq_block(struct qed_hwfn *p_hwfn,
 118                          struct qed_spq_entry *p_ent,
 119                          u8 *p_fw_ret, bool skip_quick_poll)
 120 {
 121         struct qed_spq_comp_done *comp_done;
 122         int rc;
 123
 124         /* A relatively short polling period w/o sleeping, to allow the FW to
 125          * complete the ramrod and thus possibly to avoid the following sleeps.
 126          */
 127         if (!skip_quick_poll) {
 128                 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, false);
 129                 if (!rc)
 130                         return 0;
 131         }
 132
 133         /* Move to polling with a sleeping period between iterations */
 134         rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true);
 135         if (!rc)
 136                 return 0;
 137
 138         DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n");
 139         rc = qed_mcp_drain(p_hwfn, p_hwfn->p_main_ptt);
 140         if (rc) {
 141                 DP_NOTICE(p_hwfn, "MCP drain failed\n");
 142                 goto err;
 143         }
 144
 145         /* Retry after drain */
 146         rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true);
 147         if (!rc)
 148                 return 0;
 149
 150         comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie;
 151         if (comp_done->done == 1) {
 152                 if (p_fw_ret)
 153                         *p_fw_ret = comp_done->fw_return_code;
 154                 return 0;
 155         }
 156 err:
 157         DP_NOTICE(p_hwfn,
 158                   "Ramrod is stuck [CID %08x cmd %02x protocol %02x echo %04x]\n",
 159                   le32_to_cpu(p_ent->elem.hdr.cid),
 160                   p_ent->elem.hdr.cmd_id,
 161                   p_ent->elem.hdr.protocol_id,
 162                   le16_to_cpu(p_ent->elem.hdr.echo));
 163
 164         return -EBUSY;
 165 }
 166
 167 /***************************************************************************
 168 * SPQ entries inner API
 169 ***************************************************************************/
 170 static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn,
 171                               struct qed_spq_entry *p_ent)
 172 {
 173         p_ent->flags = 0;
 174
 175         switch (p_ent->comp_mode) {
 176         case QED_SPQ_MODE_EBLOCK:
 177         case QED_SPQ_MODE_BLOCK:
 178                 p_ent->comp_cb.function = qed_spq_blocking_cb;
 179                 break;
 180         case QED_SPQ_MODE_CB:
 181                 break;
 182         default:
 183                 DP_NOTICE(p_hwfn, "Unknown SPQE completion mode %d\n",
 184                           p_ent->comp_mode);
 185                 return -EINVAL;
 186         }
 187
 188         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 189                    "Ramrod header: [CID 0x%08x CMD 0x%02x protocol 0x%02x] Data pointer: [%08x:%08x] Completion Mode: %s\n",
 190                    p_ent->elem.hdr.cid,
 191                    p_ent->elem.hdr.cmd_id,
 192                    p_ent->elem.hdr.protocol_id,
 193                    p_ent->elem.data_ptr.hi,
 194                    p_ent->elem.data_ptr.lo,
 195                    D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK,
 196                            QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK",
 197                            "MODE_CB"));
 198
 199         return 0;
 200 }
 201
 202 /***************************************************************************
 203 * HSI access
 204 ***************************************************************************/
 205 static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn,
 206                                   struct qed_spq *p_spq)
 207 {
 208         u16                             pq;
 209         struct qed_cxt_info             cxt_info;
 210         struct core_conn_context        *p_cxt;
 211         union qed_qm_pq_params          pq_params;
 212         int                             rc;
 213
 214         cxt_info.iid = p_spq->cid;
 215
 216         rc = qed_cxt_get_cid_info(p_hwfn, &cxt_info);
 217
 218         if (rc < 0) {
 219                 DP_NOTICE(p_hwfn, "Cannot find context info for cid=%d\n",
 220                           p_spq->cid);
 221                 return;
 222         }
 223
 224         p_cxt = cxt_info.p_cxt;
 225
 226         SET_FIELD(p_cxt->xstorm_ag_context.flags10,
 227                   XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1);
 228         SET_FIELD(p_cxt->xstorm_ag_context.flags1,
 229                   XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1);
 230         SET_FIELD(p_cxt->xstorm_ag_context.flags9,
 231                   XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1);
 232
 233         /* QM physical queue */
 234         memset(&pq_params, 0, sizeof(pq_params));
 235         pq_params.core.tc = LB_TC;
 236         pq = qed_get_qm_pq(p_hwfn, PROTOCOLID_CORE, &pq_params);
 237         p_cxt->xstorm_ag_context.physical_q0 = cpu_to_le16(pq);
 238
 239         p_cxt->xstorm_st_context.spq_base_lo =
 240                 DMA_LO_LE(p_spq->chain.p_phys_addr);
 241         p_cxt->xstorm_st_context.spq_base_hi =
 242                 DMA_HI_LE(p_spq->chain.p_phys_addr);
 243
 244         DMA_REGPAIR_LE(p_cxt->xstorm_st_context.consolid_base_addr,
 245                        p_hwfn->p_consq->chain.p_phys_addr);
 246 }
 247
 248 static int qed_spq_hw_post(struct qed_hwfn *p_hwfn,
 249                            struct qed_spq *p_spq, struct qed_spq_entry *p_ent)
 250 {
 251         struct qed_chain *p_chain = &p_hwfn->p_spq->chain;
 252         u16 echo = qed_chain_get_prod_idx(p_chain);
 253         struct slow_path_element        *elem;
 254         struct core_db_data             db;
 255
 256         p_ent->elem.hdr.echo    = cpu_to_le16(echo);
 257         elem = qed_chain_produce(p_chain);
 258         if (!elem) {
 259                 DP_NOTICE(p_hwfn, "Failed to produce from SPQ chain\n");
 260                 return -EINVAL;
 261         }
 262
 263         *elem = p_ent->elem; /* struct assignment */
 264
 265         /* send a doorbell on the slow hwfn session */
 266         memset(&db, 0, sizeof(db));
 267         SET_FIELD(db.params, CORE_DB_DATA_DEST, DB_DEST_XCM);
 268         SET_FIELD(db.params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
 269         SET_FIELD(db.params, CORE_DB_DATA_AGG_VAL_SEL,
 270                   DQ_XCM_CORE_SPQ_PROD_CMD);
 271         db.agg_flags = DQ_XCM_CORE_DQ_CF_CMD;
 272         db.spq_prod = cpu_to_le16(qed_chain_get_prod_idx(p_chain));
 273
 274         /* make sure the SPQE is updated before the doorbell */
 275         wmb();
 276
 277         DOORBELL(p_hwfn, qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY), *(u32 *)&db);
 278
 279         /* make sure doorbell is rang */
 280         wmb();
 281
 282         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 283                    "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x agg_params: %02x, prod: %04x\n",
 284                    qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY),
 285                    p_spq->cid, db.params, db.agg_flags,
 286                    qed_chain_get_prod_idx(p_chain));
 287
 288         return 0;
 289 }
 290
 291 /***************************************************************************
 292 * Asynchronous events
 293 ***************************************************************************/
 294 static int
 295 qed_async_event_completion(struct qed_hwfn *p_hwfn,
 296                            struct event_ring_entry *p_eqe)
 297 {
 298         switch (p_eqe->protocol_id) {
 299 #if IS_ENABLED(CONFIG_QED_RDMA)
 300         case PROTOCOLID_ROCE:
 301                 qed_roce_async_event(p_hwfn, p_eqe->opcode,
 302                                      &p_eqe->data.rdma_data);
 303                 return 0;
 304 #endif
 305         case PROTOCOLID_COMMON:
 306                 return qed_sriov_eqe_event(p_hwfn,
 307                                            p_eqe->opcode,
 308                                            p_eqe->echo, &p_eqe->data);
 309         case PROTOCOLID_ISCSI:
 310                 if (!IS_ENABLED(CONFIG_QED_ISCSI))
 311                         return -EINVAL;
 312
 313                 if (p_hwfn->p_iscsi_info->event_cb) {
 314                         struct qed_iscsi_info *p_iscsi = p_hwfn->p_iscsi_info;
 315
 316                         return p_iscsi->event_cb(p_iscsi->event_context,
 317                                                  p_eqe->opcode, &p_eqe->data);
 318                 } else {
 319                         DP_NOTICE(p_hwfn,
 320                                   "iSCSI async completion is not set\n");
 321                         return -EINVAL;
 322                 }
 323         default:
 324                 DP_NOTICE(p_hwfn,
 325                           "Unknown Async completion for protocol: %d\n",
 326                           p_eqe->protocol_id);
 327                 return -EINVAL;
 328         }
 329 }
 330
 331 /***************************************************************************
 332 * EQ API
 333 ***************************************************************************/
 334 void qed_eq_prod_update(struct qed_hwfn *p_hwfn, u16 prod)
 335 {
 336         u32 addr = GTT_BAR0_MAP_REG_USDM_RAM +
 337                    USTORM_EQE_CONS_OFFSET(p_hwfn->rel_pf_id);
 338
 339         REG_WR16(p_hwfn, addr, prod);
 340
 341         /* keep prod updates ordered */
 342         mmiowb();
 343 }
 344
 345 int qed_eq_completion(struct qed_hwfn *p_hwfn, void *cookie)
 346 {
 347         struct qed_eq *p_eq = cookie;
 348         struct qed_chain *p_chain = &p_eq->chain;
 349         int rc = 0;
 350
 351         /* take a snapshot of the FW consumer */
 352         u16 fw_cons_idx = le16_to_cpu(*p_eq->p_fw_cons);
 353
 354         DP_VERBOSE(p_hwfn, QED_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx);
 355
 356         /* Need to guarantee the fw_cons index we use points to a usuable
 357          * element (to comply with our chain), so our macros would comply
 358          */
 359         if ((fw_cons_idx & qed_chain_get_usable_per_page(p_chain)) ==
 360             qed_chain_get_usable_per_page(p_chain))
 361                 fw_cons_idx += qed_chain_get_unusable_per_page(p_chain);
 362
 363         /* Complete current segment of eq entries */
 364         while (fw_cons_idx != qed_chain_get_cons_idx(p_chain)) {
 365                 struct event_ring_entry *p_eqe = qed_chain_consume(p_chain);
 366
 367                 if (!p_eqe) {
 368                         rc = -EINVAL;
 369                         break;
 370                 }
 371
 372                 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 373                            "op %x prot %x res0 %x echo %x fwret %x flags %x\n",
 374                            p_eqe->opcode,
 375                            p_eqe->protocol_id,
 376                            p_eqe->reserved0,
 377                            le16_to_cpu(p_eqe->echo),
 378                            p_eqe->fw_return_code,
 379                            p_eqe->flags);
 380
 381                 if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) {
 382                         if (qed_async_event_completion(p_hwfn, p_eqe))
 383                                 rc = -EINVAL;
 384                 } else if (qed_spq_completion(p_hwfn,
 385                                               p_eqe->echo,
 386                                               p_eqe->fw_return_code,
 387                                               &p_eqe->data)) {
 388                         rc = -EINVAL;
 389                 }
 390
 391                 qed_chain_recycle_consumed(p_chain);
 392         }
 393
 394         qed_eq_prod_update(p_hwfn, qed_chain_get_prod_idx(p_chain));
 395
 396         return rc;
 397 }
 398
 399 struct qed_eq *qed_eq_alloc(struct qed_hwfn *p_hwfn, u16 num_elem)
 400 {
 401         struct qed_eq *p_eq;
 402
 403         /* Allocate EQ struct */
 404         p_eq = kzalloc(sizeof(*p_eq), GFP_KERNEL);
 405         if (!p_eq)
 406                 return NULL;
 407
 408         /* Allocate and initialize EQ chain*/
 409         if (qed_chain_alloc(p_hwfn->cdev,
 410                             QED_CHAIN_USE_TO_PRODUCE,
 411                             QED_CHAIN_MODE_PBL,
 412                             QED_CHAIN_CNT_TYPE_U16,
 413                             num_elem,
 414                             sizeof(union event_ring_element),
 415                             &p_eq->chain))
 416                 goto eq_allocate_fail;
 417
 418         /* register EQ completion on the SP SB */
 419         qed_int_register_cb(p_hwfn, qed_eq_completion,
 420                             p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons);
 421
 422         return p_eq;
 423
 424 eq_allocate_fail:
 425         qed_eq_free(p_hwfn, p_eq);
 426         return NULL;
 427 }
 428
 429 void qed_eq_setup(struct qed_hwfn *p_hwfn, struct qed_eq *p_eq)
 430 {
 431         qed_chain_reset(&p_eq->chain);
 432 }
 433
 434 void qed_eq_free(struct qed_hwfn *p_hwfn, struct qed_eq *p_eq)
 435 {
 436         if (!p_eq)
 437                 return;
 438         qed_chain_free(p_hwfn->cdev, &p_eq->chain);
 439         kfree(p_eq);
 440 }
 441
 442 /***************************************************************************
 443 * CQE API - manipulate EQ functionality
 444 ***************************************************************************/
 445 static int qed_cqe_completion(struct qed_hwfn *p_hwfn,
 446                               struct eth_slow_path_rx_cqe *cqe,
 447                               enum protocol_type protocol)
 448 {
 449         if (IS_VF(p_hwfn->cdev))
 450                 return 0;
 451
 452         /* @@@tmp - it's possible we'll eventually want to handle some
 453          * actual commands that can arrive here, but for now this is only
 454          * used to complete the ramrod using the echo value on the cqe
 455          */
 456         return qed_spq_completion(p_hwfn, cqe->echo, 0, NULL);
 457 }
 458
 459 int qed_eth_cqe_completion(struct qed_hwfn *p_hwfn,
 460                            struct eth_slow_path_rx_cqe *cqe)
 461 {
 462         int rc;
 463
 464         rc = qed_cqe_completion(p_hwfn, cqe, PROTOCOLID_ETH);
 465         if (rc)
 466                 DP_NOTICE(p_hwfn,
 467                           "Failed to handle RXQ CQE [cmd 0x%02x]\n",
 468                           cqe->ramrod_cmd_id);
 469
 470         return rc;
 471 }
 472
 473 /***************************************************************************
 474 * Slow hwfn Queue (spq)
 475 ***************************************************************************/
 476 void qed_spq_setup(struct qed_hwfn *p_hwfn)
 477 {
 478         struct qed_spq *p_spq = p_hwfn->p_spq;
 479         struct qed_spq_entry *p_virt = NULL;
 480         dma_addr_t p_phys = 0;
 481         u32 i, capacity;
 482
 483         INIT_LIST_HEAD(&p_spq->pending);
 484         INIT_LIST_HEAD(&p_spq->completion_pending);
 485         INIT_LIST_HEAD(&p_spq->free_pool);
 486         INIT_LIST_HEAD(&p_spq->unlimited_pending);
 487         spin_lock_init(&p_spq->lock);
 488
 489         /* SPQ empty pool */
 490         p_phys  = p_spq->p_phys + offsetof(struct qed_spq_entry, ramrod);
 491         p_virt  = p_spq->p_virt;
 492
 493         capacity = qed_chain_get_capacity(&p_spq->chain);
 494         for (i = 0; i < capacity; i++) {
 495                 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys);
 496
 497                 list_add_tail(&p_virt->list, &p_spq->free_pool);
 498
 499                 p_virt++;
 500                 p_phys += sizeof(struct qed_spq_entry);
 501         }
 502
 503         /* Statistics */
 504         p_spq->normal_count             = 0;
 505         p_spq->comp_count               = 0;
 506         p_spq->comp_sent_count          = 0;
 507         p_spq->unlimited_pending_count  = 0;
 508
 509         bitmap_zero(p_spq->p_comp_bitmap, SPQ_RING_SIZE);
 510         p_spq->comp_bitmap_idx = 0;
 511
 512         /* SPQ cid, cannot fail */
 513         qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_spq->cid);
 514         qed_spq_hw_initialize(p_hwfn, p_spq);
 515
 516         /* reset the chain itself */
 517         qed_chain_reset(&p_spq->chain);
 518 }
 519
 520 int qed_spq_alloc(struct qed_hwfn *p_hwfn)
 521 {
 522         struct qed_spq_entry *p_virt = NULL;
 523         struct qed_spq *p_spq = NULL;
 524         dma_addr_t p_phys = 0;
 525         u32 capacity;
 526
 527         /* SPQ struct */
 528         p_spq = kzalloc(sizeof(struct qed_spq), GFP_KERNEL);
 529         if (!p_spq)
 530                 return -ENOMEM;
 531
 532         /* SPQ ring  */
 533         if (qed_chain_alloc(p_hwfn->cdev,
 534                             QED_CHAIN_USE_TO_PRODUCE,
 535                             QED_CHAIN_MODE_SINGLE,
 536                             QED_CHAIN_CNT_TYPE_U16,
 537                             0,   /* N/A when the mode is SINGLE */
 538                             sizeof(struct slow_path_element),
 539                             &p_spq->chain))
 540                 goto spq_allocate_fail;
 541
 542         /* allocate and fill the SPQ elements (incl. ramrod data list) */
 543         capacity = qed_chain_get_capacity(&p_spq->chain);
 544         p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
 545                                     capacity * sizeof(struct qed_spq_entry),
 546                                     &p_phys, GFP_KERNEL);
 547         if (!p_virt)
 548                 goto spq_allocate_fail;
 549
 550         p_spq->p_virt = p_virt;
 551         p_spq->p_phys = p_phys;
 552         p_hwfn->p_spq = p_spq;
 553
 554         return 0;
 555
 556 spq_allocate_fail:
 557         qed_chain_free(p_hwfn->cdev, &p_spq->chain);
 558         kfree(p_spq);
 559         return -ENOMEM;
 560 }
 561
 562 void qed_spq_free(struct qed_hwfn *p_hwfn)
 563 {
 564         struct qed_spq *p_spq = p_hwfn->p_spq;
 565         u32 capacity;
 566
 567         if (!p_spq)
 568                 return;
 569
 570         if (p_spq->p_virt) {
 571                 capacity = qed_chain_get_capacity(&p_spq->chain);
 572                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 573                                   capacity *
 574                                   sizeof(struct qed_spq_entry),
 575                                   p_spq->p_virt, p_spq->p_phys);
 576         }
 577
 578         qed_chain_free(p_hwfn->cdev, &p_spq->chain);
 579         ;
 580         kfree(p_spq);
 581 }
 582
 583 int qed_spq_get_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry **pp_ent)
 584 {
 585         struct qed_spq *p_spq = p_hwfn->p_spq;
 586         struct qed_spq_entry *p_ent = NULL;
 587         int rc = 0;
 588
 589         spin_lock_bh(&p_spq->lock);
 590
 591         if (list_empty(&p_spq->free_pool)) {
 592                 p_ent = kzalloc(sizeof(*p_ent), GFP_ATOMIC);
 593                 if (!p_ent) {
 594                         DP_NOTICE(p_hwfn,
 595                                   "Failed to allocate an SPQ entry for a pending ramrod\n");
 596                         rc = -ENOMEM;
 597                         goto out_unlock;
 598                 }
 599                 p_ent->queue = &p_spq->unlimited_pending;
 600         } else {
 601                 p_ent = list_first_entry(&p_spq->free_pool,
 602                                          struct qed_spq_entry, list);
 603                 list_del(&p_ent->list);
 604                 p_ent->queue = &p_spq->pending;
 605         }
 606
 607         *pp_ent = p_ent;
 608
 609 out_unlock:
 610         spin_unlock_bh(&p_spq->lock);
 611         return rc;
 612 }
 613
 614 /* Locked variant; Should be called while the SPQ lock is taken */
 615 static void __qed_spq_return_entry(struct qed_hwfn *p_hwfn,
 616                                    struct qed_spq_entry *p_ent)
 617 {
 618         list_add_tail(&p_ent->list, &p_hwfn->p_spq->free_pool);
 619 }
 620
 621 void qed_spq_return_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry *p_ent)
 622 {
 623         spin_lock_bh(&p_hwfn->p_spq->lock);
 624         __qed_spq_return_entry(p_hwfn, p_ent);
 625         spin_unlock_bh(&p_hwfn->p_spq->lock);
 626 }
 627
 628 /**
 629  * @brief qed_spq_add_entry - adds a new entry to the pending
 630  *        list. Should be used while lock is being held.
 631  *
 632  * Addes an entry to the pending list is there is room (en empty
 633  * element is available in the free_pool), or else places the
 634  * entry in the unlimited_pending pool.
 635  *
 636  * @param p_hwfn
 637  * @param p_ent
 638  * @param priority
 639  *
 640  * @return int
 641  */
 642 static int qed_spq_add_entry(struct qed_hwfn *p_hwfn,
 643                              struct qed_spq_entry *p_ent,
 644                              enum spq_priority priority)
 645 {
 646         struct qed_spq *p_spq = p_hwfn->p_spq;
 647
 648         if (p_ent->queue == &p_spq->unlimited_pending) {
 649
 650                 if (list_empty(&p_spq->free_pool)) {
 651                         list_add_tail(&p_ent->list, &p_spq->unlimited_pending);
 652                         p_spq->unlimited_pending_count++;
 653
 654                         return 0;
 655                 } else {
 656                         struct qed_spq_entry *p_en2;
 657
 658                         p_en2 = list_first_entry(&p_spq->free_pool,
 659                                                  struct qed_spq_entry, list);
 660                         list_del(&p_en2->list);
 661
 662                         /* Copy the ring element physical pointer to the new
 663                          * entry, since we are about to override the entire ring
 664                          * entry and don't want to lose the pointer.
 665                          */
 666                         p_ent->elem.data_ptr = p_en2->elem.data_ptr;
 667
 668                         *p_en2 = *p_ent;
 669
 670                         /* EBLOCK responsible to free the allocated p_ent */
 671                         if (p_ent->comp_mode != QED_SPQ_MODE_EBLOCK)
 672                                 kfree(p_ent);
 673
 674                         p_ent = p_en2;
 675                 }
 676         }
 677
 678         /* entry is to be placed in 'pending' queue */
 679         switch (priority) {
 680         case QED_SPQ_PRIORITY_NORMAL:
 681                 list_add_tail(&p_ent->list, &p_spq->pending);
 682                 p_spq->normal_count++;
 683                 break;
 684         case QED_SPQ_PRIORITY_HIGH:
 685                 list_add(&p_ent->list, &p_spq->pending);
 686                 p_spq->high_count++;
 687                 break;
 688         default:
 689                 return -EINVAL;
 690         }
 691
 692         return 0;
 693 }
 694
 695 /***************************************************************************
 696 * Accessor
 697 ***************************************************************************/
 698 u32 qed_spq_get_cid(struct qed_hwfn *p_hwfn)
 699 {
 700         if (!p_hwfn->p_spq)
 701                 return 0xffffffff;      /* illegal */
 702         return p_hwfn->p_spq->cid;
 703 }
 704
 705 /***************************************************************************
 706 * Posting new Ramrods
 707 ***************************************************************************/
 708 static int qed_spq_post_list(struct qed_hwfn *p_hwfn,
 709                              struct list_head *head, u32 keep_reserve)
 710 {
 711         struct qed_spq *p_spq = p_hwfn->p_spq;
 712         int rc;
 713
 714         while (qed_chain_get_elem_left(&p_spq->chain) > keep_reserve &&
 715                !list_empty(head)) {
 716                 struct qed_spq_entry *p_ent =
 717                         list_first_entry(head, struct qed_spq_entry, list);
 718                 list_del(&p_ent->list);
 719                 list_add_tail(&p_ent->list, &p_spq->completion_pending);
 720                 p_spq->comp_sent_count++;
 721
 722                 rc = qed_spq_hw_post(p_hwfn, p_spq, p_ent);
 723                 if (rc) {
 724                         list_del(&p_ent->list);
 725                         __qed_spq_return_entry(p_hwfn, p_ent);
 726                         return rc;
 727                 }
 728         }
 729
 730         return 0;
 731 }
 732
 733 static int qed_spq_pend_post(struct qed_hwfn *p_hwfn)
 734 {
 735         struct qed_spq *p_spq = p_hwfn->p_spq;
 736         struct qed_spq_entry *p_ent = NULL;
 737
 738         while (!list_empty(&p_spq->free_pool)) {
 739                 if (list_empty(&p_spq->unlimited_pending))
 740                         break;
 741
 742                 p_ent = list_first_entry(&p_spq->unlimited_pending,
 743                                          struct qed_spq_entry, list);
 744                 if (!p_ent)
 745                         return -EINVAL;
 746
 747                 list_del(&p_ent->list);
 748
 749                 qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority);
 750         }
 751
 752         return qed_spq_post_list(p_hwfn, &p_spq->pending,
 753                                  SPQ_HIGH_PRI_RESERVE_DEFAULT);
 754 }
 755
 756 int qed_spq_post(struct qed_hwfn *p_hwfn,
 757                  struct qed_spq_entry *p_ent, u8 *fw_return_code)
 758 {
 759         int rc = 0;
 760         struct qed_spq *p_spq = p_hwfn ? p_hwfn->p_spq : NULL;
 761         bool b_ret_ent = true;
 762
 763         if (!p_hwfn)
 764                 return -EINVAL;
 765
 766         if (!p_ent) {
 767                 DP_NOTICE(p_hwfn, "Got a NULL pointer\n");
 768                 return -EINVAL;
 769         }
 770
 771         /* Complete the entry */
 772         rc = qed_spq_fill_entry(p_hwfn, p_ent);
 773
 774         spin_lock_bh(&p_spq->lock);
 775
 776         /* Check return value after LOCK is taken for cleaner error flow */
 777         if (rc)
 778                 goto spq_post_fail;
 779
 780         /* Add the request to the pending queue */
 781         rc = qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority);
 782         if (rc)
 783                 goto spq_post_fail;
 784
 785         rc = qed_spq_pend_post(p_hwfn);
 786         if (rc) {
 787                 /* Since it's possible that pending failed for a different
 788                  * entry [although unlikely], the failed entry was already
 789                  * dealt with; No need to return it here.
 790                  */
 791                 b_ret_ent = false;
 792                 goto spq_post_fail;
 793         }
 794
 795         spin_unlock_bh(&p_spq->lock);
 796
 797         if (p_ent->comp_mode == QED_SPQ_MODE_EBLOCK) {
 798                 /* For entries in QED BLOCK mode, the completion code cannot
 799                  * perform the necessary cleanup - if it did, we couldn't
 800                  * access p_ent here to see whether it's successful or not.
 801                  * Thus, after gaining the answer perform the cleanup here.
 802                  */
 803                 rc = qed_spq_block(p_hwfn, p_ent, fw_return_code,
 804                                    p_ent->queue == &p_spq->unlimited_pending);
 805
 806                 if (p_ent->queue == &p_spq->unlimited_pending) {
 807                         /* This is an allocated p_ent which does not need to
 808                          * return to pool.
 809                          */
 810                         kfree(p_ent);
 811                         return rc;
 812                 }
 813
 814                 if (rc)
 815                         goto spq_post_fail2;
 816
 817                 /* return to pool */
 818                 qed_spq_return_entry(p_hwfn, p_ent);
 819         }
 820         return rc;
 821
 822 spq_post_fail2:
 823         spin_lock_bh(&p_spq->lock);
 824         list_del(&p_ent->list);
 825         qed_chain_return_produced(&p_spq->chain);
 826
 827 spq_post_fail:
 828         /* return to the free pool */
 829         if (b_ret_ent)
 830                 __qed_spq_return_entry(p_hwfn, p_ent);
 831         spin_unlock_bh(&p_spq->lock);
 832
 833         return rc;
 834 }
 835
 836 int qed_spq_completion(struct qed_hwfn *p_hwfn,
 837                        __le16 echo,
 838                        u8 fw_return_code,
 839                        union event_ring_data *p_data)
 840 {
 841         struct qed_spq          *p_spq;
 842         struct qed_spq_entry    *p_ent = NULL;
 843         struct qed_spq_entry    *tmp;
 844         struct qed_spq_entry    *found = NULL;
 845         int                     rc;
 846
 847         if (!p_hwfn)
 848                 return -EINVAL;
 849
 850         p_spq = p_hwfn->p_spq;
 851         if (!p_spq)
 852                 return -EINVAL;
 853
 854         spin_lock_bh(&p_spq->lock);
 855         list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending, list) {
 856                 if (p_ent->elem.hdr.echo == echo) {
 857                         u16 pos = le16_to_cpu(echo) % SPQ_RING_SIZE;
 858
 859                         list_del(&p_ent->list);
 860
 861                         /* Avoid overriding of SPQ entries when getting
 862                          * out-of-order completions, by marking the completions
 863                          * in a bitmap and increasing the chain consumer only
 864                          * for the first successive completed entries.
 865                          */
 866                         __set_bit(pos, p_spq->p_comp_bitmap);
 867
 868                         while (test_bit(p_spq->comp_bitmap_idx,
 869                                         p_spq->p_comp_bitmap)) {
 870                                 __clear_bit(p_spq->comp_bitmap_idx,
 871                                             p_spq->p_comp_bitmap);
 872                                 p_spq->comp_bitmap_idx++;
 873                                 qed_chain_return_produced(&p_spq->chain);
 874                         }
 875
 876                         p_spq->comp_count++;
 877                         found = p_ent;
 878                         break;
 879                 }
 880
 881                 /* This is relatively uncommon - depends on scenarios
 882                  * which have mutliple per-PF sent ramrods.
 883                  */
 884                 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 885                            "Got completion for echo %04x - doesn't match echo %04x in completion pending list\n",
 886                            le16_to_cpu(echo),
 887                            le16_to_cpu(p_ent->elem.hdr.echo));
 888         }
 889
 890         /* Release lock before callback, as callback may post
 891          * an additional ramrod.
 892          */
 893         spin_unlock_bh(&p_spq->lock);
 894
 895         if (!found) {
 896                 DP_NOTICE(p_hwfn,
 897                           "Failed to find an entry this EQE [echo %04x] completes\n",
 898                           le16_to_cpu(echo));
 899                 return -EEXIST;
 900         }
 901
 902         DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
 903                    "Complete EQE [echo %04x]: func %p cookie %p)\n",
 904                    le16_to_cpu(echo),
 905                    p_ent->comp_cb.function, p_ent->comp_cb.cookie);
 906         if (found->comp_cb.function)
 907                 found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data,
 908                                         fw_return_code);
 909         else
 910                 DP_VERBOSE(p_hwfn,
 911                            QED_MSG_SPQ,
 912                            "Got a completion without a callback function\n");
 913
 914         if ((found->comp_mode != QED_SPQ_MODE_EBLOCK) ||
 915             (found->queue == &p_spq->unlimited_pending))
 916                 /* EBLOCK  is responsible for returning its own entry into the
 917                  * free list, unless it originally added the entry into the
 918                  * unlimited pending list.
 919                  */
 920                 qed_spq_return_entry(p_hwfn, found);
 921
 922         /* Attempt to post pending requests */
 923         spin_lock_bh(&p_spq->lock);
 924         rc = qed_spq_pend_post(p_hwfn);
 925         spin_unlock_bh(&p_spq->lock);
 926
 927         return rc;
 928 }
 929
 930 struct qed_consq *qed_consq_alloc(struct qed_hwfn *p_hwfn)
 931 {
 932         struct qed_consq *p_consq;
 933
 934         /* Allocate ConsQ struct */
 935         p_consq = kzalloc(sizeof(*p_consq), GFP_KERNEL);
 936         if (!p_consq)
 937                 return NULL;
 938
 939         /* Allocate and initialize EQ chain*/
 940         if (qed_chain_alloc(p_hwfn->cdev,
 941                             QED_CHAIN_USE_TO_PRODUCE,
 942                             QED_CHAIN_MODE_PBL,
 943                             QED_CHAIN_CNT_TYPE_U16,
 944                             QED_CHAIN_PAGE_SIZE / 0x80,
 945                             0x80, &p_consq->chain))
 946                 goto consq_allocate_fail;
 947
 948         return p_consq;
 949
 950 consq_allocate_fail:
 951         qed_consq_free(p_hwfn, p_consq);
 952         return NULL;
 953 }
 954
 955 void qed_consq_setup(struct qed_hwfn *p_hwfn, struct qed_consq *p_consq)
 956 {
 957         qed_chain_reset(&p_consq->chain);
 958 }
 959
 960 void qed_consq_free(struct qed_hwfn *p_hwfn, struct qed_consq *p_consq)
 961 {
 962         if (!p_consq)
 963                 return;
 964         qed_chain_free(p_hwfn->cdev, &p_consq->chain);
 965         kfree(p_consq);
 966 }