drivers/net/ethernet/cavium/liquidio/octeon_droq.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/pci.h>
  19 #include <linux/netdevice.h>
  20 #include <linux/vmalloc.h>
  21 #include "liquidio_common.h"
  22 #include "octeon_droq.h"
  23 #include "octeon_iq.h"
  24 #include "response_manager.h"
  25 #include "octeon_device.h"
  26 #include "octeon_main.h"
  27 #include "octeon_network.h"
  28 #include "cn66xx_regs.h"
  29 #include "cn66xx_device.h"
  30 #include "cn23xx_pf_device.h"
  31 #include "cn23xx_vf_device.h"
  32
  33 struct niclist {
  34         struct list_head list;
  35         void *ptr;
  36 };
  37
  38 struct __dispatch {
  39         struct list_head list;
  40         struct octeon_recv_info *rinfo;
  41         octeon_dispatch_fn_t disp_fn;
  42 };
  43
  44 /** Get the argument that the user set when registering dispatch
  45  *  function for a given opcode/subcode.
  46  *  @param  octeon_dev - the octeon device pointer.
  47  *  @param  opcode     - the opcode for which the dispatch argument
  48  *                       is to be checked.
  49  *  @param  subcode    - the subcode for which the dispatch argument
  50  *                       is to be checked.
  51  *  @return  Success: void * (argument to the dispatch function)
  52  *  @return  Failure: NULL
  53  *
  54  */
  55 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
  56                                             u16 opcode, u16 subcode)
  57 {
  58         int idx;
  59         struct list_head *dispatch;
  60         void *fn_arg = NULL;
  61         u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
  62
  63         idx = combined_opcode & OCTEON_OPCODE_MASK;
  64
  65         spin_lock_bh(&octeon_dev->dispatch.lock);
  66
  67         if (octeon_dev->dispatch.count == 0) {
  68                 spin_unlock_bh(&octeon_dev->dispatch.lock);
  69                 return NULL;
  70         }
  71
  72         if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
  73                 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
  74         } else {
  75                 list_for_each(dispatch,
  76                               &octeon_dev->dispatch.dlist[idx].list) {
  77                         if (((struct octeon_dispatch *)dispatch)->opcode ==
  78                             combined_opcode) {
  79                                 fn_arg = ((struct octeon_dispatch *)
  80                                           dispatch)->arg;
  81                                 break;
  82                         }
  83                 }
  84         }
  85
  86         spin_unlock_bh(&octeon_dev->dispatch.lock);
  87         return fn_arg;
  88 }
  89
  90 /** Check for packets on Droq. This function should be called with lock held.
  91  *  @param  droq - Droq on which count is checked.
  92  *  @return Returns packet count.
  93  */
  94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
  95 {
  96         u32 pkt_count = 0;
  97         u32 last_count;
  98
  99         pkt_count = readl(droq->pkts_sent_reg);
 100
 101         last_count = pkt_count - droq->pkt_count;
 102         droq->pkt_count = pkt_count;
 103
 104         /* we shall write to cnts  at napi irq enable or end of droq tasklet */
 105         if (last_count)
 106                 atomic_add(last_count, &droq->pkts_pending);
 107
 108         return last_count;
 109 }
 110
 111 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
 112 {
 113         u32 count = 0;
 114
 115         /* max_empty_descs is the max. no. of descs that can have no buffers.
 116          * If the empty desc count goes beyond this value, we cannot safely
 117          * read in a 64K packet sent by Octeon
 118          * (64K is max pkt size from Octeon)
 119          */
 120         droq->max_empty_descs = 0;
 121
 122         do {
 123                 droq->max_empty_descs++;
 124                 count += droq->buffer_size;
 125         } while (count < (64 * 1024));
 126
 127         droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
 128 }
 129
 130 static void octeon_droq_reset_indices(struct octeon_droq *droq)
 131 {
 132         droq->read_idx = 0;
 133         droq->write_idx = 0;
 134         droq->refill_idx = 0;
 135         droq->refill_count = 0;
 136         atomic_set(&droq->pkts_pending, 0);
 137 }
 138
 139 static void
 140 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
 141                                  struct octeon_droq *droq)
 142 {
 143         u32 i;
 144         struct octeon_skb_page_info *pg_info;
 145
 146         for (i = 0; i < droq->max_count; i++) {
 147                 pg_info = &droq->recv_buf_list[i].pg_info;
 148
 149                 if (pg_info->dma)
 150                         lio_unmap_ring(oct->pci_dev,
 151                                        (u64)pg_info->dma);
 152                 pg_info->dma = 0;
 153
 154                 if (pg_info->page)
 155                         recv_buffer_destroy(droq->recv_buf_list[i].buffer,
 156                                             pg_info);
 157
 158                 if (droq->desc_ring && droq->desc_ring[i].info_ptr)
 159                         lio_unmap_ring_info(oct->pci_dev,
 160                                             (u64)droq->
 161                                             desc_ring[i].info_ptr,
 162                                             OCT_DROQ_INFO_SIZE);
 163                 droq->recv_buf_list[i].buffer = NULL;
 164         }
 165
 166         octeon_droq_reset_indices(droq);
 167 }
 168
 169 static int
 170 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
 171                                struct octeon_droq *droq)
 172 {
 173         u32 i;
 174         void *buf;
 175         struct octeon_droq_desc *desc_ring = droq->desc_ring;
 176
 177         for (i = 0; i < droq->max_count; i++) {
 178                 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
 179
 180                 if (!buf) {
 181                         dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
 182                                 __func__);
 183                         droq->stats.rx_alloc_failure++;
 184                         return -ENOMEM;
 185                 }
 186
 187                 droq->recv_buf_list[i].buffer = buf;
 188                 droq->recv_buf_list[i].data = get_rbd(buf);
 189                 droq->info_list[i].length = 0;
 190
 191                 /* map ring buffers into memory */
 192                 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
 193                 desc_ring[i].buffer_ptr =
 194                         lio_map_ring(droq->recv_buf_list[i].buffer);
 195         }
 196
 197         octeon_droq_reset_indices(droq);
 198
 199         octeon_droq_compute_max_packet_bufs(droq);
 200
 201         return 0;
 202 }
 203
 204 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
 205 {
 206         struct octeon_droq *droq = oct->droq[q_no];
 207
 208         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 209
 210         octeon_droq_destroy_ring_buffers(oct, droq);
 211         vfree(droq->recv_buf_list);
 212
 213         if (droq->info_base_addr)
 214                 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
 215                                        droq->info_alloc_size,
 216                                        droq->info_base_addr,
 217                                        droq->info_list_dma);
 218
 219         if (droq->desc_ring)
 220                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 221                              droq->desc_ring, droq->desc_ring_dma);
 222
 223         memset(droq, 0, OCT_DROQ_SIZE);
 224
 225         return 0;
 226 }
 227
 228 int octeon_init_droq(struct octeon_device *oct,
 229                      u32 q_no,
 230                      u32 num_descs,
 231                      u32 desc_size,
 232                      void *app_ctx)
 233 {
 234         struct octeon_droq *droq;
 235         u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
 236         u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
 237         int orig_node = dev_to_node(&oct->pci_dev->dev);
 238         int numa_node = cpu_to_node(q_no % num_online_cpus());
 239
 240         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 241
 242         droq = oct->droq[q_no];
 243         memset(droq, 0, OCT_DROQ_SIZE);
 244
 245         droq->oct_dev = oct;
 246         droq->q_no = q_no;
 247         if (app_ctx)
 248                 droq->app_ctx = app_ctx;
 249         else
 250                 droq->app_ctx = (void *)(size_t)q_no;
 251
 252         c_num_descs = num_descs;
 253         c_buf_size = desc_size;
 254         if (OCTEON_CN6XXX(oct)) {
 255                 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
 256
 257                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
 258                 c_refill_threshold =
 259                         (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
 260         } else if (OCTEON_CN23XX_PF(oct)) {
 261                 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
 262
 263                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 264                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 265         } else if (OCTEON_CN23XX_VF(oct)) {
 266                 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);
 267
 268                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 269                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 270         } else {
 271                 return 1;
 272         }
 273
 274         droq->max_count = c_num_descs;
 275         droq->buffer_size = c_buf_size;
 276
 277         desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
 278         set_dev_node(&oct->pci_dev->dev, numa_node);
 279         droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 280                                         (dma_addr_t *)&droq->desc_ring_dma);
 281         set_dev_node(&oct->pci_dev->dev, orig_node);
 282         if (!droq->desc_ring)
 283                 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 284                                         (dma_addr_t *)&droq->desc_ring_dma);
 285
 286         if (!droq->desc_ring) {
 287                 dev_err(&oct->pci_dev->dev,
 288                         "Output queue %d ring alloc failed\n", q_no);
 289                 return 1;
 290         }
 291
 292         dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 293                 q_no, droq->desc_ring, droq->desc_ring_dma);
 294         dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
 295                 droq->max_count);
 296
 297         droq->info_list =
 298                 cnnic_numa_alloc_aligned_dma((droq->max_count *
 299                                               OCT_DROQ_INFO_SIZE),
 300                                              &droq->info_alloc_size,
 301                                              &droq->info_base_addr,
 302                                              numa_node);
 303         if (!droq->info_list) {
 304                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
 305                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 306                              droq->desc_ring, droq->desc_ring_dma);
 307                 return 1;
 308         }
 309
 310         droq->recv_buf_list = (struct octeon_recv_buffer *)
 311                               vmalloc_node(droq->max_count *
 312                                                 OCT_DROQ_RECVBUF_SIZE,
 313                                                 numa_node);
 314         if (!droq->recv_buf_list)
 315                 droq->recv_buf_list = (struct octeon_recv_buffer *)
 316                                       vmalloc(droq->max_count *
 317                                                 OCT_DROQ_RECVBUF_SIZE);
 318         if (!droq->recv_buf_list) {
 319                 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
 320                 goto init_droq_fail;
 321         }
 322
 323         if (octeon_droq_setup_ring_buffers(oct, droq))
 324                 goto init_droq_fail;
 325
 326         droq->pkts_per_intr = c_pkts_per_intr;
 327         droq->refill_threshold = c_refill_threshold;
 328
 329         dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
 330                 droq->max_empty_descs);
 331
 332         spin_lock_init(&droq->lock);
 333
 334         INIT_LIST_HEAD(&droq->dispatch_list);
 335
 336         /* For 56xx Pass1, this function won't be called, so no checks. */
 337         oct->fn_list.setup_oq_regs(oct, q_no);
 338
 339         oct->io_qmask.oq |= BIT_ULL(q_no);
 340
 341         return 0;
 342
 343 init_droq_fail:
 344         octeon_delete_droq(oct, q_no);
 345         return 1;
 346 }
 347
 348 /* octeon_create_recv_info
 349  * Parameters:
 350  *  octeon_dev - pointer to the octeon device structure
 351  *  droq       - droq in which the packet arrived.
 352  *  buf_cnt    - no. of buffers used by the packet.
 353  *  idx        - index in the descriptor for the first buffer in the packet.
 354  * Description:
 355  *  Allocates a recv_info_t and copies the buffer addresses for packet data
 356  *  into the recv_pkt space which starts at an 8B offset from recv_info_t.
 357  *  Flags the descriptors for refill later. If available descriptors go
 358  *  below the threshold to receive a 64K pkt, new buffers are first allocated
 359  *  before the recv_pkt_t is created.
 360  *  This routine will be called in interrupt context.
 361  * Returns:
 362  *  Success: Pointer to recv_info_t
 363  *  Failure: NULL.
 364  * Locks:
 365  *  The droq->lock is held when this routine is called.
 366  */
 367 static inline struct octeon_recv_info *octeon_create_recv_info(
 368                 struct octeon_device *octeon_dev,
 369                 struct octeon_droq *droq,
 370                 u32 buf_cnt,
 371                 u32 idx)
 372 {
 373         struct octeon_droq_info *info;
 374         struct octeon_recv_pkt *recv_pkt;
 375         struct octeon_recv_info *recv_info;
 376         u32 i, bytes_left;
 377         struct octeon_skb_page_info *pg_info;
 378
 379         info = &droq->info_list[idx];
 380
 381         recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
 382         if (!recv_info)
 383                 return NULL;
 384
 385         recv_pkt = recv_info->recv_pkt;
 386         recv_pkt->rh = info->rh;
 387         recv_pkt->length = (u32)info->length;
 388         recv_pkt->buffer_count = (u16)buf_cnt;
 389         recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
 390
 391         i = 0;
 392         bytes_left = (u32)info->length;
 393
 394         while (buf_cnt) {
 395                 {
 396                         pg_info = &droq->recv_buf_list[idx].pg_info;
 397
 398                         lio_unmap_ring(octeon_dev->pci_dev,
 399                                        (u64)pg_info->dma);
 400                         pg_info->page = NULL;
 401                         pg_info->dma = 0;
 402                 }
 403
 404                 recv_pkt->buffer_size[i] =
 405                         (bytes_left >=
 406                          droq->buffer_size) ? droq->buffer_size : bytes_left;
 407
 408                 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
 409                 droq->recv_buf_list[idx].buffer = NULL;
 410
 411                 idx = incr_index(idx, 1, droq->max_count);
 412                 bytes_left -= droq->buffer_size;
 413                 i++;
 414                 buf_cnt--;
 415         }
 416
 417         return recv_info;
 418 }
 419
 420 /* If we were not able to refill all buffers, try to move around
 421  * the buffers that were not dispatched.
 422  */
 423 static inline u32
 424 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
 425                                 struct octeon_droq_desc *desc_ring)
 426 {
 427         u32 desc_refilled = 0;
 428
 429         u32 refill_index = droq->refill_idx;
 430
 431         while (refill_index != droq->read_idx) {
 432                 if (droq->recv_buf_list[refill_index].buffer) {
 433                         droq->recv_buf_list[droq->refill_idx].buffer =
 434                                 droq->recv_buf_list[refill_index].buffer;
 435                         droq->recv_buf_list[droq->refill_idx].data =
 436                                 droq->recv_buf_list[refill_index].data;
 437                         desc_ring[droq->refill_idx].buffer_ptr =
 438                                 desc_ring[refill_index].buffer_ptr;
 439                         droq->recv_buf_list[refill_index].buffer = NULL;
 440                         desc_ring[refill_index].buffer_ptr = 0;
 441                         do {
 442                                 droq->refill_idx = incr_index(droq->refill_idx,
 443                                                               1,
 444                                                               droq->max_count);
 445                                 desc_refilled++;
 446                                 droq->refill_count--;
 447                         } while (droq->recv_buf_list[droq->refill_idx].
 448                                  buffer);
 449                 }
 450                 refill_index = incr_index(refill_index, 1, droq->max_count);
 451         }                       /* while */
 452         return desc_refilled;
 453 }
 454
 455 /* octeon_droq_refill
 456  * Parameters:
 457  *  droq       - droq in which descriptors require new buffers.
 458  * Description:
 459  *  Called during normal DROQ processing in interrupt mode or by the poll
 460  *  thread to refill the descriptors from which buffers were dispatched
 461  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
 462  *  up buffers (that were not dispatched) to form a contiguous ring.
 463  * Returns:
 464  *  No of descriptors refilled.
 465  * Locks:
 466  *  This routine is called with droq->lock held.
 467  */
 468 static u32
 469 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
 470 {
 471         struct octeon_droq_desc *desc_ring;
 472         void *buf = NULL;
 473         u8 *data;
 474         u32 desc_refilled = 0;
 475         struct octeon_skb_page_info *pg_info;
 476
 477         desc_ring = droq->desc_ring;
 478
 479         while (droq->refill_count && (desc_refilled < droq->max_count)) {
 480                 /* If a valid buffer exists (happens if there is no dispatch),
 481                  * reuse
 482                  * the buffer, else allocate.
 483                  */
 484                 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
 485                         pg_info =
 486                                 &droq->recv_buf_list[droq->refill_idx].pg_info;
 487                         /* Either recycle the existing pages or go for
 488                          * new page alloc
 489                          */
 490                         if (pg_info->page)
 491                                 buf = recv_buffer_reuse(octeon_dev, pg_info);
 492                         else
 493                                 buf = recv_buffer_alloc(octeon_dev, pg_info);
 494                         /* If a buffer could not be allocated, no point in
 495                          * continuing
 496                          */
 497                         if (!buf) {
 498                                 droq->stats.rx_alloc_failure++;
 499                                 break;
 500                         }
 501                         droq->recv_buf_list[droq->refill_idx].buffer =
 502                                 buf;
 503                         data = get_rbd(buf);
 504                 } else {
 505                         data = get_rbd(droq->recv_buf_list
 506                                        [droq->refill_idx].buffer);
 507                 }
 508
 509                 droq->recv_buf_list[droq->refill_idx].data = data;
 510
 511                 desc_ring[droq->refill_idx].buffer_ptr =
 512                         lio_map_ring(droq->recv_buf_list[droq->
 513                                      refill_idx].buffer);
 514                 /* Reset any previous values in the length field. */
 515                 droq->info_list[droq->refill_idx].length = 0;
 516
 517                 droq->refill_idx = incr_index(droq->refill_idx, 1,
 518                                               droq->max_count);
 519                 desc_refilled++;
 520                 droq->refill_count--;
 521         }
 522
 523         if (droq->refill_count)
 524                 desc_refilled +=
 525                         octeon_droq_refill_pullup_descs(droq, desc_ring);
 526
 527         /* if droq->refill_count
 528          * The refill count would not change in pass two. We only moved buffers
 529          * to close the gap in the ring, but we would still have the same no. of
 530          * buffers to refill.
 531          */
 532         return desc_refilled;
 533 }
 534
 535 static inline u32
 536 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
 537 {
 538         u32 buf_cnt = 0;
 539
 540         while (total_len > (buf_size * buf_cnt))
 541                 buf_cnt++;
 542         return buf_cnt;
 543 }
 544
 545 static int
 546 octeon_droq_dispatch_pkt(struct octeon_device *oct,
 547                          struct octeon_droq *droq,
 548                          union octeon_rh *rh,
 549                          struct octeon_droq_info *info)
 550 {
 551         u32 cnt;
 552         octeon_dispatch_fn_t disp_fn;
 553         struct octeon_recv_info *rinfo;
 554
 555         cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
 556
 557         disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
 558                                       (u16)rh->r.subcode);
 559         if (disp_fn) {
 560                 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
 561                 if (rinfo) {
 562                         struct __dispatch *rdisp = rinfo->rsvd;
 563
 564                         rdisp->rinfo = rinfo;
 565                         rdisp->disp_fn = disp_fn;
 566                         rinfo->recv_pkt->rh = *rh;
 567                         list_add_tail(&rdisp->list,
 568                                       &droq->dispatch_list);
 569                 } else {
 570                         droq->stats.dropped_nomem++;
 571                 }
 572         } else {
 573                 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
 574                         (unsigned int)rh->r.opcode,
 575                         (unsigned int)rh->r.subcode);
 576                 droq->stats.dropped_nodispatch++;
 577         }
 578
 579         return cnt;
 580 }
 581
 582 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
 583                                             struct octeon_droq *droq,
 584                                             u32 cnt)
 585 {
 586         u32 i = 0, buf_cnt;
 587         struct octeon_droq_info *info;
 588
 589         for (i = 0; i < cnt; i++) {
 590                 info = &droq->info_list[droq->read_idx];
 591                 octeon_swap_8B_data((u64 *)info, 2);
 592
 593                 if (info->length) {
 594                         info->length -= OCT_RH_SIZE;
 595                         droq->stats.bytes_received += info->length;
 596                         buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
 597                                                            (u32)info->length);
 598                 } else {
 599                         dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
 600                         buf_cnt = 1;
 601                 }
 602
 603                 droq->read_idx = incr_index(droq->read_idx, buf_cnt,
 604                                             droq->max_count);
 605                 droq->refill_count += buf_cnt;
 606         }
 607 }
 608
 609 static u32
 610 octeon_droq_fast_process_packets(struct octeon_device *oct,
 611                                  struct octeon_droq *droq,
 612                                  u32 pkts_to_process)
 613 {
 614         struct octeon_droq_info *info;
 615         union octeon_rh *rh;
 616         u32 pkt, total_len = 0, pkt_count;
 617
 618         pkt_count = pkts_to_process;
 619
 620         for (pkt = 0; pkt < pkt_count; pkt++) {
 621                 u32 pkt_len = 0;
 622                 struct sk_buff *nicbuf = NULL;
 623                 struct octeon_skb_page_info *pg_info;
 624                 void *buf;
 625
 626                 info = &droq->info_list[droq->read_idx];
 627                 octeon_swap_8B_data((u64 *)info, 2);
 628
 629                 if (!info->length) {
 630                         dev_err(&oct->pci_dev->dev,
 631                                 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
 632                                 droq->q_no, droq->read_idx, pkt_count);
 633                         print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
 634                                              (u8 *)info,
 635                                              OCT_DROQ_INFO_SIZE);
 636                         break;
 637                 }
 638
 639                 /* Len of resp hdr in included in the received data len. */
 640                 info->length -= OCT_RH_SIZE;
 641                 rh = &info->rh;
 642
 643                 total_len += (u32)info->length;
 644                 if (opcode_slow_path(rh)) {
 645                         u32 buf_cnt;
 646
 647                         buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
 648                         droq->read_idx = incr_index(droq->read_idx,
 649                                                     buf_cnt, droq->max_count);
 650                         droq->refill_count += buf_cnt;
 651                 } else {
 652                         if (info->length <= droq->buffer_size) {
 653                                 pkt_len = (u32)info->length;
 654                                 nicbuf = droq->recv_buf_list[
 655                                         droq->read_idx].buffer;
 656                                 pg_info = &droq->recv_buf_list[
 657                                         droq->read_idx].pg_info;
 658                                 if (recv_buffer_recycle(oct, pg_info))
 659                                         pg_info->page = NULL;
 660                                 droq->recv_buf_list[droq->read_idx].buffer =
 661                                         NULL;
 662
 663                                 droq->read_idx = incr_index(droq->read_idx, 1,
 664                                                             droq->max_count);
 665                                 droq->refill_count++;
 666                         } else {
 667                                 nicbuf = octeon_fast_packet_alloc((u32)
 668                                                                   info->length);
 669                                 pkt_len = 0;
 670                                 /* nicbuf allocation can fail. We'll handle it
 671                                  * inside the loop.
 672                                  */
 673                                 while (pkt_len < info->length) {
 674                                         int cpy_len, idx = droq->read_idx;
 675
 676                                         cpy_len = ((pkt_len + droq->buffer_size)
 677                                                    > info->length) ?
 678                                                 ((u32)info->length - pkt_len) :
 679                                                 droq->buffer_size;
 680
 681                                         if (nicbuf) {
 682                                                 octeon_fast_packet_next(droq,
 683                                                                         nicbuf,
 684                                                                         cpy_len,
 685                                                                         idx);
 686                                                 buf = droq->recv_buf_list[idx].
 687                                                         buffer;
 688                                                 recv_buffer_fast_free(buf);
 689                                                 droq->recv_buf_list[idx].buffer
 690                                                         = NULL;
 691                                         } else {
 692                                                 droq->stats.rx_alloc_failure++;
 693                                         }
 694
 695                                         pkt_len += cpy_len;
 696                                         droq->read_idx =
 697                                                 incr_index(droq->read_idx, 1,
 698                                                            droq->max_count);
 699                                         droq->refill_count++;
 700                                 }
 701                         }
 702
 703                         if (nicbuf) {
 704                                 if (droq->ops.fptr) {
 705                                         droq->ops.fptr(oct->octeon_id,
 706                                                        nicbuf, pkt_len,
 707                                                        rh, &droq->napi,
 708                                                        droq->ops.farg);
 709                                 } else {
 710                                         recv_buffer_free(nicbuf);
 711                                 }
 712                         }
 713                 }
 714
 715                 if (droq->refill_count >= droq->refill_threshold) {
 716                         int desc_refilled = octeon_droq_refill(oct, droq);
 717
 718                         /* Flush the droq descriptor data to memory to be sure
 719                          * that when we update the credits the data in memory
 720                          * is accurate.
 721                          */
 722                         wmb();
 723                         writel((desc_refilled), droq->pkts_credit_reg);
 724                         /* make sure mmio write completes */
 725                         mmiowb();
 726                 }
 727
 728         }                       /* for (each packet)... */
 729
 730         /* Increment refill_count by the number of buffers processed. */
 731         droq->stats.pkts_received += pkt;
 732         droq->stats.bytes_received += total_len;
 733
 734         if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
 735                 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
 736
 737                 droq->stats.dropped_toomany += (pkts_to_process - pkt);
 738                 return pkts_to_process;
 739         }
 740
 741         return pkt;
 742 }
 743
 744 int
 745 octeon_droq_process_packets(struct octeon_device *oct,
 746                             struct octeon_droq *droq,
 747                             u32 budget)
 748 {
 749         u32 pkt_count = 0, pkts_processed = 0;
 750         struct list_head *tmp, *tmp2;
 751
 752         /* Grab the droq lock */
 753         spin_lock(&droq->lock);
 754
 755         octeon_droq_check_hw_for_pkts(droq);
 756         pkt_count = atomic_read(&droq->pkts_pending);
 757
 758         if (!pkt_count) {
 759                 spin_unlock(&droq->lock);
 760                 return 0;
 761         }
 762
 763         if (pkt_count > budget)
 764                 pkt_count = budget;
 765
 766         pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
 767
 768         atomic_sub(pkts_processed, &droq->pkts_pending);
 769
 770         /* Release the spin lock */
 771         spin_unlock(&droq->lock);
 772
 773         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 774                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 775
 776                 list_del(tmp);
 777                 rdisp->disp_fn(rdisp->rinfo,
 778                                octeon_get_dispatch_arg
 779                                (oct,
 780                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 781                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 782         }
 783
 784         /* If there are packets pending. schedule tasklet again */
 785         if (atomic_read(&droq->pkts_pending))
 786                 return 1;
 787
 788         return 0;
 789 }
 790
 791 /**
 792  * Utility function to poll for packets. check_hw_for_packets must be
 793  * called before calling this routine.
 794  */
 795
 796 static int
 797 octeon_droq_process_poll_pkts(struct octeon_device *oct,
 798                               struct octeon_droq *droq, u32 budget)
 799 {
 800         struct list_head *tmp, *tmp2;
 801         u32 pkts_available = 0, pkts_processed = 0;
 802         u32 total_pkts_processed = 0;
 803
 804         if (budget > droq->max_count)
 805                 budget = droq->max_count;
 806
 807         spin_lock(&droq->lock);
 808
 809         while (total_pkts_processed < budget) {
 810                 octeon_droq_check_hw_for_pkts(droq);
 811
 812                 pkts_available = min((budget - total_pkts_processed),
 813                                      (u32)(atomic_read(&droq->pkts_pending)));
 814
 815                 if (pkts_available == 0)
 816                         break;
 817
 818                 pkts_processed =
 819                         octeon_droq_fast_process_packets(oct, droq,
 820                                                          pkts_available);
 821
 822                 atomic_sub(pkts_processed, &droq->pkts_pending);
 823
 824                 total_pkts_processed += pkts_processed;
 825         }
 826
 827         spin_unlock(&droq->lock);
 828
 829         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 830                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 831
 832                 list_del(tmp);
 833                 rdisp->disp_fn(rdisp->rinfo,
 834                                octeon_get_dispatch_arg
 835                                (oct,
 836                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 837                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 838         }
 839
 840         return total_pkts_processed;
 841 }
 842
 843 int
 844 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
 845                              u32 arg)
 846 {
 847         struct octeon_droq *droq;
 848
 849         droq = oct->droq[q_no];
 850
 851         if (cmd == POLL_EVENT_PROCESS_PKTS)
 852                 return octeon_droq_process_poll_pkts(oct, droq, arg);
 853
 854         if (cmd == POLL_EVENT_PENDING_PKTS) {
 855                 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
 856
 857                 return  octeon_droq_process_packets(oct, droq, pkt_cnt);
 858         }
 859
 860         if (cmd == POLL_EVENT_ENABLE_INTR) {
 861                 u32 value;
 862                 unsigned long flags;
 863
 864                 /* Enable Pkt Interrupt */
 865                 switch (oct->chip_id) {
 866                 case OCTEON_CN66XX:
 867                 case OCTEON_CN68XX: {
 868                         struct octeon_cn6xxx *cn6xxx =
 869                                 (struct octeon_cn6xxx *)oct->chip;
 870                         spin_lock_irqsave
 871                                 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 872                         value =
 873                                 octeon_read_csr(oct,
 874                                                 CN6XXX_SLI_PKT_TIME_INT_ENB);
 875                         value |= (1 << q_no);
 876                         octeon_write_csr(oct,
 877                                          CN6XXX_SLI_PKT_TIME_INT_ENB,
 878                                          value);
 879                         value =
 880                                 octeon_read_csr(oct,
 881                                                 CN6XXX_SLI_PKT_CNT_INT_ENB);
 882                         value |= (1 << q_no);
 883                         octeon_write_csr(oct,
 884                                          CN6XXX_SLI_PKT_CNT_INT_ENB,
 885                                          value);
 886
 887                         /* don't bother flushing the enables */
 888
 889                         spin_unlock_irqrestore
 890                                 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 891                         return 0;
 892                 }
 893                 break;
 894                 case OCTEON_CN23XX_PF_VID: {
 895                         lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 896                 }
 897                 break;
 898
 899                 case OCTEON_CN23XX_VF_VID:
 900                         lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 901                 break;
 902                 }
 903                 return 0;
 904         }
 905
 906         dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
 907         return -EINVAL;
 908 }
 909
 910 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
 911                              struct octeon_droq_ops *ops)
 912 {
 913         struct octeon_droq *droq;
 914         unsigned long flags;
 915         struct octeon_config *oct_cfg = NULL;
 916
 917         oct_cfg = octeon_get_conf(oct);
 918
 919         if (!oct_cfg)
 920                 return -EINVAL;
 921
 922         if (!(ops)) {
 923                 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
 924                         __func__);
 925                 return -EINVAL;
 926         }
 927
 928         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 929                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 930                         __func__, q_no, (oct->num_oqs - 1));
 931                 return -EINVAL;
 932         }
 933
 934         droq = oct->droq[q_no];
 935
 936         spin_lock_irqsave(&droq->lock, flags);
 937
 938         memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
 939
 940         spin_unlock_irqrestore(&droq->lock, flags);
 941
 942         return 0;
 943 }
 944
 945 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
 946 {
 947         unsigned long flags;
 948         struct octeon_droq *droq;
 949         struct octeon_config *oct_cfg = NULL;
 950
 951         oct_cfg = octeon_get_conf(oct);
 952
 953         if (!oct_cfg)
 954                 return -EINVAL;
 955
 956         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 957                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 958                         __func__, q_no, oct->num_oqs - 1);
 959                 return -EINVAL;
 960         }
 961
 962         droq = oct->droq[q_no];
 963
 964         if (!droq) {
 965                 dev_info(&oct->pci_dev->dev,
 966                          "Droq id (%d) not available.\n", q_no);
 967                 return 0;
 968         }
 969
 970         spin_lock_irqsave(&droq->lock, flags);
 971
 972         droq->ops.fptr = NULL;
 973         droq->ops.farg = NULL;
 974         droq->ops.drop_on_max = 0;
 975
 976         spin_unlock_irqrestore(&droq->lock, flags);
 977
 978         return 0;
 979 }
 980
 981 int octeon_create_droq(struct octeon_device *oct,
 982                        u32 q_no, u32 num_descs,
 983                        u32 desc_size, void *app_ctx)
 984 {
 985         struct octeon_droq *droq;
 986         int numa_node = cpu_to_node(q_no % num_online_cpus());
 987
 988         if (oct->droq[q_no]) {
 989                 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
 990                         q_no);
 991                 return 1;
 992         }
 993
 994         /* Allocate the DS for the new droq. */
 995         droq = vmalloc_node(sizeof(*droq), numa_node);
 996         if (!droq)
 997                 droq = vmalloc(sizeof(*droq));
 998         if (!droq)
 999                 return -1;
1000
1001         memset(droq, 0, sizeof(struct octeon_droq));
1002
1003         /*Disable the pkt o/p for this Q  */
1004         octeon_set_droq_pkt_op(oct, q_no, 0);
1005         oct->droq[q_no] = droq;
1006
1007         /* Initialize the Droq */
1008         if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
1009                 vfree(oct->droq[q_no]);
1010                 oct->droq[q_no] = NULL;
1011                 return -1;
1012         }
1013
1014         oct->num_oqs++;
1015
1016         dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
1017                 oct->num_oqs);
1018
1019         /* Global Droq register settings */
1020
1021         /* As of now not required, as setting are done for all 32 Droqs at
1022          * the same time.
1023          */
1024         return 0;
1025 }