2 * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/skbuff.h>
33 #include <linux/netdevice.h>
34 #include <linux/etherdevice.h>
35 #include <linux/if_vlan.h>
37 #include <linux/tcp.h>
38 #include <linux/dma-mapping.h>
44 #include "firmware_exports.h"
48 #define SGE_RX_SM_BUF_SIZE 1536
50 #define SGE_RX_COPY_THRES 256
51 #define SGE_RX_PULL_LEN 128
53 #define SGE_PG_RSVD SMP_CACHE_BYTES
55 * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
56 * It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs
59 #define FL0_PG_CHUNK_SIZE 2048
60 #define FL0_PG_ORDER 0
61 #define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
62 #define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
63 #define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
64 #define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
66 #define SGE_RX_DROP_THRES 16
67 #define RX_RECLAIM_PERIOD (HZ/4)
70 * Max number of Rx buffers we replenish at a time.
72 #define MAX_RX_REFILL 16U
74 * Period of the Tx buffer reclaim timer. This timer does not need to run
75 * frequently as Tx buffers are usually reclaimed by new Tx packets.
77 #define TX_RECLAIM_PERIOD (HZ / 4)
78 #define TX_RECLAIM_TIMER_CHUNK 64U
79 #define TX_RECLAIM_CHUNK 16U
81 /* WR size in bytes */
82 #define WR_LEN (WR_FLITS * 8)
85 * Types of Tx queues in each queue set. Order here matters, do not change.
87 enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
89 /* Values for sge_txq.flags */
91 TXQ_RUNNING = 1 << 0, /* fetch engine is running */
92 TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
96 __be64 flit[TX_DESC_FLITS];
106 struct tx_sw_desc { /* SW state per Tx descriptor */
108 u8 eop; /* set if last descriptor for packet */
109 u8 addr_idx; /* buffer index of first SGL entry in descriptor */
110 u8 fragidx; /* first page fragment associated with descriptor */
111 s8 sflit; /* start flit of first SGL entry in descriptor */
114 struct rx_sw_desc { /* SW state per Rx descriptor */
117 struct fl_pg_chunk pg_chunk;
119 DECLARE_PCI_UNMAP_ADDR(dma_addr);
122 struct rsp_desc { /* response queue descriptor */
123 struct rss_header rss_hdr;
131 * Holds unmapping information for Tx packets that need deferred unmapping.
132 * This structure lives at skb->head and must be allocated by callers.
134 struct deferred_unmap_info {
135 struct pci_dev *pdev;
136 dma_addr_t addr[MAX_SKB_FRAGS + 1];
140 * Maps a number of flits to the number of Tx descriptors that can hold them.
143 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
145 * HW allows up to 4 descriptors to be combined into a WR.
147 static u8 flit_desc_map[] = {
149 #if SGE_NUM_GENBITS == 1
150 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
151 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
152 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
153 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
154 #elif SGE_NUM_GENBITS == 2
155 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
156 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
157 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
158 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
160 # error "SGE_NUM_GENBITS must be 1 or 2"
164 static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
166 return container_of(q, struct sge_qset, fl[qidx]);
169 static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
171 return container_of(q, struct sge_qset, rspq);
174 static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
176 return container_of(q, struct sge_qset, txq[qidx]);
180 * refill_rspq - replenish an SGE response queue
181 * @adapter: the adapter
182 * @q: the response queue to replenish
183 * @credits: how many new responses to make available
185 * Replenishes a response queue by making the supplied number of responses
188 static inline void refill_rspq(struct adapter *adapter,
189 const struct sge_rspq *q, unsigned int credits)
192 t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
193 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
197 * need_skb_unmap - does the platform need unmapping of sk_buffs?
199 * Returns true if the platfrom needs sk_buff unmapping. The compiler
200 * optimizes away unecessary code if this returns true.
202 static inline int need_skb_unmap(void)
205 * This structure is used to tell if the platfrom needs buffer
206 * unmapping by checking if DECLARE_PCI_UNMAP_ADDR defines anything.
209 DECLARE_PCI_UNMAP_ADDR(addr);
212 return sizeof(struct dummy) != 0;
216 * unmap_skb - unmap a packet main body and its page fragments
218 * @q: the Tx queue containing Tx descriptors for the packet
219 * @cidx: index of Tx descriptor
220 * @pdev: the PCI device
222 * Unmap the main body of an sk_buff and its page fragments, if any.
223 * Because of the fairly complicated structure of our SGLs and the desire
224 * to conserve space for metadata, the information necessary to unmap an
225 * sk_buff is spread across the sk_buff itself (buffer lengths), the HW Tx
226 * descriptors (the physical addresses of the various data buffers), and
227 * the SW descriptor state (assorted indices). The send functions
228 * initialize the indices for the first packet descriptor so we can unmap
229 * the buffers held in the first Tx descriptor here, and we have enough
230 * information at this point to set the state for the next Tx descriptor.
232 * Note that it is possible to clean up the first descriptor of a packet
233 * before the send routines have written the next descriptors, but this
234 * race does not cause any problem. We just end up writing the unmapping
235 * info for the descriptor first.
237 static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
238 unsigned int cidx, struct pci_dev *pdev)
240 const struct sg_ent *sgp;
241 struct tx_sw_desc *d = &q->sdesc[cidx];
242 int nfrags, frag_idx, curflit, j = d->addr_idx;
244 sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit];
245 frag_idx = d->fragidx;
247 if (frag_idx == 0 && skb_headlen(skb)) {
248 pci_unmap_single(pdev, be64_to_cpu(sgp->addr[0]),
249 skb_headlen(skb), PCI_DMA_TODEVICE);
253 curflit = d->sflit + 1 + j;
254 nfrags = skb_shinfo(skb)->nr_frags;
256 while (frag_idx < nfrags && curflit < WR_FLITS) {
257 pci_unmap_page(pdev, be64_to_cpu(sgp->addr[j]),
258 skb_shinfo(skb)->frags[frag_idx].size,
269 if (frag_idx < nfrags) { /* SGL continues into next Tx descriptor */
270 d = cidx + 1 == q->size ? q->sdesc : d + 1;
271 d->fragidx = frag_idx;
273 d->sflit = curflit - WR_FLITS - j; /* sflit can be -1 */
278 * free_tx_desc - reclaims Tx descriptors and their buffers
279 * @adapter: the adapter
280 * @q: the Tx queue to reclaim descriptors from
281 * @n: the number of descriptors to reclaim
283 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
284 * Tx buffers. Called with the Tx queue lock held.
286 static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
289 struct tx_sw_desc *d;
290 struct pci_dev *pdev = adapter->pdev;
291 unsigned int cidx = q->cidx;
293 const int need_unmap = need_skb_unmap() &&
294 q->cntxt_id >= FW_TUNNEL_SGEEC_START;
298 if (d->skb) { /* an SGL is present */
300 unmap_skb(d->skb, q, cidx, pdev);
305 if (++cidx == q->size) {
314 * reclaim_completed_tx - reclaims completed Tx descriptors
315 * @adapter: the adapter
316 * @q: the Tx queue to reclaim completed descriptors from
317 * @chunk: maximum number of descriptors to reclaim
319 * Reclaims Tx descriptors that the SGE has indicated it has processed,
320 * and frees the associated buffers if possible. Called with the Tx
323 static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
327 unsigned int reclaim = q->processed - q->cleaned;
329 reclaim = min(chunk, reclaim);
331 free_tx_desc(adapter, q, reclaim);
332 q->cleaned += reclaim;
333 q->in_use -= reclaim;
335 return q->processed - q->cleaned;
339 * should_restart_tx - are there enough resources to restart a Tx queue?
342 * Checks if there are enough descriptors to restart a suspended Tx queue.
344 static inline int should_restart_tx(const struct sge_txq *q)
346 unsigned int r = q->processed - q->cleaned;
348 return q->in_use - r < (q->size >> 1);
351 static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
352 struct rx_sw_desc *d)
354 if (q->use_pages && d->pg_chunk.page) {
355 (*d->pg_chunk.p_cnt)--;
356 if (!*d->pg_chunk.p_cnt)
359 q->alloc_size, PCI_DMA_FROMDEVICE);
361 put_page(d->pg_chunk.page);
362 d->pg_chunk.page = NULL;
364 pci_unmap_single(pdev, pci_unmap_addr(d, dma_addr),
365 q->buf_size, PCI_DMA_FROMDEVICE);
372 * free_rx_bufs - free the Rx buffers on an SGE free list
373 * @pdev: the PCI device associated with the adapter
374 * @rxq: the SGE free list to clean up
376 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
377 * this queue should be stopped before calling this function.
379 static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
381 unsigned int cidx = q->cidx;
383 while (q->credits--) {
384 struct rx_sw_desc *d = &q->sdesc[cidx];
387 clear_rx_desc(pdev, q, d);
388 if (++cidx == q->size)
392 if (q->pg_chunk.page) {
393 __free_pages(q->pg_chunk.page, q->order);
394 q->pg_chunk.page = NULL;
399 * add_one_rx_buf - add a packet buffer to a free-buffer list
400 * @va: buffer start VA
401 * @len: the buffer length
402 * @d: the HW Rx descriptor to write
403 * @sd: the SW Rx descriptor to write
404 * @gen: the generation bit value
405 * @pdev: the PCI device associated with the adapter
407 * Add a buffer of the given length to the supplied HW and SW Rx
410 static inline int add_one_rx_buf(void *va, unsigned int len,
411 struct rx_desc *d, struct rx_sw_desc *sd,
412 unsigned int gen, struct pci_dev *pdev)
416 mapping = pci_map_single(pdev, va, len, PCI_DMA_FROMDEVICE);
417 if (unlikely(pci_dma_mapping_error(pdev, mapping)))
420 pci_unmap_addr_set(sd, dma_addr, mapping);
422 d->addr_lo = cpu_to_be32(mapping);
423 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
425 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
426 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
430 static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
433 d->addr_lo = cpu_to_be32(mapping);
434 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
436 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
437 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
441 static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
442 struct rx_sw_desc *sd, gfp_t gfp,
445 if (!q->pg_chunk.page) {
448 q->pg_chunk.page = alloc_pages(gfp, order);
449 if (unlikely(!q->pg_chunk.page))
451 q->pg_chunk.va = page_address(q->pg_chunk.page);
452 q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
454 q->pg_chunk.offset = 0;
455 mapping = pci_map_page(adapter->pdev, q->pg_chunk.page,
456 0, q->alloc_size, PCI_DMA_FROMDEVICE);
457 q->pg_chunk.mapping = mapping;
459 sd->pg_chunk = q->pg_chunk;
461 prefetch(sd->pg_chunk.p_cnt);
463 q->pg_chunk.offset += q->buf_size;
464 if (q->pg_chunk.offset == (PAGE_SIZE << order))
465 q->pg_chunk.page = NULL;
467 q->pg_chunk.va += q->buf_size;
468 get_page(q->pg_chunk.page);
471 if (sd->pg_chunk.offset == 0)
472 *sd->pg_chunk.p_cnt = 1;
474 *sd->pg_chunk.p_cnt += 1;
479 static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
481 if (q->pend_cred >= q->credits / 4) {
483 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
488 * refill_fl - refill an SGE free-buffer list
489 * @adapter: the adapter
490 * @q: the free-list to refill
491 * @n: the number of new buffers to allocate
492 * @gfp: the gfp flags for allocating new buffers
494 * (Re)populate an SGE free-buffer list with up to @n new packet buffers,
495 * allocated with the supplied gfp flags. The caller must assure that
496 * @n does not exceed the queue's capacity.
498 static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
500 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
501 struct rx_desc *d = &q->desc[q->pidx];
502 unsigned int count = 0;
509 if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
511 nomem: q->alloc_failed++;
514 mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
515 pci_unmap_addr_set(sd, dma_addr, mapping);
517 add_one_rx_chunk(mapping, d, q->gen);
518 pci_dma_sync_single_for_device(adap->pdev, mapping,
519 q->buf_size - SGE_PG_RSVD,
524 struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
529 buf_start = skb->data;
530 err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
533 clear_rx_desc(adap->pdev, q, sd);
540 if (++q->pidx == q->size) {
550 q->pend_cred += count;
556 static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
558 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits),
559 GFP_ATOMIC | __GFP_COMP);
563 * recycle_rx_buf - recycle a receive buffer
564 * @adapter: the adapter
565 * @q: the SGE free list
566 * @idx: index of buffer to recycle
568 * Recycles the specified buffer on the given free list by adding it at
569 * the next available slot on the list.
571 static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
574 struct rx_desc *from = &q->desc[idx];
575 struct rx_desc *to = &q->desc[q->pidx];
577 q->sdesc[q->pidx] = q->sdesc[idx];
578 to->addr_lo = from->addr_lo; /* already big endian */
579 to->addr_hi = from->addr_hi; /* likewise */
581 to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
582 to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
584 if (++q->pidx == q->size) {
595 * alloc_ring - allocate resources for an SGE descriptor ring
596 * @pdev: the PCI device
597 * @nelem: the number of descriptors
598 * @elem_size: the size of each descriptor
599 * @sw_size: the size of the SW state associated with each ring element
600 * @phys: the physical address of the allocated ring
601 * @metadata: address of the array holding the SW state for the ring
603 * Allocates resources for an SGE descriptor ring, such as Tx queues,
604 * free buffer lists, or response queues. Each SGE ring requires
605 * space for its HW descriptors plus, optionally, space for the SW state
606 * associated with each HW entry (the metadata). The function returns
607 * three values: the virtual address for the HW ring (the return value
608 * of the function), the physical address of the HW ring, and the address
611 static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
612 size_t sw_size, dma_addr_t * phys, void *metadata)
614 size_t len = nelem * elem_size;
616 void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
620 if (sw_size && metadata) {
621 s = kcalloc(nelem, sw_size, GFP_KERNEL);
624 dma_free_coherent(&pdev->dev, len, p, *phys);
627 *(void **)metadata = s;
634 * t3_reset_qset - reset a sge qset
637 * Reset the qset structure.
638 * the NAPI structure is preserved in the event of
639 * the qset's reincarnation, for example during EEH recovery.
641 static void t3_reset_qset(struct sge_qset *q)
644 !(q->adap->flags & NAPI_INIT)) {
645 memset(q, 0, sizeof(*q));
650 memset(&q->rspq, 0, sizeof(q->rspq));
651 memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET);
652 memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
654 q->tx_reclaim_timer.function = NULL; /* for t3_stop_sge_timers() */
655 q->rx_reclaim_timer.function = NULL;
657 napi_free_frags(&q->napi);
662 * free_qset - free the resources of an SGE queue set
663 * @adapter: the adapter owning the queue set
666 * Release the HW and SW resources associated with an SGE queue set, such
667 * as HW contexts, packet buffers, and descriptor rings. Traffic to the
668 * queue set must be quiesced prior to calling this.
670 static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
673 struct pci_dev *pdev = adapter->pdev;
675 for (i = 0; i < SGE_RXQ_PER_SET; ++i)
677 spin_lock_irq(&adapter->sge.reg_lock);
678 t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
679 spin_unlock_irq(&adapter->sge.reg_lock);
680 free_rx_bufs(pdev, &q->fl[i]);
681 kfree(q->fl[i].sdesc);
682 dma_free_coherent(&pdev->dev,
684 sizeof(struct rx_desc), q->fl[i].desc,
688 for (i = 0; i < SGE_TXQ_PER_SET; ++i)
689 if (q->txq[i].desc) {
690 spin_lock_irq(&adapter->sge.reg_lock);
691 t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
692 spin_unlock_irq(&adapter->sge.reg_lock);
693 if (q->txq[i].sdesc) {
694 free_tx_desc(adapter, &q->txq[i],
696 kfree(q->txq[i].sdesc);
698 dma_free_coherent(&pdev->dev,
700 sizeof(struct tx_desc),
701 q->txq[i].desc, q->txq[i].phys_addr);
702 __skb_queue_purge(&q->txq[i].sendq);
706 spin_lock_irq(&adapter->sge.reg_lock);
707 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
708 spin_unlock_irq(&adapter->sge.reg_lock);
709 dma_free_coherent(&pdev->dev,
710 q->rspq.size * sizeof(struct rsp_desc),
711 q->rspq.desc, q->rspq.phys_addr);
718 * init_qset_cntxt - initialize an SGE queue set context info
720 * @id: the queue set id
722 * Initializes the TIDs and context ids for the queues of a queue set.
724 static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
726 qs->rspq.cntxt_id = id;
727 qs->fl[0].cntxt_id = 2 * id;
728 qs->fl[1].cntxt_id = 2 * id + 1;
729 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
730 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
731 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
732 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
733 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
737 * sgl_len - calculates the size of an SGL of the given capacity
738 * @n: the number of SGL entries
740 * Calculates the number of flits needed for a scatter/gather list that
741 * can hold the given number of entries.
743 static inline unsigned int sgl_len(unsigned int n)
745 /* alternatively: 3 * (n / 2) + 2 * (n & 1) */
746 return (3 * n) / 2 + (n & 1);
750 * flits_to_desc - returns the num of Tx descriptors for the given flits
751 * @n: the number of flits
753 * Calculates the number of Tx descriptors needed for the supplied number
756 static inline unsigned int flits_to_desc(unsigned int n)
758 BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
759 return flit_desc_map[n];
763 * get_packet - return the next ingress packet buffer from a free list
764 * @adap: the adapter that received the packet
765 * @fl: the SGE free list holding the packet
766 * @len: the packet length including any SGE padding
767 * @drop_thres: # of remaining buffers before we start dropping packets
769 * Get the next packet from a free list and complete setup of the
770 * sk_buff. If the packet is small we make a copy and recycle the
771 * original buffer, otherwise we use the original buffer itself. If a
772 * positive drop threshold is supplied packets are dropped and their
773 * buffers recycled if (a) the number of remaining buffers is under the
774 * threshold and the packet is too big to copy, or (b) the packet should
775 * be copied but there is no memory for the copy.
777 static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
778 unsigned int len, unsigned int drop_thres)
780 struct sk_buff *skb = NULL;
781 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
783 prefetch(sd->skb->data);
786 if (len <= SGE_RX_COPY_THRES) {
787 skb = alloc_skb(len, GFP_ATOMIC);
788 if (likely(skb != NULL)) {
790 pci_dma_sync_single_for_cpu(adap->pdev,
791 pci_unmap_addr(sd, dma_addr), len,
793 memcpy(skb->data, sd->skb->data, len);
794 pci_dma_sync_single_for_device(adap->pdev,
795 pci_unmap_addr(sd, dma_addr), len,
797 } else if (!drop_thres)
800 recycle_rx_buf(adap, fl, fl->cidx);
804 if (unlikely(fl->credits < drop_thres) &&
805 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1),
806 GFP_ATOMIC | __GFP_COMP) == 0)
810 pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
811 fl->buf_size, PCI_DMA_FROMDEVICE);
814 __refill_fl(adap, fl);
819 * get_packet_pg - return the next ingress packet buffer from a free list
820 * @adap: the adapter that received the packet
821 * @fl: the SGE free list holding the packet
822 * @len: the packet length including any SGE padding
823 * @drop_thres: # of remaining buffers before we start dropping packets
825 * Get the next packet from a free list populated with page chunks.
826 * If the packet is small we make a copy and recycle the original buffer,
827 * otherwise we attach the original buffer as a page fragment to a fresh
828 * sk_buff. If a positive drop threshold is supplied packets are dropped
829 * and their buffers recycled if (a) the number of remaining buffers is
830 * under the threshold and the packet is too big to copy, or (b) there's
833 * Note: this function is similar to @get_packet but deals with Rx buffers
834 * that are page chunks rather than sk_buffs.
836 static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
837 struct sge_rspq *q, unsigned int len,
838 unsigned int drop_thres)
840 struct sk_buff *newskb, *skb;
841 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
843 dma_addr_t dma_addr = pci_unmap_addr(sd, dma_addr);
845 newskb = skb = q->pg_skb;
846 if (!skb && (len <= SGE_RX_COPY_THRES)) {
847 newskb = alloc_skb(len, GFP_ATOMIC);
848 if (likely(newskb != NULL)) {
849 __skb_put(newskb, len);
850 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
852 memcpy(newskb->data, sd->pg_chunk.va, len);
853 pci_dma_sync_single_for_device(adap->pdev, dma_addr,
856 } else if (!drop_thres)
860 recycle_rx_buf(adap, fl, fl->cidx);
865 if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
868 prefetch(sd->pg_chunk.p_cnt);
871 newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
873 if (unlikely(!newskb)) {
879 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
881 (*sd->pg_chunk.p_cnt)--;
882 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
883 pci_unmap_page(adap->pdev,
884 sd->pg_chunk.mapping,
888 __skb_put(newskb, SGE_RX_PULL_LEN);
889 memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
890 skb_fill_page_desc(newskb, 0, sd->pg_chunk.page,
891 sd->pg_chunk.offset + SGE_RX_PULL_LEN,
892 len - SGE_RX_PULL_LEN);
894 newskb->data_len = len - SGE_RX_PULL_LEN;
895 newskb->truesize += newskb->data_len;
897 skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags,
899 sd->pg_chunk.offset, len);
901 newskb->data_len += len;
902 newskb->truesize += len;
907 * We do not refill FLs here, we let the caller do it to overlap a
914 * get_imm_packet - return the next ingress packet buffer from a response
915 * @resp: the response descriptor containing the packet data
917 * Return a packet containing the immediate data of the given response.
919 static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
921 struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
924 __skb_put(skb, IMMED_PKT_SIZE);
925 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
931 * calc_tx_descs - calculate the number of Tx descriptors for a packet
934 * Returns the number of Tx descriptors needed for the given Ethernet
935 * packet. Ethernet packets require addition of WR and CPL headers.
937 static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
941 if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
944 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
945 if (skb_shinfo(skb)->gso_size)
947 return flits_to_desc(flits);
951 * make_sgl - populate a scatter/gather list for a packet
953 * @sgp: the SGL to populate
954 * @start: start address of skb main body data to include in the SGL
955 * @len: length of skb main body data to include in the SGL
956 * @pdev: the PCI device
958 * Generates a scatter/gather list for the buffers that make up a packet
959 * and returns the SGL size in 8-byte words. The caller must size the SGL
962 static inline unsigned int make_sgl(const struct sk_buff *skb,
963 struct sg_ent *sgp, unsigned char *start,
964 unsigned int len, struct pci_dev *pdev)
967 unsigned int i, j = 0, nfrags;
970 mapping = pci_map_single(pdev, start, len, PCI_DMA_TODEVICE);
971 sgp->len[0] = cpu_to_be32(len);
972 sgp->addr[0] = cpu_to_be64(mapping);
976 nfrags = skb_shinfo(skb)->nr_frags;
977 for (i = 0; i < nfrags; i++) {
978 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
980 mapping = pci_map_page(pdev, frag->page, frag->page_offset,
981 frag->size, PCI_DMA_TODEVICE);
982 sgp->len[j] = cpu_to_be32(frag->size);
983 sgp->addr[j] = cpu_to_be64(mapping);
990 return ((nfrags + (len != 0)) * 3) / 2 + j;
994 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
998 * Ring the doorbel if a Tx queue is asleep. There is a natural race,
999 * where the HW is going to sleep just after we checked, however,
1000 * then the interrupt handler will detect the outstanding TX packet
1001 * and ring the doorbell for us.
1003 * When GTS is disabled we unconditionally ring the doorbell.
1005 static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
1008 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1009 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1010 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1011 t3_write_reg(adap, A_SG_KDOORBELL,
1012 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1015 wmb(); /* write descriptors before telling HW */
1016 t3_write_reg(adap, A_SG_KDOORBELL,
1017 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1021 static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
1023 #if SGE_NUM_GENBITS == 2
1024 d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
1029 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
1030 * @ndesc: number of Tx descriptors spanned by the SGL
1031 * @skb: the packet corresponding to the WR
1032 * @d: first Tx descriptor to be written
1033 * @pidx: index of above descriptors
1034 * @q: the SGE Tx queue
1036 * @flits: number of flits to the start of the SGL in the first descriptor
1037 * @sgl_flits: the SGL size in flits
1038 * @gen: the Tx descriptor generation
1039 * @wr_hi: top 32 bits of WR header based on WR type (big endian)
1040 * @wr_lo: low 32 bits of WR header based on WR type (big endian)
1042 * Write a work request header and an associated SGL. If the SGL is
1043 * small enough to fit into one Tx descriptor it has already been written
1044 * and we just need to write the WR header. Otherwise we distribute the
1045 * SGL across the number of descriptors it spans.
1047 static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
1048 struct tx_desc *d, unsigned int pidx,
1049 const struct sge_txq *q,
1050 const struct sg_ent *sgl,
1051 unsigned int flits, unsigned int sgl_flits,
1052 unsigned int gen, __be32 wr_hi,
1055 struct work_request_hdr *wrp = (struct work_request_hdr *)d;
1056 struct tx_sw_desc *sd = &q->sdesc[pidx];
1059 if (need_skb_unmap()) {
1065 if (likely(ndesc == 1)) {
1067 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1068 V_WR_SGLSFLT(flits)) | wr_hi;
1070 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
1071 V_WR_GEN(gen)) | wr_lo;
1074 unsigned int ogen = gen;
1075 const u64 *fp = (const u64 *)sgl;
1076 struct work_request_hdr *wp = wrp;
1078 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1079 V_WR_SGLSFLT(flits)) | wr_hi;
1082 unsigned int avail = WR_FLITS - flits;
1084 if (avail > sgl_flits)
1086 memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
1096 if (++pidx == q->size) {
1104 wrp = (struct work_request_hdr *)d;
1105 wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
1106 V_WR_SGLSFLT(1)) | wr_hi;
1107 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
1109 V_WR_GEN(gen)) | wr_lo;
1114 wrp->wr_hi |= htonl(F_WR_EOP);
1116 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1117 wr_gen2((struct tx_desc *)wp, ogen);
1118 WARN_ON(ndesc != 0);
1123 * write_tx_pkt_wr - write a TX_PKT work request
1124 * @adap: the adapter
1125 * @skb: the packet to send
1126 * @pi: the egress interface
1127 * @pidx: index of the first Tx descriptor to write
1128 * @gen: the generation value to use
1130 * @ndesc: number of descriptors the packet will occupy
1131 * @compl: the value of the COMPL bit to use
1133 * Generate a TX_PKT work request to send the supplied packet.
1135 static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
1136 const struct port_info *pi,
1137 unsigned int pidx, unsigned int gen,
1138 struct sge_txq *q, unsigned int ndesc,
1141 unsigned int flits, sgl_flits, cntrl, tso_info;
1142 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1143 struct tx_desc *d = &q->desc[pidx];
1144 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1146 cpl->len = htonl(skb->len);
1147 cntrl = V_TXPKT_INTF(pi->port_id);
1149 if (vlan_tx_tag_present(skb) && pi->vlan_grp)
1150 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(vlan_tx_tag_get(skb));
1152 tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1155 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1158 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1159 hdr->cntrl = htonl(cntrl);
1160 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1161 CPL_ETH_II : CPL_ETH_II_VLAN;
1162 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1163 V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1164 V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1165 hdr->lso_info = htonl(tso_info);
1168 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1169 cntrl |= F_TXPKT_IPCSUM_DIS; /* SW calculates IP csum */
1170 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1171 cpl->cntrl = htonl(cntrl);
1173 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1174 q->sdesc[pidx].skb = NULL;
1176 skb_copy_from_linear_data(skb, &d->flit[2],
1179 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1181 flits = (skb->len + 7) / 8 + 2;
1182 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1183 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1184 | F_WR_SOP | F_WR_EOP | compl);
1186 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1187 V_WR_TID(q->token));
1196 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1197 sgl_flits = make_sgl(skb, sgp, skb->data, skb_headlen(skb), adap->pdev);
1199 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1200 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1201 htonl(V_WR_TID(q->token)));
1204 static inline void t3_stop_tx_queue(struct netdev_queue *txq,
1205 struct sge_qset *qs, struct sge_txq *q)
1207 netif_tx_stop_queue(txq);
1208 set_bit(TXQ_ETH, &qs->txq_stopped);
1213 * eth_xmit - add a packet to the Ethernet Tx queue
1215 * @dev: the egress net device
1217 * Add a packet to an SGE Tx queue. Runs with softirqs disabled.
1219 netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1222 unsigned int ndesc, pidx, credits, gen, compl;
1223 const struct port_info *pi = netdev_priv(dev);
1224 struct adapter *adap = pi->adapter;
1225 struct netdev_queue *txq;
1226 struct sge_qset *qs;
1230 * The chip min packet length is 9 octets but play safe and reject
1231 * anything shorter than an Ethernet header.
1233 if (unlikely(skb->len < ETH_HLEN)) {
1235 return NETDEV_TX_OK;
1238 qidx = skb_get_queue_mapping(skb);
1240 q = &qs->txq[TXQ_ETH];
1241 txq = netdev_get_tx_queue(dev, qidx);
1243 reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1245 credits = q->size - q->in_use;
1246 ndesc = calc_tx_descs(skb);
1248 if (unlikely(credits < ndesc)) {
1249 t3_stop_tx_queue(txq, qs, q);
1250 dev_err(&adap->pdev->dev,
1251 "%s: Tx ring %u full while queue awake!\n",
1252 dev->name, q->cntxt_id & 7);
1253 return NETDEV_TX_BUSY;
1257 if (unlikely(credits - ndesc < q->stop_thres)) {
1258 t3_stop_tx_queue(txq, qs, q);
1260 if (should_restart_tx(q) &&
1261 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1263 netif_tx_start_queue(txq);
1268 q->unacked += ndesc;
1269 compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1273 if (q->pidx >= q->size) {
1278 /* update port statistics */
1279 if (skb->ip_summed == CHECKSUM_COMPLETE)
1280 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1281 if (skb_shinfo(skb)->gso_size)
1282 qs->port_stats[SGE_PSTAT_TSO]++;
1283 if (vlan_tx_tag_present(skb) && pi->vlan_grp)
1284 qs->port_stats[SGE_PSTAT_VLANINS]++;
1287 * We do not use Tx completion interrupts to free DMAd Tx packets.
1288 * This is good for performance but means that we rely on new Tx
1289 * packets arriving to run the destructors of completed packets,
1290 * which open up space in their sockets' send queues. Sometimes
1291 * we do not get such new packets causing Tx to stall. A single
1292 * UDP transmitter is a good example of this situation. We have
1293 * a clean up timer that periodically reclaims completed packets
1294 * but it doesn't run often enough (nor do we want it to) to prevent
1295 * lengthy stalls. A solution to this problem is to run the
1296 * destructor early, after the packet is queued but before it's DMAd.
1297 * A cons is that we lie to socket memory accounting, but the amount
1298 * of extra memory is reasonable (limited by the number of Tx
1299 * descriptors), the packets do actually get freed quickly by new
1300 * packets almost always, and for protocols like TCP that wait for
1301 * acks to really free up the data the extra memory is even less.
1302 * On the positive side we run the destructors on the sending CPU
1303 * rather than on a potentially different completing CPU, usually a
1304 * good thing. We also run them without holding our Tx queue lock,
1305 * unlike what reclaim_completed_tx() would otherwise do.
1307 * Run the destructor before telling the DMA engine about the packet
1308 * to make sure it doesn't complete and get freed prematurely.
1310 if (likely(!skb_shared(skb)))
1313 write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl);
1314 check_ring_tx_db(adap, q);
1315 return NETDEV_TX_OK;
1319 * write_imm - write a packet into a Tx descriptor as immediate data
1320 * @d: the Tx descriptor to write
1322 * @len: the length of packet data to write as immediate data
1323 * @gen: the generation bit value to write
1325 * Writes a packet as immediate data into a Tx descriptor. The packet
1326 * contains a work request at its beginning. We must write the packet
1327 * carefully so the SGE doesn't read it accidentally before it's written
1330 static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1331 unsigned int len, unsigned int gen)
1333 struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1334 struct work_request_hdr *to = (struct work_request_hdr *)d;
1336 if (likely(!skb->data_len))
1337 memcpy(&to[1], &from[1], len - sizeof(*from));
1339 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1341 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1342 V_WR_BCNTLFLT(len & 7));
1344 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1345 V_WR_LEN((len + 7) / 8));
1351 * check_desc_avail - check descriptor availability on a send queue
1352 * @adap: the adapter
1353 * @q: the send queue
1354 * @skb: the packet needing the descriptors
1355 * @ndesc: the number of Tx descriptors needed
1356 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1358 * Checks if the requested number of Tx descriptors is available on an
1359 * SGE send queue. If the queue is already suspended or not enough
1360 * descriptors are available the packet is queued for later transmission.
1361 * Must be called with the Tx queue locked.
1363 * Returns 0 if enough descriptors are available, 1 if there aren't
1364 * enough descriptors and the packet has been queued, and 2 if the caller
1365 * needs to retry because there weren't enough descriptors at the
1366 * beginning of the call but some freed up in the mean time.
1368 static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1369 struct sk_buff *skb, unsigned int ndesc,
1372 if (unlikely(!skb_queue_empty(&q->sendq))) {
1373 addq_exit:__skb_queue_tail(&q->sendq, skb);
1376 if (unlikely(q->size - q->in_use < ndesc)) {
1377 struct sge_qset *qs = txq_to_qset(q, qid);
1379 set_bit(qid, &qs->txq_stopped);
1380 smp_mb__after_clear_bit();
1382 if (should_restart_tx(q) &&
1383 test_and_clear_bit(qid, &qs->txq_stopped))
1393 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1394 * @q: the SGE control Tx queue
1396 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1397 * that send only immediate data (presently just the control queues) and
1398 * thus do not have any sk_buffs to release.
1400 static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1402 unsigned int reclaim = q->processed - q->cleaned;
1404 q->in_use -= reclaim;
1405 q->cleaned += reclaim;
1408 static inline int immediate(const struct sk_buff *skb)
1410 return skb->len <= WR_LEN;
1414 * ctrl_xmit - send a packet through an SGE control Tx queue
1415 * @adap: the adapter
1416 * @q: the control queue
1419 * Send a packet through an SGE control Tx queue. Packets sent through
1420 * a control queue must fit entirely as immediate data in a single Tx
1421 * descriptor and have no page fragments.
1423 static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1424 struct sk_buff *skb)
1427 struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1429 if (unlikely(!immediate(skb))) {
1432 return NET_XMIT_SUCCESS;
1435 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1436 wrp->wr_lo = htonl(V_WR_TID(q->token));
1438 spin_lock(&q->lock);
1439 again:reclaim_completed_tx_imm(q);
1441 ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1442 if (unlikely(ret)) {
1444 spin_unlock(&q->lock);
1450 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1453 if (++q->pidx >= q->size) {
1457 spin_unlock(&q->lock);
1459 t3_write_reg(adap, A_SG_KDOORBELL,
1460 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1461 return NET_XMIT_SUCCESS;
1465 * restart_ctrlq - restart a suspended control queue
1466 * @qs: the queue set cotaining the control queue
1468 * Resumes transmission on a suspended Tx control queue.
1470 static void restart_ctrlq(unsigned long data)
1472 struct sk_buff *skb;
1473 struct sge_qset *qs = (struct sge_qset *)data;
1474 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1476 spin_lock(&q->lock);
1477 again:reclaim_completed_tx_imm(q);
1479 while (q->in_use < q->size &&
1480 (skb = __skb_dequeue(&q->sendq)) != NULL) {
1482 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1484 if (++q->pidx >= q->size) {
1491 if (!skb_queue_empty(&q->sendq)) {
1492 set_bit(TXQ_CTRL, &qs->txq_stopped);
1493 smp_mb__after_clear_bit();
1495 if (should_restart_tx(q) &&
1496 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1501 spin_unlock(&q->lock);
1503 t3_write_reg(qs->adap, A_SG_KDOORBELL,
1504 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1508 * Send a management message through control queue 0
1510 int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1514 ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1521 * deferred_unmap_destructor - unmap a packet when it is freed
1524 * This is the packet destructor used for Tx packets that need to remain
1525 * mapped until they are freed rather than until their Tx descriptors are
1528 static void deferred_unmap_destructor(struct sk_buff *skb)
1531 const dma_addr_t *p;
1532 const struct skb_shared_info *si;
1533 const struct deferred_unmap_info *dui;
1535 dui = (struct deferred_unmap_info *)skb->head;
1538 if (skb->tail - skb->transport_header)
1539 pci_unmap_single(dui->pdev, *p++,
1540 skb->tail - skb->transport_header,
1543 si = skb_shinfo(skb);
1544 for (i = 0; i < si->nr_frags; i++)
1545 pci_unmap_page(dui->pdev, *p++, si->frags[i].size,
1549 static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1550 const struct sg_ent *sgl, int sgl_flits)
1553 struct deferred_unmap_info *dui;
1555 dui = (struct deferred_unmap_info *)skb->head;
1557 for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1558 *p++ = be64_to_cpu(sgl->addr[0]);
1559 *p++ = be64_to_cpu(sgl->addr[1]);
1562 *p = be64_to_cpu(sgl->addr[0]);
1566 * write_ofld_wr - write an offload work request
1567 * @adap: the adapter
1568 * @skb: the packet to send
1570 * @pidx: index of the first Tx descriptor to write
1571 * @gen: the generation value to use
1572 * @ndesc: number of descriptors the packet will occupy
1574 * Write an offload work request to send the supplied packet. The packet
1575 * data already carry the work request with most fields populated.
1577 static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1578 struct sge_txq *q, unsigned int pidx,
1579 unsigned int gen, unsigned int ndesc)
1581 unsigned int sgl_flits, flits;
1582 struct work_request_hdr *from;
1583 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1584 struct tx_desc *d = &q->desc[pidx];
1586 if (immediate(skb)) {
1587 q->sdesc[pidx].skb = NULL;
1588 write_imm(d, skb, skb->len, gen);
1592 /* Only TX_DATA builds SGLs */
1594 from = (struct work_request_hdr *)skb->data;
1595 memcpy(&d->flit[1], &from[1],
1596 skb_transport_offset(skb) - sizeof(*from));
1598 flits = skb_transport_offset(skb) / 8;
1599 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1600 sgl_flits = make_sgl(skb, sgp, skb_transport_header(skb),
1601 skb->tail - skb->transport_header,
1603 if (need_skb_unmap()) {
1604 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1605 skb->destructor = deferred_unmap_destructor;
1608 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1609 gen, from->wr_hi, from->wr_lo);
1613 * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
1616 * Returns the number of Tx descriptors needed for the given offload
1617 * packet. These packets are already fully constructed.
1619 static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1621 unsigned int flits, cnt;
1623 if (skb->len <= WR_LEN)
1624 return 1; /* packet fits as immediate data */
1626 flits = skb_transport_offset(skb) / 8; /* headers */
1627 cnt = skb_shinfo(skb)->nr_frags;
1628 if (skb->tail != skb->transport_header)
1630 return flits_to_desc(flits + sgl_len(cnt));
1634 * ofld_xmit - send a packet through an offload queue
1635 * @adap: the adapter
1636 * @q: the Tx offload queue
1639 * Send an offload packet through an SGE offload queue.
1641 static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1642 struct sk_buff *skb)
1645 unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1647 spin_lock(&q->lock);
1648 again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1650 ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1651 if (unlikely(ret)) {
1653 skb->priority = ndesc; /* save for restart */
1654 spin_unlock(&q->lock);
1664 if (q->pidx >= q->size) {
1668 spin_unlock(&q->lock);
1670 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1671 check_ring_tx_db(adap, q);
1672 return NET_XMIT_SUCCESS;
1676 * restart_offloadq - restart a suspended offload queue
1677 * @qs: the queue set cotaining the offload queue
1679 * Resumes transmission on a suspended Tx offload queue.
1681 static void restart_offloadq(unsigned long data)
1683 struct sk_buff *skb;
1684 struct sge_qset *qs = (struct sge_qset *)data;
1685 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1686 const struct port_info *pi = netdev_priv(qs->netdev);
1687 struct adapter *adap = pi->adapter;
1689 spin_lock(&q->lock);
1690 again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1692 while ((skb = skb_peek(&q->sendq)) != NULL) {
1693 unsigned int gen, pidx;
1694 unsigned int ndesc = skb->priority;
1696 if (unlikely(q->size - q->in_use < ndesc)) {
1697 set_bit(TXQ_OFLD, &qs->txq_stopped);
1698 smp_mb__after_clear_bit();
1700 if (should_restart_tx(q) &&
1701 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1711 if (q->pidx >= q->size) {
1715 __skb_unlink(skb, &q->sendq);
1716 spin_unlock(&q->lock);
1718 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1719 spin_lock(&q->lock);
1721 spin_unlock(&q->lock);
1724 set_bit(TXQ_RUNNING, &q->flags);
1725 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1728 t3_write_reg(adap, A_SG_KDOORBELL,
1729 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1733 * queue_set - return the queue set a packet should use
1736 * Maps a packet to the SGE queue set it should use. The desired queue
1737 * set is carried in bits 1-3 in the packet's priority.
1739 static inline int queue_set(const struct sk_buff *skb)
1741 return skb->priority >> 1;
1745 * is_ctrl_pkt - return whether an offload packet is a control packet
1748 * Determines whether an offload packet should use an OFLD or a CTRL
1749 * Tx queue. This is indicated by bit 0 in the packet's priority.
1751 static inline int is_ctrl_pkt(const struct sk_buff *skb)
1753 return skb->priority & 1;
1757 * t3_offload_tx - send an offload packet
1758 * @tdev: the offload device to send to
1761 * Sends an offload packet. We use the packet priority to select the
1762 * appropriate Tx queue as follows: bit 0 indicates whether the packet
1763 * should be sent as regular or control, bits 1-3 select the queue set.
1765 int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1767 struct adapter *adap = tdev2adap(tdev);
1768 struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1770 if (unlikely(is_ctrl_pkt(skb)))
1771 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1773 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1777 * offload_enqueue - add an offload packet to an SGE offload receive queue
1778 * @q: the SGE response queue
1781 * Add a new offload packet to an SGE response queue's offload packet
1782 * queue. If the packet is the first on the queue it schedules the RX
1783 * softirq to process the queue.
1785 static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1787 int was_empty = skb_queue_empty(&q->rx_queue);
1789 __skb_queue_tail(&q->rx_queue, skb);
1792 struct sge_qset *qs = rspq_to_qset(q);
1794 napi_schedule(&qs->napi);
1799 * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
1800 * @tdev: the offload device that will be receiving the packets
1801 * @q: the SGE response queue that assembled the bundle
1802 * @skbs: the partial bundle
1803 * @n: the number of packets in the bundle
1805 * Delivers a (partial) bundle of Rx offload packets to an offload device.
1807 static inline void deliver_partial_bundle(struct t3cdev *tdev,
1809 struct sk_buff *skbs[], int n)
1812 q->offload_bundles++;
1813 tdev->recv(tdev, skbs, n);
1818 * ofld_poll - NAPI handler for offload packets in interrupt mode
1819 * @dev: the network device doing the polling
1820 * @budget: polling budget
1822 * The NAPI handler for offload packets when a response queue is serviced
1823 * by the hard interrupt handler, i.e., when it's operating in non-polling
1824 * mode. Creates small packet batches and sends them through the offload
1825 * receive handler. Batches need to be of modest size as we do prefetches
1826 * on the packets in each.
1828 static int ofld_poll(struct napi_struct *napi, int budget)
1830 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1831 struct sge_rspq *q = &qs->rspq;
1832 struct adapter *adapter = qs->adap;
1835 while (work_done < budget) {
1836 struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE];
1837 struct sk_buff_head queue;
1840 spin_lock_irq(&q->lock);
1841 __skb_queue_head_init(&queue);
1842 skb_queue_splice_init(&q->rx_queue, &queue);
1843 if (skb_queue_empty(&queue)) {
1844 napi_complete(napi);
1845 spin_unlock_irq(&q->lock);
1848 spin_unlock_irq(&q->lock);
1851 skb_queue_walk_safe(&queue, skb, tmp) {
1852 if (work_done >= budget)
1856 __skb_unlink(skb, &queue);
1857 prefetch(skb->data);
1858 skbs[ngathered] = skb;
1859 if (++ngathered == RX_BUNDLE_SIZE) {
1860 q->offload_bundles++;
1861 adapter->tdev.recv(&adapter->tdev, skbs,
1866 if (!skb_queue_empty(&queue)) {
1867 /* splice remaining packets back onto Rx queue */
1868 spin_lock_irq(&q->lock);
1869 skb_queue_splice(&queue, &q->rx_queue);
1870 spin_unlock_irq(&q->lock);
1872 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1879 * rx_offload - process a received offload packet
1880 * @tdev: the offload device receiving the packet
1881 * @rq: the response queue that received the packet
1883 * @rx_gather: a gather list of packets if we are building a bundle
1884 * @gather_idx: index of the next available slot in the bundle
1886 * Process an ingress offload pakcet and add it to the offload ingress
1887 * queue. Returns the index of the next available slot in the bundle.
1889 static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1890 struct sk_buff *skb, struct sk_buff *rx_gather[],
1891 unsigned int gather_idx)
1893 skb_reset_mac_header(skb);
1894 skb_reset_network_header(skb);
1895 skb_reset_transport_header(skb);
1898 rx_gather[gather_idx++] = skb;
1899 if (gather_idx == RX_BUNDLE_SIZE) {
1900 tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1902 rq->offload_bundles++;
1905 offload_enqueue(rq, skb);
1911 * restart_tx - check whether to restart suspended Tx queues
1912 * @qs: the queue set to resume
1914 * Restarts suspended Tx queues of an SGE queue set if they have enough
1915 * free resources to resume operation.
1917 static void restart_tx(struct sge_qset *qs)
1919 if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1920 should_restart_tx(&qs->txq[TXQ_ETH]) &&
1921 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1922 qs->txq[TXQ_ETH].restarts++;
1923 if (netif_running(qs->netdev))
1924 netif_tx_wake_queue(qs->tx_q);
1927 if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
1928 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1929 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1930 qs->txq[TXQ_OFLD].restarts++;
1931 tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk);
1933 if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
1934 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
1935 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
1936 qs->txq[TXQ_CTRL].restarts++;
1937 tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk);
1942 * cxgb3_arp_process - process an ARP request probing a private IP address
1943 * @adapter: the adapter
1944 * @skb: the skbuff containing the ARP request
1946 * Check if the ARP request is probing the private IP address
1947 * dedicated to iSCSI, generate an ARP reply if so.
1949 static void cxgb3_arp_process(struct port_info *pi, struct sk_buff *skb)
1951 struct net_device *dev = skb->dev;
1953 unsigned char *arp_ptr;
1960 skb_reset_network_header(skb);
1963 if (arp->ar_op != htons(ARPOP_REQUEST))
1966 arp_ptr = (unsigned char *)(arp + 1);
1968 arp_ptr += dev->addr_len;
1969 memcpy(&sip, arp_ptr, sizeof(sip));
1970 arp_ptr += sizeof(sip);
1971 arp_ptr += dev->addr_len;
1972 memcpy(&tip, arp_ptr, sizeof(tip));
1974 if (tip != pi->iscsi_ipv4addr)
1977 arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
1978 pi->iscsic.mac_addr, sha);
1982 static inline int is_arp(struct sk_buff *skb)
1984 return skb->protocol == htons(ETH_P_ARP);
1987 static void cxgb3_process_iscsi_prov_pack(struct port_info *pi,
1988 struct sk_buff *skb)
1991 cxgb3_arp_process(pi, skb);
1995 if (pi->iscsic.recv)
1996 pi->iscsic.recv(pi, skb);
2001 * rx_eth - process an ingress ethernet packet
2002 * @adap: the adapter
2003 * @rq: the response queue that received the packet
2005 * @pad: amount of padding at the start of the buffer
2007 * Process an ingress ethernet pakcet and deliver it to the stack.
2008 * The padding is 2 if the packet was delivered in an Rx buffer and 0
2009 * if it was immediate data in a response.
2011 static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
2012 struct sk_buff *skb, int pad, int lro)
2014 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
2015 struct sge_qset *qs = rspq_to_qset(rq);
2016 struct port_info *pi;
2018 skb_pull(skb, sizeof(*p) + pad);
2019 skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
2020 pi = netdev_priv(skb->dev);
2021 if ((pi->rx_offload & T3_RX_CSUM) && p->csum_valid &&
2022 p->csum == htons(0xffff) && !p->fragment) {
2023 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2024 skb->ip_summed = CHECKSUM_UNNECESSARY;
2026 skb->ip_summed = CHECKSUM_NONE;
2027 skb_record_rx_queue(skb, qs - &adap->sge.qs[0]);
2029 if (unlikely(p->vlan_valid)) {
2030 struct vlan_group *grp = pi->vlan_grp;
2032 qs->port_stats[SGE_PSTAT_VLANEX]++;
2035 vlan_gro_receive(&qs->napi, grp,
2036 ntohs(p->vlan), skb);
2038 if (unlikely(pi->iscsic.flags)) {
2039 unsigned short vtag = ntohs(p->vlan) &
2041 skb->dev = vlan_group_get_device(grp,
2043 cxgb3_process_iscsi_prov_pack(pi, skb);
2045 __vlan_hwaccel_rx(skb, grp, ntohs(p->vlan),
2049 dev_kfree_skb_any(skb);
2050 } else if (rq->polling) {
2052 napi_gro_receive(&qs->napi, skb);
2054 if (unlikely(pi->iscsic.flags))
2055 cxgb3_process_iscsi_prov_pack(pi, skb);
2056 netif_receive_skb(skb);
2062 static inline int is_eth_tcp(u32 rss)
2064 return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE;
2068 * lro_add_page - add a page chunk to an LRO session
2069 * @adap: the adapter
2070 * @qs: the associated queue set
2071 * @fl: the free list containing the page chunk to add
2072 * @len: packet length
2073 * @complete: Indicates the last fragment of a frame
2075 * Add a received packet contained in a page chunk to an existing LRO
2078 static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
2079 struct sge_fl *fl, int len, int complete)
2081 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2082 struct port_info *pi = netdev_priv(qs->netdev);
2083 struct sk_buff *skb = NULL;
2084 struct cpl_rx_pkt *cpl;
2085 struct skb_frag_struct *rx_frag;
2090 skb = napi_get_frags(&qs->napi);
2096 pci_dma_sync_single_for_cpu(adap->pdev,
2097 pci_unmap_addr(sd, dma_addr),
2098 fl->buf_size - SGE_PG_RSVD,
2099 PCI_DMA_FROMDEVICE);
2101 (*sd->pg_chunk.p_cnt)--;
2102 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
2103 pci_unmap_page(adap->pdev,
2104 sd->pg_chunk.mapping,
2106 PCI_DMA_FROMDEVICE);
2109 put_page(sd->pg_chunk.page);
2115 rx_frag = skb_shinfo(skb)->frags;
2116 nr_frags = skb_shinfo(skb)->nr_frags;
2119 offset = 2 + sizeof(struct cpl_rx_pkt);
2120 cpl = qs->lro_va = sd->pg_chunk.va + 2;
2122 if ((pi->rx_offload & T3_RX_CSUM) &&
2123 cpl->csum_valid && cpl->csum == htons(0xffff)) {
2124 skb->ip_summed = CHECKSUM_UNNECESSARY;
2125 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2127 skb->ip_summed = CHECKSUM_NONE;
2133 rx_frag += nr_frags;
2134 rx_frag->page = sd->pg_chunk.page;
2135 rx_frag->page_offset = sd->pg_chunk.offset + offset;
2136 rx_frag->size = len;
2139 skb->data_len += len;
2140 skb->truesize += len;
2141 skb_shinfo(skb)->nr_frags++;
2146 skb_record_rx_queue(skb, qs - &adap->sge.qs[0]);
2148 if (unlikely(cpl->vlan_valid)) {
2149 struct vlan_group *grp = pi->vlan_grp;
2151 if (likely(grp != NULL)) {
2152 vlan_gro_frags(&qs->napi, grp, ntohs(cpl->vlan));
2156 napi_gro_frags(&qs->napi);
2160 * handle_rsp_cntrl_info - handles control information in a response
2161 * @qs: the queue set corresponding to the response
2162 * @flags: the response control flags
2164 * Handles the control information of an SGE response, such as GTS
2165 * indications and completion credits for the queue set's Tx queues.
2166 * HW coalesces credits, we don't do any extra SW coalescing.
2168 static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
2170 unsigned int credits;
2173 if (flags & F_RSPD_TXQ0_GTS)
2174 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2177 credits = G_RSPD_TXQ0_CR(flags);
2179 qs->txq[TXQ_ETH].processed += credits;
2181 credits = G_RSPD_TXQ2_CR(flags);
2183 qs->txq[TXQ_CTRL].processed += credits;
2186 if (flags & F_RSPD_TXQ1_GTS)
2187 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2189 credits = G_RSPD_TXQ1_CR(flags);
2191 qs->txq[TXQ_OFLD].processed += credits;
2195 * check_ring_db - check if we need to ring any doorbells
2196 * @adapter: the adapter
2197 * @qs: the queue set whose Tx queues are to be examined
2198 * @sleeping: indicates which Tx queue sent GTS
2200 * Checks if some of a queue set's Tx queues need to ring their doorbells
2201 * to resume transmission after idling while they still have unprocessed
2204 static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
2205 unsigned int sleeping)
2207 if (sleeping & F_RSPD_TXQ0_GTS) {
2208 struct sge_txq *txq = &qs->txq[TXQ_ETH];
2210 if (txq->cleaned + txq->in_use != txq->processed &&
2211 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2212 set_bit(TXQ_RUNNING, &txq->flags);
2213 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2214 V_EGRCNTX(txq->cntxt_id));
2218 if (sleeping & F_RSPD_TXQ1_GTS) {
2219 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
2221 if (txq->cleaned + txq->in_use != txq->processed &&
2222 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2223 set_bit(TXQ_RUNNING, &txq->flags);
2224 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2225 V_EGRCNTX(txq->cntxt_id));
2231 * is_new_response - check if a response is newly written
2232 * @r: the response descriptor
2233 * @q: the response queue
2235 * Returns true if a response descriptor contains a yet unprocessed
2238 static inline int is_new_response(const struct rsp_desc *r,
2239 const struct sge_rspq *q)
2241 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2244 static inline void clear_rspq_bufstate(struct sge_rspq * const q)
2247 q->rx_recycle_buf = 0;
2250 #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2251 #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2252 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2253 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2254 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2256 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
2257 #define NOMEM_INTR_DELAY 2500
2260 * process_responses - process responses from an SGE response queue
2261 * @adap: the adapter
2262 * @qs: the queue set to which the response queue belongs
2263 * @budget: how many responses can be processed in this round
2265 * Process responses from an SGE response queue up to the supplied budget.
2266 * Responses include received packets as well as credits and other events
2267 * for the queues that belong to the response queue's queue set.
2268 * A negative budget is effectively unlimited.
2270 * Additionally choose the interrupt holdoff time for the next interrupt
2271 * on this queue. If the system is under memory shortage use a fairly
2272 * long delay to help recovery.
2274 static int process_responses(struct adapter *adap, struct sge_qset *qs,
2277 struct sge_rspq *q = &qs->rspq;
2278 struct rsp_desc *r = &q->desc[q->cidx];
2279 int budget_left = budget;
2280 unsigned int sleeping = 0;
2281 struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
2284 q->next_holdoff = q->holdoff_tmr;
2286 while (likely(budget_left && is_new_response(r, q))) {
2287 int packet_complete, eth, ethpad = 2, lro = qs->lro_enabled;
2288 struct sk_buff *skb = NULL;
2289 u32 len, flags = ntohl(r->flags);
2290 __be32 rss_hi = *(const __be32 *)r,
2291 rss_lo = r->rss_hdr.rss_hash_val;
2293 eth = r->rss_hdr.opcode == CPL_RX_PKT;
2295 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
2296 skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
2300 memcpy(__skb_put(skb, AN_PKT_SIZE), r, AN_PKT_SIZE);
2301 skb->data[0] = CPL_ASYNC_NOTIF;
2302 rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
2304 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2305 skb = get_imm_packet(r);
2306 if (unlikely(!skb)) {
2308 q->next_holdoff = NOMEM_INTR_DELAY;
2310 /* consume one credit since we tried */
2316 } else if ((len = ntohl(r->len_cq)) != 0) {
2319 lro &= eth && is_eth_tcp(rss_hi);
2321 fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2322 if (fl->use_pages) {
2323 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
2326 #if L1_CACHE_BYTES < 128
2327 prefetch(addr + L1_CACHE_BYTES);
2329 __refill_fl(adap, fl);
2331 lro_add_page(adap, qs, fl,
2333 flags & F_RSPD_EOP);
2337 skb = get_packet_pg(adap, fl, q,
2340 SGE_RX_DROP_THRES : 0);
2343 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2344 eth ? SGE_RX_DROP_THRES : 0);
2345 if (unlikely(!skb)) {
2349 } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2352 if (++fl->cidx == fl->size)
2357 if (flags & RSPD_CTRL_MASK) {
2358 sleeping |= flags & RSPD_GTS_MASK;
2359 handle_rsp_cntrl_info(qs, flags);
2363 if (unlikely(++q->cidx == q->size)) {
2370 if (++q->credits >= (q->size / 4)) {
2371 refill_rspq(adap, q, q->credits);
2375 packet_complete = flags &
2376 (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID |
2377 F_RSPD_ASYNC_NOTIF);
2379 if (skb != NULL && packet_complete) {
2381 rx_eth(adap, q, skb, ethpad, lro);
2384 /* Preserve the RSS info in csum & priority */
2386 skb->priority = rss_lo;
2387 ngathered = rx_offload(&adap->tdev, q, skb,
2392 if (flags & F_RSPD_EOP)
2393 clear_rspq_bufstate(q);
2398 deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2401 check_ring_db(adap, qs, sleeping);
2403 smp_mb(); /* commit Tx queue .processed updates */
2404 if (unlikely(qs->txq_stopped != 0))
2407 budget -= budget_left;
2411 static inline int is_pure_response(const struct rsp_desc *r)
2413 __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2415 return (n | r->len_cq) == 0;
2419 * napi_rx_handler - the NAPI handler for Rx processing
2420 * @napi: the napi instance
2421 * @budget: how many packets we can process in this round
2423 * Handler for new data events when using NAPI.
2425 static int napi_rx_handler(struct napi_struct *napi, int budget)
2427 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2428 struct adapter *adap = qs->adap;
2429 int work_done = process_responses(adap, qs, budget);
2431 if (likely(work_done < budget)) {
2432 napi_complete(napi);
2435 * Because we don't atomically flush the following
2436 * write it is possible that in very rare cases it can
2437 * reach the device in a way that races with a new
2438 * response being written plus an error interrupt
2439 * causing the NAPI interrupt handler below to return
2440 * unhandled status to the OS. To protect against
2441 * this would require flushing the write and doing
2442 * both the write and the flush with interrupts off.
2443 * Way too expensive and unjustifiable given the
2444 * rarity of the race.
2446 * The race cannot happen at all with MSI-X.
2448 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2449 V_NEWTIMER(qs->rspq.next_holdoff) |
2450 V_NEWINDEX(qs->rspq.cidx));
2456 * Returns true if the device is already scheduled for polling.
2458 static inline int napi_is_scheduled(struct napi_struct *napi)
2460 return test_bit(NAPI_STATE_SCHED, &napi->state);
2464 * process_pure_responses - process pure responses from a response queue
2465 * @adap: the adapter
2466 * @qs: the queue set owning the response queue
2467 * @r: the first pure response to process
2469 * A simpler version of process_responses() that handles only pure (i.e.,
2470 * non data-carrying) responses. Such respones are too light-weight to
2471 * justify calling a softirq under NAPI, so we handle them specially in
2472 * the interrupt handler. The function is called with a pointer to a
2473 * response, which the caller must ensure is a valid pure response.
2475 * Returns 1 if it encounters a valid data-carrying response, 0 otherwise.
2477 static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2480 struct sge_rspq *q = &qs->rspq;
2481 unsigned int sleeping = 0;
2484 u32 flags = ntohl(r->flags);
2487 if (unlikely(++q->cidx == q->size)) {
2494 if (flags & RSPD_CTRL_MASK) {
2495 sleeping |= flags & RSPD_GTS_MASK;
2496 handle_rsp_cntrl_info(qs, flags);
2500 if (++q->credits >= (q->size / 4)) {
2501 refill_rspq(adap, q, q->credits);
2504 } while (is_new_response(r, q) && is_pure_response(r));
2507 check_ring_db(adap, qs, sleeping);
2509 smp_mb(); /* commit Tx queue .processed updates */
2510 if (unlikely(qs->txq_stopped != 0))
2513 return is_new_response(r, q);
2517 * handle_responses - decide what to do with new responses in NAPI mode
2518 * @adap: the adapter
2519 * @q: the response queue
2521 * This is used by the NAPI interrupt handlers to decide what to do with
2522 * new SGE responses. If there are no new responses it returns -1. If
2523 * there are new responses and they are pure (i.e., non-data carrying)
2524 * it handles them straight in hard interrupt context as they are very
2525 * cheap and don't deliver any packets. Finally, if there are any data
2526 * signaling responses it schedules the NAPI handler. Returns 1 if it
2527 * schedules NAPI, 0 if all new responses were pure.
2529 * The caller must ascertain NAPI is not already running.
2531 static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2533 struct sge_qset *qs = rspq_to_qset(q);
2534 struct rsp_desc *r = &q->desc[q->cidx];
2536 if (!is_new_response(r, q))
2538 if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2539 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2540 V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2543 napi_schedule(&qs->napi);
2548 * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case
2549 * (i.e., response queue serviced in hard interrupt).
2551 irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2553 struct sge_qset *qs = cookie;
2554 struct adapter *adap = qs->adap;
2555 struct sge_rspq *q = &qs->rspq;
2557 spin_lock(&q->lock);
2558 if (process_responses(adap, qs, -1) == 0)
2559 q->unhandled_irqs++;
2560 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2561 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2562 spin_unlock(&q->lock);
2567 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
2568 * (i.e., response queue serviced by NAPI polling).
2570 static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2572 struct sge_qset *qs = cookie;
2573 struct sge_rspq *q = &qs->rspq;
2575 spin_lock(&q->lock);
2577 if (handle_responses(qs->adap, q) < 0)
2578 q->unhandled_irqs++;
2579 spin_unlock(&q->lock);
2584 * The non-NAPI MSI interrupt handler. This needs to handle data events from
2585 * SGE response queues as well as error and other async events as they all use
2586 * the same MSI vector. We use one SGE response queue per port in this mode
2587 * and protect all response queues with queue 0's lock.
2589 static irqreturn_t t3_intr_msi(int irq, void *cookie)
2591 int new_packets = 0;
2592 struct adapter *adap = cookie;
2593 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2595 spin_lock(&q->lock);
2597 if (process_responses(adap, &adap->sge.qs[0], -1)) {
2598 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2599 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2603 if (adap->params.nports == 2 &&
2604 process_responses(adap, &adap->sge.qs[1], -1)) {
2605 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2607 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2608 V_NEWTIMER(q1->next_holdoff) |
2609 V_NEWINDEX(q1->cidx));
2613 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2614 q->unhandled_irqs++;
2616 spin_unlock(&q->lock);
2620 static int rspq_check_napi(struct sge_qset *qs)
2622 struct sge_rspq *q = &qs->rspq;
2624 if (!napi_is_scheduled(&qs->napi) &&
2625 is_new_response(&q->desc[q->cidx], q)) {
2626 napi_schedule(&qs->napi);
2633 * The MSI interrupt handler for the NAPI case (i.e., response queues serviced
2634 * by NAPI polling). Handles data events from SGE response queues as well as
2635 * error and other async events as they all use the same MSI vector. We use
2636 * one SGE response queue per port in this mode and protect all response
2637 * queues with queue 0's lock.
2639 static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2642 struct adapter *adap = cookie;
2643 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2645 spin_lock(&q->lock);
2647 new_packets = rspq_check_napi(&adap->sge.qs[0]);
2648 if (adap->params.nports == 2)
2649 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2650 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2651 q->unhandled_irqs++;
2653 spin_unlock(&q->lock);
2658 * A helper function that processes responses and issues GTS.
2660 static inline int process_responses_gts(struct adapter *adap,
2661 struct sge_rspq *rq)
2665 work = process_responses(adap, rspq_to_qset(rq), -1);
2666 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2667 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2672 * The legacy INTx interrupt handler. This needs to handle data events from
2673 * SGE response queues as well as error and other async events as they all use
2674 * the same interrupt pin. We use one SGE response queue per port in this mode
2675 * and protect all response queues with queue 0's lock.
2677 static irqreturn_t t3_intr(int irq, void *cookie)
2679 int work_done, w0, w1;
2680 struct adapter *adap = cookie;
2681 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2682 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2684 spin_lock(&q0->lock);
2686 w0 = is_new_response(&q0->desc[q0->cidx], q0);
2687 w1 = adap->params.nports == 2 &&
2688 is_new_response(&q1->desc[q1->cidx], q1);
2690 if (likely(w0 | w1)) {
2691 t3_write_reg(adap, A_PL_CLI, 0);
2692 t3_read_reg(adap, A_PL_CLI); /* flush */
2695 process_responses_gts(adap, q0);
2698 process_responses_gts(adap, q1);
2700 work_done = w0 | w1;
2702 work_done = t3_slow_intr_handler(adap);
2704 spin_unlock(&q0->lock);
2705 return IRQ_RETVAL(work_done != 0);
2709 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
2710 * Handles data events from SGE response queues as well as error and other
2711 * async events as they all use the same interrupt pin. We use one SGE
2712 * response queue per port in this mode and protect all response queues with
2715 static irqreturn_t t3b_intr(int irq, void *cookie)
2718 struct adapter *adap = cookie;
2719 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2721 t3_write_reg(adap, A_PL_CLI, 0);
2722 map = t3_read_reg(adap, A_SG_DATA_INTR);
2724 if (unlikely(!map)) /* shared interrupt, most likely */
2727 spin_lock(&q0->lock);
2729 if (unlikely(map & F_ERRINTR))
2730 t3_slow_intr_handler(adap);
2732 if (likely(map & 1))
2733 process_responses_gts(adap, q0);
2736 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2738 spin_unlock(&q0->lock);
2743 * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards.
2744 * Handles data events from SGE response queues as well as error and other
2745 * async events as they all use the same interrupt pin. We use one SGE
2746 * response queue per port in this mode and protect all response queues with
2749 static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2752 struct adapter *adap = cookie;
2753 struct sge_qset *qs0 = &adap->sge.qs[0];
2754 struct sge_rspq *q0 = &qs0->rspq;
2756 t3_write_reg(adap, A_PL_CLI, 0);
2757 map = t3_read_reg(adap, A_SG_DATA_INTR);
2759 if (unlikely(!map)) /* shared interrupt, most likely */
2762 spin_lock(&q0->lock);
2764 if (unlikely(map & F_ERRINTR))
2765 t3_slow_intr_handler(adap);
2767 if (likely(map & 1))
2768 napi_schedule(&qs0->napi);
2771 napi_schedule(&adap->sge.qs[1].napi);
2773 spin_unlock(&q0->lock);
2778 * t3_intr_handler - select the top-level interrupt handler
2779 * @adap: the adapter
2780 * @polling: whether using NAPI to service response queues
2782 * Selects the top-level interrupt handler based on the type of interrupts
2783 * (MSI-X, MSI, or legacy) and whether NAPI will be used to service the
2786 irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2788 if (adap->flags & USING_MSIX)
2789 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2790 if (adap->flags & USING_MSI)
2791 return polling ? t3_intr_msi_napi : t3_intr_msi;
2792 if (adap->params.rev > 0)
2793 return polling ? t3b_intr_napi : t3b_intr;
2797 #define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2798 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2799 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2800 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2802 #define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2803 #define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2807 * t3_sge_err_intr_handler - SGE async event interrupt handler
2808 * @adapter: the adapter
2810 * Interrupt handler for SGE asynchronous (non-data) events.
2812 void t3_sge_err_intr_handler(struct adapter *adapter)
2814 unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
2817 if (status & SGE_PARERR)
2818 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
2819 status & SGE_PARERR);
2820 if (status & SGE_FRAMINGERR)
2821 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
2822 status & SGE_FRAMINGERR);
2824 if (status & F_RSPQCREDITOVERFOW)
2825 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2827 if (status & F_RSPQDISABLED) {
2828 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2831 "packet delivered to disabled response queue "
2832 "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2835 if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2836 CH_ALERT(adapter, "SGE dropped %s priority doorbell\n",
2837 status & F_HIPIODRBDROPERR ? "high" : "lo");
2839 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2840 if (status & SGE_FATALERR)
2841 t3_fatal_err(adapter);
2845 * sge_timer_tx - perform periodic maintenance of an SGE qset
2846 * @data: the SGE queue set to maintain
2848 * Runs periodically from a timer to perform maintenance of an SGE queue
2849 * set. It performs two tasks:
2851 * Cleans up any completed Tx descriptors that may still be pending.
2852 * Normal descriptor cleanup happens when new packets are added to a Tx
2853 * queue so this timer is relatively infrequent and does any cleanup only
2854 * if the Tx queue has not seen any new packets in a while. We make a
2855 * best effort attempt to reclaim descriptors, in that we don't wait
2856 * around if we cannot get a queue's lock (which most likely is because
2857 * someone else is queueing new packets and so will also handle the clean
2858 * up). Since control queues use immediate data exclusively we don't
2859 * bother cleaning them up here.
2862 static void sge_timer_tx(unsigned long data)
2864 struct sge_qset *qs = (struct sge_qset *)data;
2865 struct port_info *pi = netdev_priv(qs->netdev);
2866 struct adapter *adap = pi->adapter;
2867 unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
2868 unsigned long next_period;
2870 if (__netif_tx_trylock(qs->tx_q)) {
2871 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
2872 TX_RECLAIM_TIMER_CHUNK);
2873 __netif_tx_unlock(qs->tx_q);
2876 if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2877 tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
2878 TX_RECLAIM_TIMER_CHUNK);
2879 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2882 next_period = TX_RECLAIM_PERIOD >>
2883 (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
2884 TX_RECLAIM_TIMER_CHUNK);
2885 mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
2889 * sge_timer_rx - perform periodic maintenance of an SGE qset
2890 * @data: the SGE queue set to maintain
2892 * a) Replenishes Rx queues that have run out due to memory shortage.
2893 * Normally new Rx buffers are added when existing ones are consumed but
2894 * when out of memory a queue can become empty. We try to add only a few
2895 * buffers here, the queue will be replenished fully as these new buffers
2896 * are used up if memory shortage has subsided.
2898 * b) Return coalesced response queue credits in case a response queue is
2902 static void sge_timer_rx(unsigned long data)
2905 struct sge_qset *qs = (struct sge_qset *)data;
2906 struct port_info *pi = netdev_priv(qs->netdev);
2907 struct adapter *adap = pi->adapter;
2910 lock = adap->params.rev > 0 ?
2911 &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
2913 if (!spin_trylock_irq(lock))
2916 if (napi_is_scheduled(&qs->napi))
2919 if (adap->params.rev < 4) {
2920 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2922 if (status & (1 << qs->rspq.cntxt_id)) {
2924 if (qs->rspq.credits) {
2926 refill_rspq(adap, &qs->rspq, 1);
2927 qs->rspq.restarted++;
2928 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2929 1 << qs->rspq.cntxt_id);
2934 if (qs->fl[0].credits < qs->fl[0].size)
2935 __refill_fl(adap, &qs->fl[0]);
2936 if (qs->fl[1].credits < qs->fl[1].size)
2937 __refill_fl(adap, &qs->fl[1]);
2940 spin_unlock_irq(lock);
2942 mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
2946 * t3_update_qset_coalesce - update coalescing settings for a queue set
2947 * @qs: the SGE queue set
2948 * @p: new queue set parameters
2950 * Update the coalescing settings for an SGE queue set. Nothing is done
2951 * if the queue set is not initialized yet.
2953 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
2955 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */
2956 qs->rspq.polling = p->polling;
2957 qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
2961 * t3_sge_alloc_qset - initialize an SGE queue set
2962 * @adapter: the adapter
2963 * @id: the queue set id
2964 * @nports: how many Ethernet ports will be using this queue set
2965 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2966 * @p: configuration parameters for this queue set
2967 * @ntxq: number of Tx queues for the queue set
2968 * @netdev: net device associated with this queue set
2969 * @netdevq: net device TX queue associated with this queue set
2971 * Allocate resources and initialize an SGE queue set. A queue set
2972 * comprises a response queue, two Rx free-buffer queues, and up to 3
2973 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2974 * queue, offload queue, and control queue.
2976 int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
2977 int irq_vec_idx, const struct qset_params *p,
2978 int ntxq, struct net_device *dev,
2979 struct netdev_queue *netdevq)
2981 int i, avail, ret = -ENOMEM;
2982 struct sge_qset *q = &adapter->sge.qs[id];
2984 init_qset_cntxt(q, id);
2985 setup_timer(&q->tx_reclaim_timer, sge_timer_tx, (unsigned long)q);
2986 setup_timer(&q->rx_reclaim_timer, sge_timer_rx, (unsigned long)q);
2988 q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
2989 sizeof(struct rx_desc),
2990 sizeof(struct rx_sw_desc),
2991 &q->fl[0].phys_addr, &q->fl[0].sdesc);
2995 q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
2996 sizeof(struct rx_desc),
2997 sizeof(struct rx_sw_desc),
2998 &q->fl[1].phys_addr, &q->fl[1].sdesc);
3002 q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
3003 sizeof(struct rsp_desc), 0,
3004 &q->rspq.phys_addr, NULL);
3008 for (i = 0; i < ntxq; ++i) {
3010 * The control queue always uses immediate data so does not
3011 * need to keep track of any sk_buffs.
3013 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
3015 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
3016 sizeof(struct tx_desc), sz,
3017 &q->txq[i].phys_addr,
3019 if (!q->txq[i].desc)
3023 q->txq[i].size = p->txq_size[i];
3024 spin_lock_init(&q->txq[i].lock);
3025 skb_queue_head_init(&q->txq[i].sendq);
3028 tasklet_init(&q->txq[TXQ_OFLD].qresume_tsk, restart_offloadq,
3030 tasklet_init(&q->txq[TXQ_CTRL].qresume_tsk, restart_ctrlq,
3033 q->fl[0].gen = q->fl[1].gen = 1;
3034 q->fl[0].size = p->fl_size;
3035 q->fl[1].size = p->jumbo_size;
3038 q->rspq.size = p->rspq_size;
3039 spin_lock_init(&q->rspq.lock);
3040 skb_queue_head_init(&q->rspq.rx_queue);
3042 q->txq[TXQ_ETH].stop_thres = nports *
3043 flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
3045 #if FL0_PG_CHUNK_SIZE > 0
3046 q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
3048 q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
3050 #if FL1_PG_CHUNK_SIZE > 0
3051 q->fl[1].buf_size = FL1_PG_CHUNK_SIZE;
3053 q->fl[1].buf_size = is_offload(adapter) ?
3054 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
3055 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
3058 q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
3059 q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
3060 q->fl[0].order = FL0_PG_ORDER;
3061 q->fl[1].order = FL1_PG_ORDER;
3062 q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
3063 q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
3065 spin_lock_irq(&adapter->sge.reg_lock);
3067 /* FL threshold comparison uses < */
3068 ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
3069 q->rspq.phys_addr, q->rspq.size,
3070 q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
3074 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
3075 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
3076 q->fl[i].phys_addr, q->fl[i].size,
3077 q->fl[i].buf_size - SGE_PG_RSVD,
3078 p->cong_thres, 1, 0);
3083 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
3084 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
3085 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
3091 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
3092 USE_GTS, SGE_CNTXT_OFLD, id,
3093 q->txq[TXQ_OFLD].phys_addr,
3094 q->txq[TXQ_OFLD].size, 0, 1, 0);
3100 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
3102 q->txq[TXQ_CTRL].phys_addr,
3103 q->txq[TXQ_CTRL].size,
3104 q->txq[TXQ_CTRL].token, 1, 0);
3109 spin_unlock_irq(&adapter->sge.reg_lock);
3114 t3_update_qset_coalesce(q, p);
3116 avail = refill_fl(adapter, &q->fl[0], q->fl[0].size,
3117 GFP_KERNEL | __GFP_COMP);
3119 CH_ALERT(adapter, "free list queue 0 initialization failed\n");
3122 if (avail < q->fl[0].size)
3123 CH_WARN(adapter, "free list queue 0 enabled with %d credits\n",
3126 avail = refill_fl(adapter, &q->fl[1], q->fl[1].size,
3127 GFP_KERNEL | __GFP_COMP);
3128 if (avail < q->fl[1].size)
3129 CH_WARN(adapter, "free list queue 1 enabled with %d credits\n",
3131 refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
3133 t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
3134 V_NEWTIMER(q->rspq.holdoff_tmr));
3139 spin_unlock_irq(&adapter->sge.reg_lock);
3141 t3_free_qset(adapter, q);
3146 * t3_start_sge_timers - start SGE timer call backs
3147 * @adap: the adapter
3149 * Starts each SGE queue set's timer call back
3151 void t3_start_sge_timers(struct adapter *adap)
3155 for (i = 0; i < SGE_QSETS; ++i) {
3156 struct sge_qset *q = &adap->sge.qs[i];
3158 if (q->tx_reclaim_timer.function)
3159 mod_timer(&q->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
3161 if (q->rx_reclaim_timer.function)
3162 mod_timer(&q->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
3167 * t3_stop_sge_timers - stop SGE timer call backs
3168 * @adap: the adapter
3170 * Stops each SGE queue set's timer call back
3172 void t3_stop_sge_timers(struct adapter *adap)
3176 for (i = 0; i < SGE_QSETS; ++i) {
3177 struct sge_qset *q = &adap->sge.qs[i];
3179 if (q->tx_reclaim_timer.function)
3180 del_timer_sync(&q->tx_reclaim_timer);
3181 if (q->rx_reclaim_timer.function)
3182 del_timer_sync(&q->rx_reclaim_timer);
3187 * t3_free_sge_resources - free SGE resources
3188 * @adap: the adapter
3190 * Frees resources used by the SGE queue sets.
3192 void t3_free_sge_resources(struct adapter *adap)
3196 for (i = 0; i < SGE_QSETS; ++i)
3197 t3_free_qset(adap, &adap->sge.qs[i]);
3201 * t3_sge_start - enable SGE
3202 * @adap: the adapter
3204 * Enables the SGE for DMAs. This is the last step in starting packet
3207 void t3_sge_start(struct adapter *adap)
3209 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
3213 * t3_sge_stop - disable SGE operation
3214 * @adap: the adapter
3216 * Disables the DMA engine. This can be called in emeregencies (e.g.,
3217 * from error interrupts) or from normal process context. In the latter
3218 * case it also disables any pending queue restart tasklets. Note that
3219 * if it is called in interrupt context it cannot disable the restart
3220 * tasklets as it cannot wait, however the tasklets will have no effect
3221 * since the doorbells are disabled and the driver will call this again
3222 * later from process context, at which time the tasklets will be stopped
3223 * if they are still running.
3225 void t3_sge_stop(struct adapter *adap)
3227 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
3228 if (!in_interrupt()) {
3231 for (i = 0; i < SGE_QSETS; ++i) {
3232 struct sge_qset *qs = &adap->sge.qs[i];
3234 tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk);
3235 tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk);
3241 * t3_sge_init - initialize SGE
3242 * @adap: the adapter
3243 * @p: the SGE parameters
3245 * Performs SGE initialization needed every time after a chip reset.
3246 * We do not initialize any of the queue sets here, instead the driver
3247 * top-level must request those individually. We also do not enable DMA
3248 * here, that should be done after the queues have been set up.
3250 void t3_sge_init(struct adapter *adap, struct sge_params *p)
3252 unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
3254 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
3255 F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
3256 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
3257 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
3258 #if SGE_NUM_GENBITS == 1
3259 ctrl |= F_EGRGENCTRL;
3261 if (adap->params.rev > 0) {
3262 if (!(adap->flags & (USING_MSIX | USING_MSI)))
3263 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
3265 t3_write_reg(adap, A_SG_CONTROL, ctrl);
3266 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
3267 V_LORCQDRBTHRSH(512));
3268 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
3269 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
3270 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
3271 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
3272 adap->params.rev < T3_REV_C ? 1000 : 500);
3273 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
3274 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
3275 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
3276 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
3277 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
3281 * t3_sge_prep - one-time SGE initialization
3282 * @adap: the associated adapter
3283 * @p: SGE parameters
3285 * Performs one-time initialization of SGE SW state. Includes determining
3286 * defaults for the assorted SGE parameters, which admins can change until
3287 * they are used to initialize the SGE.
3289 void t3_sge_prep(struct adapter *adap, struct sge_params *p)
3293 p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
3294 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3296 for (i = 0; i < SGE_QSETS; ++i) {
3297 struct qset_params *q = p->qset + i;
3299 q->polling = adap->params.rev > 0;
3300 q->coalesce_usecs = 5;
3301 q->rspq_size = 1024;
3303 q->jumbo_size = 512;
3304 q->txq_size[TXQ_ETH] = 1024;
3305 q->txq_size[TXQ_OFLD] = 1024;
3306 q->txq_size[TXQ_CTRL] = 256;
3310 spin_lock_init(&adap->sge.reg_lock);
3314 * t3_get_desc - dump an SGE descriptor for debugging purposes
3315 * @qs: the queue set
3316 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
3317 * @idx: the descriptor index in the queue
3318 * @data: where to dump the descriptor contents
3320 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
3321 * size of the descriptor.
3323 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
3324 unsigned char *data)
3330 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size)
3332 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc));
3333 return sizeof(struct tx_desc);
3337 if (!qs->rspq.desc || idx >= qs->rspq.size)
3339 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc));
3340 return sizeof(struct rsp_desc);
3344 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size)
3346 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc));
3347 return sizeof(struct rx_desc);