1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
16 #include <linux/tcp.h>
18 #include <linux/ipv6.h>
19 #include <linux/crc32.h>
20 #include <linux/if_vlan.h>
21 #include <linux/jiffies.h>
22 #include <linux/phy.h>
24 #include <net/ip6_checksum.h>
26 #include "emac-sgmii.h"
29 #define SINGLE_PAUSE_MODE 0x10000000
30 #define DEBUG_MODE 0x08000000
31 #define BROAD_EN 0x04000000
32 #define MULTI_ALL 0x02000000
33 #define RX_CHKSUM_EN 0x01000000
34 #define HUGE 0x00800000
35 #define SPEED(x) (((x) & 0x3) << 20)
36 #define SPEED_MASK SPEED(0x3)
37 #define SIMR 0x00080000
38 #define TPAUSE 0x00010000
39 #define PROM_MODE 0x00008000
40 #define VLAN_STRIP 0x00004000
41 #define PRLEN_BMSK 0x00003c00
43 #define HUGEN 0x00000200
44 #define FLCHK 0x00000100
45 #define PCRCE 0x00000080
46 #define CRCE 0x00000040
47 #define FULLD 0x00000020
48 #define MAC_LP_EN 0x00000010
49 #define RXFC 0x00000008
50 #define TXFC 0x00000004
51 #define RXEN 0x00000002
52 #define TXEN 0x00000001
54 /* EMAC_DESC_CTRL_3 */
55 #define RFD_RING_SIZE_BMSK 0xfff
57 /* EMAC_DESC_CTRL_4 */
58 #define RX_BUFFER_SIZE_BMSK 0xffff
60 /* EMAC_DESC_CTRL_6 */
61 #define RRD_RING_SIZE_BMSK 0xfff
63 /* EMAC_DESC_CTRL_9 */
64 #define TPD_RING_SIZE_BMSK 0xffff
67 #define NUM_TXF_BURST_PREF_BMSK 0xffff0000
68 #define NUM_TXF_BURST_PREF_SHFT 16
69 #define LS_8023_SP 0x80
73 #define NUM_TPD_BURST_PREF_BMSK 0xf
74 #define NUM_TPD_BURST_PREF_SHFT 0
77 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff
80 #define TXF_HWM_BMSK 0xfff0000
81 #define TXF_LWM_BMSK 0xfff
84 #define RXQ_EN BIT(31)
85 #define CUT_THRU_EN BIT(30)
86 #define RSS_HASH_EN BIT(29)
87 #define NUM_RFD_BURST_PREF_BMSK 0x3f00000
88 #define NUM_RFD_BURST_PREF_SHFT 20
89 #define IDT_TABLE_SIZE_BMSK 0x1ff00
90 #define IDT_TABLE_SIZE_SHFT 8
94 #define JUMBO_1KAH_BMSK 0xf000
95 #define JUMBO_1KAH_SHFT 12
96 #define RFD_PREF_LOW_TH 0x10
97 #define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0
98 #define RFD_PREF_LOW_THRESHOLD_SHFT 6
99 #define RFD_PREF_UP_TH 0x10
100 #define RFD_PREF_UP_THRESHOLD_BMSK 0x3f
101 #define RFD_PREF_UP_THRESHOLD_SHFT 0
103 /* EMAC_RXQ_CTRL_2 */
104 #define RXF_DOF_THRESFHOLD 0x1a0
105 #define RXF_DOF_THRESHOLD_BMSK 0xfff0000
106 #define RXF_DOF_THRESHOLD_SHFT 16
107 #define RXF_UOF_THRESFHOLD 0xbe
108 #define RXF_UOF_THRESHOLD_BMSK 0xfff
109 #define RXF_UOF_THRESHOLD_SHFT 0
111 /* EMAC_RXQ_CTRL_3 */
112 #define RXD_TIMER_BMSK 0xffff0000
113 #define RXD_THRESHOLD_BMSK 0xfff
114 #define RXD_THRESHOLD_SHFT 0
117 #define DMAW_DLY_CNT_BMSK 0xf0000
118 #define DMAW_DLY_CNT_SHFT 16
119 #define DMAR_DLY_CNT_BMSK 0xf800
120 #define DMAR_DLY_CNT_SHFT 11
121 #define DMAR_REQ_PRI 0x400
122 #define REGWRBLEN_BMSK 0x380
123 #define REGWRBLEN_SHFT 7
124 #define REGRDBLEN_BMSK 0x70
125 #define REGRDBLEN_SHFT 4
126 #define OUT_ORDER_MODE 0x4
127 #define ENH_ORDER_MODE 0x2
128 #define IN_ORDER_MODE 0x1
130 /* EMAC_MAILBOX_13 */
131 #define RFD3_PROC_IDX_BMSK 0xfff0000
132 #define RFD3_PROC_IDX_SHFT 16
133 #define RFD3_PROD_IDX_BMSK 0xfff
134 #define RFD3_PROD_IDX_SHFT 0
137 #define NTPD_CONS_IDX_BMSK 0xffff0000
138 #define NTPD_CONS_IDX_SHFT 16
141 #define RFD0_CONS_IDX_BMSK 0xfff
142 #define RFD0_CONS_IDX_SHFT 0
144 /* EMAC_MAILBOX_11 */
145 #define H3TPD_PROD_IDX_BMSK 0xffff0000
146 #define H3TPD_PROD_IDX_SHFT 16
148 /* EMAC_AXI_MAST_CTRL */
149 #define DATA_BYTE_SWAP 0x8
150 #define MAX_BOUND 0x2
151 #define MAX_BTYPE 0x1
153 /* EMAC_MAILBOX_12 */
154 #define H3TPD_CONS_IDX_BMSK 0xffff0000
155 #define H3TPD_CONS_IDX_SHFT 16
158 #define H2TPD_PROD_IDX_BMSK 0xffff
159 #define H2TPD_PROD_IDX_SHFT 0
161 /* EMAC_MAILBOX_10 */
162 #define H1TPD_CONS_IDX_BMSK 0xffff0000
163 #define H1TPD_CONS_IDX_SHFT 16
164 #define H2TPD_CONS_IDX_BMSK 0xffff
165 #define H2TPD_CONS_IDX_SHFT 0
167 /* EMAC_ATHR_HEADER_CTRL */
168 #define HEADER_CNT_EN 0x2
169 #define HEADER_ENABLE 0x1
172 #define RFD0_PROC_IDX_BMSK 0xfff0000
173 #define RFD0_PROC_IDX_SHFT 16
174 #define RFD0_PROD_IDX_BMSK 0xfff
175 #define RFD0_PROD_IDX_SHFT 0
178 #define RFD1_PROC_IDX_BMSK 0xfff0000
179 #define RFD1_PROC_IDX_SHFT 16
180 #define RFD1_PROD_IDX_BMSK 0xfff
181 #define RFD1_PROD_IDX_SHFT 0
184 #define RX_UNCPL_INT_EN 0x1
187 #define RFD2_CONS_IDX_BMSK 0xfff0000
188 #define RFD2_CONS_IDX_SHFT 16
189 #define RFD1_CONS_IDX_BMSK 0xfff
190 #define RFD1_CONS_IDX_SHFT 0
193 #define RFD3_CONS_IDX_BMSK 0xfff
194 #define RFD3_CONS_IDX_SHFT 0
196 /* EMAC_MAILBOX_15 */
197 #define NTPD_PROD_IDX_BMSK 0xffff
198 #define NTPD_PROD_IDX_SHFT 0
200 /* EMAC_MAILBOX_16 */
201 #define H1TPD_PROD_IDX_BMSK 0xffff
202 #define H1TPD_PROD_IDX_SHFT 0
204 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20
205 #define RXQ0_RSS_HSTYP_IPV6_EN 0x10
206 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8
207 #define RXQ0_RSS_HSTYP_IPV4_EN 0x4
209 /* EMAC_EMAC_WRAPPER_TX_TS_INX */
210 #define EMAC_WRAPPER_TX_TS_EMPTY BIT(31)
211 #define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff
215 unsigned long jiffies;
218 #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)
219 #define EMAC_RSS_IDT_SIZE 256
220 #define JUMBO_1KAH 0x4
222 #define EMAC_TPD_LAST_FRAGMENT 0x80000000
223 #define EMAC_TPD_TSTAMP_SAVE 0x80000000
225 /* EMAC Errors in emac_rrd.word[3] */
226 #define EMAC_RRD_L4F BIT(14)
227 #define EMAC_RRD_IPF BIT(15)
228 #define EMAC_RRD_CRC BIT(21)
229 #define EMAC_RRD_FAE BIT(22)
230 #define EMAC_RRD_TRN BIT(23)
231 #define EMAC_RRD_RNT BIT(24)
232 #define EMAC_RRD_INC BIT(25)
233 #define EMAC_RRD_FOV BIT(29)
234 #define EMAC_RRD_LEN BIT(30)
236 /* Error bits that will result in a received frame being discarded */
237 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
238 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
239 EMAC_RRD_FOV | EMAC_RRD_LEN)
240 #define EMAC_RRD_STATS_DW_IDX 3
242 #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
243 #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
244 #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
246 #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))
247 #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))
249 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8
251 #define ISR_RX_PKT (\
257 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
259 u32 crc32, bit, reg, mta;
261 /* Calculate the CRC of the MAC address */
262 crc32 = ether_crc(ETH_ALEN, addr);
264 /* The HASH Table is an array of 2 32-bit registers. It is
265 * treated like an array of 64 bits (BitArray[hash_value]).
266 * Use the upper 6 bits of the above CRC as the hash value.
268 reg = (crc32 >> 31) & 0x1;
269 bit = (crc32 >> 26) & 0x1F;
271 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
273 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
276 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
278 writel(0, adpt->base + EMAC_HASH_TAB_REG0);
279 writel(0, adpt->base + EMAC_HASH_TAB_REG1);
282 /* definitions for RSS */
283 #define EMAC_RSS_KEY(_i, _type) \
284 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
285 #define EMAC_RSS_TBL(_i, _type) \
286 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
288 /* Config MAC modes */
289 void emac_mac_mode_config(struct emac_adapter *adpt)
291 struct net_device *netdev = adpt->netdev;
294 mac = readl(adpt->base + EMAC_MAC_CTRL);
295 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
297 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
300 if (netdev->flags & IFF_PROMISC)
303 if (netdev->flags & IFF_ALLMULTI)
306 writel(mac, adpt->base + EMAC_MAC_CTRL);
309 /* Config descriptor rings */
310 static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
312 static const unsigned short tpd_q_offset[] = {
313 EMAC_DESC_CTRL_8, EMAC_H1TPD_BASE_ADDR_LO,
314 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO};
315 static const unsigned short rfd_q_offset[] = {
316 EMAC_DESC_CTRL_2, EMAC_DESC_CTRL_10,
317 EMAC_DESC_CTRL_12, EMAC_DESC_CTRL_13};
318 static const unsigned short rrd_q_offset[] = {
319 EMAC_DESC_CTRL_5, EMAC_DESC_CTRL_14,
320 EMAC_DESC_CTRL_15, EMAC_DESC_CTRL_16};
322 /* TPD (Transmit Packet Descriptor) */
323 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
324 adpt->base + EMAC_DESC_CTRL_1);
326 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
327 adpt->base + tpd_q_offset[0]);
329 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
330 adpt->base + EMAC_DESC_CTRL_9);
332 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
333 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
334 adpt->base + EMAC_DESC_CTRL_0);
336 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
337 adpt->base + rfd_q_offset[0]);
338 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
339 adpt->base + rrd_q_offset[0]);
341 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
342 adpt->base + EMAC_DESC_CTRL_3);
343 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
344 adpt->base + EMAC_DESC_CTRL_6);
346 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
347 adpt->base + EMAC_DESC_CTRL_4);
349 writel(0, adpt->base + EMAC_DESC_CTRL_11);
351 /* Load all of the base addresses above and ensure that triggering HW to
352 * read ring pointers is flushed
354 writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
357 /* Config transmit parameters */
358 static void emac_mac_tx_config(struct emac_adapter *adpt)
362 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
363 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
365 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
366 NUM_TPD_BURST_PREF_BMSK;
368 val |= TXQ_MODE | LS_8023_SP;
369 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
370 NUM_TXF_BURST_PREF_BMSK;
372 writel(val, adpt->base + EMAC_TXQ_CTRL_0);
373 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
374 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
377 /* Config receive parameters */
378 static void emac_mac_rx_config(struct emac_adapter *adpt)
382 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
383 NUM_RFD_BURST_PREF_BMSK;
384 val |= (SP_IPV6 | CUT_THRU_EN);
386 writel(val, adpt->base + EMAC_RXQ_CTRL_0);
388 val = readl(adpt->base + EMAC_RXQ_CTRL_1);
389 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
390 RFD_PREF_UP_THRESHOLD_BMSK);
391 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
392 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
393 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT);
394 writel(val, adpt->base + EMAC_RXQ_CTRL_1);
396 val = readl(adpt->base + EMAC_RXQ_CTRL_2);
397 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
398 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) |
399 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
400 writel(val, adpt->base + EMAC_RXQ_CTRL_2);
402 val = readl(adpt->base + EMAC_RXQ_CTRL_3);
403 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
404 val |= RXD_TH << RXD_THRESHOLD_SHFT;
405 writel(val, adpt->base + EMAC_RXQ_CTRL_3);
409 static void emac_mac_dma_config(struct emac_adapter *adpt)
411 u32 dma_ctrl = DMAR_REQ_PRI;
413 switch (adpt->dma_order) {
414 case emac_dma_ord_in:
415 dma_ctrl |= IN_ORDER_MODE;
417 case emac_dma_ord_enh:
418 dma_ctrl |= ENH_ORDER_MODE;
420 case emac_dma_ord_out:
421 dma_ctrl |= OUT_ORDER_MODE;
427 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
429 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
431 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
433 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
436 /* config DMA and ensure that configuration is flushed to HW */
437 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
440 /* set MAC address */
441 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
445 /* for example: 00-A0-C6-11-22-33
446 * 0<-->C6112233, 1<-->00A0.
450 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
451 (((u32)addr[4]) << 8) | (((u32)addr[5]));
452 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
454 /* hight 32bit word */
455 sta = (((u32)addr[0]) << 8) | (u32)addr[1];
456 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
459 static void emac_mac_config(struct emac_adapter *adpt)
461 struct net_device *netdev = adpt->netdev;
462 unsigned int max_frame;
465 emac_set_mac_address(adpt, netdev->dev_addr);
467 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
468 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
469 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
471 emac_mac_dma_rings_config(adpt);
473 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
474 adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
476 emac_mac_tx_config(adpt);
477 emac_mac_rx_config(adpt);
478 emac_mac_dma_config(adpt);
480 val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
481 val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
483 writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
484 writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
485 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
488 void emac_mac_reset(struct emac_adapter *adpt)
492 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
493 usleep_range(100, 150); /* reset may take up to 100usec */
495 /* interrupt clear-on-read */
496 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
499 static void emac_mac_start(struct emac_adapter *adpt)
501 struct phy_device *phydev = adpt->phydev;
504 /* enable tx queue */
505 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
507 /* enable rx queue */
508 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
510 /* enable mac control */
511 mac = readl(adpt->base + EMAC_MAC_CTRL);
512 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
514 mac |= TXEN | RXEN; /* enable RX/TX */
516 /* Configure MAC flow control. If set to automatic, then match
517 * whatever the PHY does. Otherwise, enable or disable it, depending
518 * on what the user configured via ethtool.
520 mac &= ~(RXFC | TXFC);
522 if (adpt->automatic) {
523 /* If it's set to automatic, then update our local values */
524 adpt->rx_flow_control = phydev->pause;
525 adpt->tx_flow_control = phydev->pause != phydev->asym_pause;
527 mac |= adpt->rx_flow_control ? RXFC : 0;
528 mac |= adpt->tx_flow_control ? TXFC : 0;
530 /* setup link speed */
532 if (phydev->speed == SPEED_1000) {
540 if (phydev->duplex == DUPLEX_FULL)
545 /* other parameters */
546 mac |= (CRCE | PCRCE);
547 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
550 mac &= ~RX_CHKSUM_EN;
551 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
552 DEBUG_MODE | SINGLE_PAUSE_MODE);
554 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
556 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
558 /* enable interrupt read clear, low power sleep mode and
562 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
563 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
564 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
566 emac_mac_mode_config(adpt);
568 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
569 (HEADER_ENABLE | HEADER_CNT_EN), 0);
572 void emac_mac_stop(struct emac_adapter *adpt)
574 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
575 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
576 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
577 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
580 /* Free all descriptors of given transmit queue */
581 static void emac_tx_q_descs_free(struct emac_adapter *adpt)
583 struct emac_tx_queue *tx_q = &adpt->tx_q;
587 /* ring already cleared, nothing to do */
588 if (!tx_q->tpd.tpbuff)
591 for (i = 0; i < tx_q->tpd.count; i++) {
592 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
594 if (tpbuf->dma_addr) {
595 dma_unmap_single(adpt->netdev->dev.parent,
596 tpbuf->dma_addr, tpbuf->length,
601 dev_kfree_skb_any(tpbuf->skb);
606 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
607 memset(tx_q->tpd.tpbuff, 0, size);
609 /* clear the descriptor ring */
610 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
612 tx_q->tpd.consume_idx = 0;
613 tx_q->tpd.produce_idx = 0;
616 /* Free all descriptors of given receive queue */
617 static void emac_rx_q_free_descs(struct emac_adapter *adpt)
619 struct device *dev = adpt->netdev->dev.parent;
620 struct emac_rx_queue *rx_q = &adpt->rx_q;
624 /* ring already cleared, nothing to do */
625 if (!rx_q->rfd.rfbuff)
628 for (i = 0; i < rx_q->rfd.count; i++) {
629 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
631 if (rfbuf->dma_addr) {
632 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
637 dev_kfree_skb(rfbuf->skb);
642 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
643 memset(rx_q->rfd.rfbuff, 0, size);
645 /* clear the descriptor rings */
646 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
647 rx_q->rrd.produce_idx = 0;
648 rx_q->rrd.consume_idx = 0;
650 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
651 rx_q->rfd.produce_idx = 0;
652 rx_q->rfd.consume_idx = 0;
655 /* Free all buffers associated with given transmit queue */
656 static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
658 struct emac_tx_queue *tx_q = &adpt->tx_q;
660 emac_tx_q_descs_free(adpt);
662 kfree(tx_q->tpd.tpbuff);
663 tx_q->tpd.tpbuff = NULL;
664 tx_q->tpd.v_addr = NULL;
665 tx_q->tpd.dma_addr = 0;
669 /* Allocate TX descriptor ring for the given transmit queue */
670 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
671 struct emac_tx_queue *tx_q)
673 struct emac_ring_header *ring_header = &adpt->ring_header;
676 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
677 tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
678 if (!tx_q->tpd.tpbuff)
681 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
682 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
683 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
684 ring_header->used += ALIGN(tx_q->tpd.size, 8);
685 tx_q->tpd.produce_idx = 0;
686 tx_q->tpd.consume_idx = 0;
691 /* Free all buffers associated with given transmit queue */
692 static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
694 struct emac_rx_queue *rx_q = &adpt->rx_q;
696 emac_rx_q_free_descs(adpt);
698 kfree(rx_q->rfd.rfbuff);
699 rx_q->rfd.rfbuff = NULL;
701 rx_q->rfd.v_addr = NULL;
702 rx_q->rfd.dma_addr = 0;
705 rx_q->rrd.v_addr = NULL;
706 rx_q->rrd.dma_addr = 0;
710 /* Allocate RX descriptor rings for the given receive queue */
711 static int emac_rx_descs_alloc(struct emac_adapter *adpt)
713 struct emac_ring_header *ring_header = &adpt->ring_header;
714 struct emac_rx_queue *rx_q = &adpt->rx_q;
717 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
718 rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
719 if (!rx_q->rfd.rfbuff)
722 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
723 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
725 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
726 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used;
727 ring_header->used += ALIGN(rx_q->rrd.size, 8);
729 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
730 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used;
731 ring_header->used += ALIGN(rx_q->rfd.size, 8);
733 rx_q->rrd.produce_idx = 0;
734 rx_q->rrd.consume_idx = 0;
736 rx_q->rfd.produce_idx = 0;
737 rx_q->rfd.consume_idx = 0;
742 /* Allocate all TX and RX descriptor rings */
743 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
745 struct emac_ring_header *ring_header = &adpt->ring_header;
746 struct device *dev = adpt->netdev->dev.parent;
747 unsigned int num_tx_descs = adpt->tx_desc_cnt;
748 unsigned int num_rx_descs = adpt->rx_desc_cnt;
751 adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
753 adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
754 adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
756 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
757 * hence the additional padding bytes are allocated.
759 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
760 num_rx_descs * (adpt->rfd_size * 4) +
761 num_rx_descs * (adpt->rrd_size * 4) +
762 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
764 ring_header->used = 0;
765 ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
766 &ring_header->dma_addr,
768 if (!ring_header->v_addr)
771 ring_header->used = ALIGN(ring_header->dma_addr, 8) -
772 ring_header->dma_addr;
774 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
776 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
780 ret = emac_rx_descs_alloc(adpt);
782 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
789 emac_tx_q_bufs_free(adpt);
791 dma_free_coherent(dev, ring_header->size,
792 ring_header->v_addr, ring_header->dma_addr);
794 ring_header->v_addr = NULL;
795 ring_header->dma_addr = 0;
796 ring_header->size = 0;
797 ring_header->used = 0;
802 /* Free all TX and RX descriptor rings */
803 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
805 struct emac_ring_header *ring_header = &adpt->ring_header;
806 struct device *dev = adpt->netdev->dev.parent;
808 emac_tx_q_bufs_free(adpt);
809 emac_rx_q_bufs_free(adpt);
811 dma_free_coherent(dev, ring_header->size,
812 ring_header->v_addr, ring_header->dma_addr);
814 ring_header->v_addr = NULL;
815 ring_header->dma_addr = 0;
816 ring_header->size = 0;
817 ring_header->used = 0;
820 /* Initialize descriptor rings */
821 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
825 adpt->tx_q.tpd.produce_idx = 0;
826 adpt->tx_q.tpd.consume_idx = 0;
827 for (i = 0; i < adpt->tx_q.tpd.count; i++)
828 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
830 adpt->rx_q.rrd.produce_idx = 0;
831 adpt->rx_q.rrd.consume_idx = 0;
832 adpt->rx_q.rfd.produce_idx = 0;
833 adpt->rx_q.rfd.consume_idx = 0;
834 for (i = 0; i < adpt->rx_q.rfd.count; i++)
835 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
838 /* Produce new receive free descriptor */
839 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
840 struct emac_rx_queue *rx_q,
843 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
845 *(hw_rfd++) = lower_32_bits(addr);
846 *hw_rfd = upper_32_bits(addr);
848 if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
849 rx_q->rfd.produce_idx = 0;
852 /* Fill up receive queue's RFD with preallocated receive buffers */
853 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
854 struct emac_rx_queue *rx_q)
856 struct emac_buffer *curr_rxbuf;
857 struct emac_buffer *next_rxbuf;
858 unsigned int count = 0;
859 u32 next_produce_idx;
861 next_produce_idx = rx_q->rfd.produce_idx + 1;
862 if (next_produce_idx == rx_q->rfd.count)
863 next_produce_idx = 0;
865 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
866 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
868 /* this always has a blank rx_buffer*/
869 while (!next_rxbuf->dma_addr) {
873 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
877 curr_rxbuf->dma_addr =
878 dma_map_single(adpt->netdev->dev.parent, skb->data,
879 curr_rxbuf->length, DMA_FROM_DEVICE);
880 ret = dma_mapping_error(adpt->netdev->dev.parent,
881 curr_rxbuf->dma_addr);
886 curr_rxbuf->skb = skb;
887 curr_rxbuf->length = adpt->rxbuf_size;
889 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
890 next_produce_idx = rx_q->rfd.produce_idx + 1;
891 if (next_produce_idx == rx_q->rfd.count)
892 next_produce_idx = 0;
894 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
895 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
900 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
902 emac_reg_update32(adpt->base + rx_q->produce_reg,
903 rx_q->produce_mask, prod_idx);
907 static void emac_adjust_link(struct net_device *netdev)
909 struct emac_adapter *adpt = netdev_priv(netdev);
910 struct emac_sgmii *sgmii = &adpt->phy;
911 struct phy_device *phydev = netdev->phydev;
914 emac_mac_start(adpt);
915 sgmii->link_up(adpt);
917 sgmii->link_down(adpt);
921 phy_print_status(phydev);
924 /* Bringup the interface/HW */
925 int emac_mac_up(struct emac_adapter *adpt)
927 struct net_device *netdev = adpt->netdev;
930 emac_mac_rx_tx_ring_reset_all(adpt);
931 emac_mac_config(adpt);
932 emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
934 adpt->phydev->irq = PHY_IGNORE_INTERRUPT;
935 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
936 PHY_INTERFACE_MODE_SGMII);
938 netdev_err(adpt->netdev, "could not connect phy\n");
942 phy_attached_print(adpt->phydev, NULL);
945 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
946 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
948 phy_start(adpt->phydev);
950 napi_enable(&adpt->rx_q.napi);
951 netif_start_queue(netdev);
956 /* Bring down the interface/HW */
957 void emac_mac_down(struct emac_adapter *adpt)
959 struct net_device *netdev = adpt->netdev;
961 netif_stop_queue(netdev);
962 napi_disable(&adpt->rx_q.napi);
964 phy_stop(adpt->phydev);
966 /* Interrupts must be disabled before the PHY is disconnected, to
967 * avoid a race condition where adjust_link is null when we get
970 writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
971 writel(0, adpt->base + EMAC_INT_MASK);
972 synchronize_irq(adpt->irq.irq);
974 phy_disconnect(adpt->phydev);
976 emac_mac_reset(adpt);
978 emac_tx_q_descs_free(adpt);
979 netdev_reset_queue(adpt->netdev);
980 emac_rx_q_free_descs(adpt);
983 /* Consume next received packet descriptor */
984 static bool emac_rx_process_rrd(struct emac_adapter *adpt,
985 struct emac_rx_queue *rx_q,
986 struct emac_rrd *rrd)
988 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
990 rrd->word[3] = *(hw_rrd + 3);
998 rrd->word[0] = *(hw_rrd++);
999 rrd->word[1] = *(hw_rrd++);
1000 rrd->word[2] = *(hw_rrd++);
1002 if (unlikely(RRD_NOR(rrd) != 1)) {
1003 netdev_err(adpt->netdev,
1004 "error: multi-RFD not support yet! nor:%lu\n",
1008 /* mark rrd as processed */
1009 RRD_UPDT_SET(rrd, 0);
1010 *hw_rrd = rrd->word[3];
1012 if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1013 rx_q->rrd.consume_idx = 0;
1018 /* Produce new transmit descriptor */
1019 static void emac_tx_tpd_create(struct emac_adapter *adpt,
1020 struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1024 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1025 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1027 if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1028 tx_q->tpd.produce_idx = 0;
1030 *(hw_tpd++) = tpd->word[0];
1031 *(hw_tpd++) = tpd->word[1];
1032 *(hw_tpd++) = tpd->word[2];
1033 *hw_tpd = tpd->word[3];
1036 /* Mark the last transmit descriptor as such (for the transmit packet) */
1037 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1038 struct emac_tx_queue *tx_q)
1041 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1044 tmp_tpd = *(hw_tpd + 1);
1045 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1046 *(hw_tpd + 1) = tmp_tpd;
1049 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1051 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1052 u32 consume_idx = RRD_SI(rrd);
1055 for (i = 0; i < RRD_NOR(rrd); i++) {
1056 rfbuf[consume_idx].skb = NULL;
1057 if (++consume_idx == rx_q->rfd.count)
1061 rx_q->rfd.consume_idx = consume_idx;
1062 rx_q->rfd.process_idx = consume_idx;
1065 /* Push the received skb to upper layers */
1066 static void emac_receive_skb(struct emac_rx_queue *rx_q,
1067 struct sk_buff *skb,
1068 u16 vlan_tag, bool vlan_flag)
1073 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1074 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1077 napi_gro_receive(&rx_q->napi, skb);
1080 /* Process receive event */
1081 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1082 int *num_pkts, int max_pkts)
1084 u32 proc_idx, hw_consume_idx, num_consume_pkts;
1085 struct net_device *netdev = adpt->netdev;
1086 struct emac_buffer *rfbuf;
1087 unsigned int count = 0;
1088 struct emac_rrd rrd;
1089 struct sk_buff *skb;
1092 reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1094 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1095 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1096 (hw_consume_idx - rx_q->rrd.consume_idx) :
1097 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1100 if (!num_consume_pkts)
1103 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1106 if (likely(RRD_NOR(&rrd) == 1)) {
1108 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1109 dma_unmap_single(adpt->netdev->dev.parent,
1110 rfbuf->dma_addr, rfbuf->length,
1112 rfbuf->dma_addr = 0;
1115 netdev_err(adpt->netdev,
1116 "error: multi-RFD not support yet!\n");
1119 emac_rx_rfd_clean(rx_q, &rrd);
1123 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1124 * with DF set are marked as error), drop packets based on the
1125 * error mask rather than the summary bit (ignoring L4F errors)
1127 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1128 netif_dbg(adpt, rx_status, adpt->netdev,
1129 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1130 rrd.word[0], rrd.word[1],
1131 rrd.word[2], rrd.word[3]);
1137 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1139 skb->protocol = eth_type_trans(skb, skb->dev);
1140 if (netdev->features & NETIF_F_RXCSUM)
1141 skb->ip_summed = RRD_L4F(&rrd) ?
1142 CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1144 skb_checksum_none_assert(skb);
1146 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1147 (bool)RRD_CVTAG(&rrd));
1150 } while (*num_pkts < max_pkts);
1153 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1155 emac_reg_update32(adpt->base + rx_q->process_reg,
1156 rx_q->process_mask, proc_idx);
1157 emac_mac_rx_descs_refill(adpt, rx_q);
1161 /* get the number of free transmit descriptors */
1162 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1164 u32 produce_idx = tx_q->tpd.produce_idx;
1165 u32 consume_idx = tx_q->tpd.consume_idx;
1167 return (consume_idx > produce_idx) ?
1168 (consume_idx - produce_idx - 1) :
1169 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1172 /* Process transmit event */
1173 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1175 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1176 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1177 struct emac_buffer *tpbuf;
1179 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1181 while (tx_q->tpd.consume_idx != hw_consume_idx) {
1182 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1183 if (tpbuf->dma_addr) {
1184 dma_unmap_single(adpt->netdev->dev.parent,
1185 tpbuf->dma_addr, tpbuf->length,
1187 tpbuf->dma_addr = 0;
1192 bytes_compl += tpbuf->skb->len;
1193 dev_kfree_skb_irq(tpbuf->skb);
1197 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1198 tx_q->tpd.consume_idx = 0;
1201 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1203 if (netif_queue_stopped(adpt->netdev))
1204 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1205 netif_wake_queue(adpt->netdev);
1208 /* Initialize all queue data structures */
1209 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1210 struct emac_adapter *adpt)
1212 adpt->rx_q.netdev = adpt->netdev;
1214 adpt->rx_q.produce_reg = EMAC_MAILBOX_0;
1215 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1216 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1218 adpt->rx_q.process_reg = EMAC_MAILBOX_0;
1219 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1220 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1222 adpt->rx_q.consume_reg = EMAC_MAILBOX_3;
1223 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1224 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1226 adpt->rx_q.irq = &adpt->irq;
1227 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT;
1229 adpt->tx_q.produce_reg = EMAC_MAILBOX_15;
1230 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1231 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1233 adpt->tx_q.consume_reg = EMAC_MAILBOX_2;
1234 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1235 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1238 /* Fill up transmit descriptors with TSO and Checksum offload information */
1239 static int emac_tso_csum(struct emac_adapter *adpt,
1240 struct emac_tx_queue *tx_q,
1241 struct sk_buff *skb,
1242 struct emac_tpd *tpd)
1244 unsigned int hdr_len;
1247 if (skb_is_gso(skb)) {
1248 if (skb_header_cloned(skb)) {
1249 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1254 if (skb->protocol == htons(ETH_P_IP)) {
1255 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1256 + ntohs(ip_hdr(skb)->tot_len);
1257 if (skb->len > pkt_len)
1258 pskb_trim(skb, pkt_len);
1261 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1262 if (unlikely(skb->len == hdr_len)) {
1263 /* we only need to do csum */
1264 netif_warn(adpt, tx_err, adpt->netdev,
1265 "tso not needed for packet with 0 data\n");
1269 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1270 ip_hdr(skb)->check = 0;
1271 tcp_hdr(skb)->check =
1272 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1275 TPD_IPV4_SET(tpd, 1);
1278 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1279 /* ipv6 tso need an extra tpd */
1280 struct emac_tpd extra_tpd;
1282 memset(tpd, 0, sizeof(*tpd));
1283 memset(&extra_tpd, 0, sizeof(extra_tpd));
1285 ipv6_hdr(skb)->payload_len = 0;
1286 tcp_hdr(skb)->check =
1287 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1288 &ipv6_hdr(skb)->daddr,
1290 TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1291 TPD_LSO_SET(&extra_tpd, 1);
1292 TPD_LSOV_SET(&extra_tpd, 1);
1293 emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1294 TPD_LSOV_SET(tpd, 1);
1297 TPD_LSO_SET(tpd, 1);
1298 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1299 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1304 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1305 unsigned int css, cso;
1307 cso = skb_transport_offset(skb);
1308 if (unlikely(cso & 0x1)) {
1309 netdev_err(adpt->netdev,
1310 "error: payload offset should be even\n");
1313 css = cso + skb->csum_offset;
1315 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1316 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1317 TPD_CSX_SET(tpd, 1);
1323 /* Fill up transmit descriptors */
1324 static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1325 struct emac_tx_queue *tx_q, struct sk_buff *skb,
1326 struct emac_tpd *tpd)
1328 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1329 unsigned int first = tx_q->tpd.produce_idx;
1330 unsigned int len = skb_headlen(skb);
1331 struct emac_buffer *tpbuf = NULL;
1332 unsigned int mapped_len = 0;
1337 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1339 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1341 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1342 tpbuf->length = mapped_len;
1343 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1344 skb->data, tpbuf->length,
1346 ret = dma_mapping_error(adpt->netdev->dev.parent,
1351 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1352 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1353 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1354 emac_tx_tpd_create(adpt, tx_q, tpd);
1358 if (mapped_len < len) {
1359 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1360 tpbuf->length = len - mapped_len;
1361 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1362 skb->data + mapped_len,
1363 tpbuf->length, DMA_TO_DEVICE);
1364 ret = dma_mapping_error(adpt->netdev->dev.parent,
1369 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1370 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1371 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1372 emac_tx_tpd_create(adpt, tx_q, tpd);
1376 for (i = 0; i < nr_frags; i++) {
1377 struct skb_frag_struct *frag;
1379 frag = &skb_shinfo(skb)->frags[i];
1381 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1382 tpbuf->length = frag->size;
1383 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1384 frag->page.p, frag->page_offset,
1385 tpbuf->length, DMA_TO_DEVICE);
1386 ret = dma_mapping_error(adpt->netdev->dev.parent,
1391 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1392 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1393 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1394 emac_tx_tpd_create(adpt, tx_q, tpd);
1400 emac_tx_tpd_mark_last(adpt, tx_q);
1402 /* The last buffer info contain the skb address,
1403 * so it will be freed after unmap
1410 /* One of the memory mappings failed, so undo everything */
1411 tx_q->tpd.produce_idx = first;
1414 tpbuf = GET_TPD_BUFFER(tx_q, first);
1415 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1416 tpbuf->length, DMA_TO_DEVICE);
1417 tpbuf->dma_addr = 0;
1420 if (++first == tx_q->tpd.count)
1427 /* Transmit the packet using specified transmit queue */
1428 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1429 struct sk_buff *skb)
1431 struct emac_tpd tpd;
1434 memset(&tpd, 0, sizeof(tpd));
1436 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1437 dev_kfree_skb_any(skb);
1438 return NETDEV_TX_OK;
1441 if (skb_vlan_tag_present(skb)) {
1444 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1445 TPD_CVLAN_TAG_SET(&tpd, tag);
1446 TPD_INSTC_SET(&tpd, 1);
1449 if (skb_network_offset(skb) != ETH_HLEN)
1450 TPD_TYP_SET(&tpd, 1);
1452 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1454 netdev_sent_queue(adpt->netdev, skb->len);
1456 /* Make sure the are enough free descriptors to hold one
1457 * maximum-sized SKB. We need one desc for each fragment,
1458 * one for the checksum (emac_tso_csum), one for TSO, and
1459 * and one for the SKB header.
1461 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1462 netif_stop_queue(adpt->netdev);
1464 /* update produce idx */
1465 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1467 emac_reg_update32(adpt->base + tx_q->produce_reg,
1468 tx_q->produce_mask, prod_idx);
1470 return NETDEV_TX_OK;