drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_clk_csr_set - dynamically set the MDC clock
 143  * @priv: driver private structure
 144  * Description: this is to dynamically set the MDC clock according to the csr
 145  * clock input.
 146  * Note:
 147  *      If a specific clk_csr value is passed from the platform
 148  *      this means that the CSR Clock Range selection cannot be
 149  *      changed at run-time and it is fixed (as reported in the driver
 150  *      documentation). Viceversa the driver will try to set the MDC
 151  *      clock dynamically according to the actual clock input.
 152  */
 153 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 154 {
 155         u32 clk_rate;
 156
 157         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 158
 159         /* Platform provided default clk_csr would be assumed valid
 160          * for all other cases except for the below mentioned ones.
 161          * For values higher than the IEEE 802.3 specified frequency
 162          * we can not estimate the proper divider as it is not known
 163          * the frequency of clk_csr_i. So we do not change the default
 164          * divider.
 165          */
 166         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 167                 if (clk_rate < CSR_F_35M)
 168                         priv->clk_csr = STMMAC_CSR_20_35M;
 169                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 170                         priv->clk_csr = STMMAC_CSR_35_60M;
 171                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 172                         priv->clk_csr = STMMAC_CSR_60_100M;
 173                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 174                         priv->clk_csr = STMMAC_CSR_100_150M;
 175                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 176                         priv->clk_csr = STMMAC_CSR_150_250M;
 177                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 178                         priv->clk_csr = STMMAC_CSR_250_300M;
 179         }
 180 }
 181
 182 static void print_pkt(unsigned char *buf, int len)
 183 {
 184         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 185         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 186 }
 187
 188 /**
 189  * stmmac_tx_avail - Get tx queue availability
 190  * @priv: driver private structure
 191  * @queue: TX queue index
 192  */
 193 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 194 {
 195         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 196         u32 avail;
 197
 198         if (tx_q->dirty_tx > tx_q->cur_tx)
 199                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 200         else
 201                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 202
 203         return avail;
 204 }
 205
 206 /**
 207  * stmmac_rx_dirty - Get RX queue dirty
 208  * @priv: driver private structure
 209  * @queue: RX queue index
 210  */
 211 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 212 {
 213         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 214         u32 dirty;
 215
 216         if (rx_q->dirty_rx <= rx_q->cur_rx)
 217                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 218         else
 219                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 220
 221         return dirty;
 222 }
 223
 224 /**
 225  * stmmac_hw_fix_mac_speed - callback for speed selection
 226  * @priv: driver private structure
 227  * Description: on some platforms (e.g. ST), some HW system configuration
 228  * registers have to be set according to the link speed negotiated.
 229  */
 230 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 231 {
 232         struct net_device *ndev = priv->dev;
 233         struct phy_device *phydev = ndev->phydev;
 234
 235         if (likely(priv->plat->fix_mac_speed))
 236                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 237 }
 238
 239 /**
 240  * stmmac_enable_eee_mode - check and enter in LPI mode
 241  * @priv: driver private structure
 242  * Description: this function is to verify and enter in LPI mode in case of
 243  * EEE.
 244  */
 245 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 246 {
 247         u32 tx_cnt = priv->plat->tx_queues_to_use;
 248         u32 queue;
 249
 250         /* check if all TX queues have the work finished */
 251         for (queue = 0; queue < tx_cnt; queue++) {
 252                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 253
 254                 if (tx_q->dirty_tx != tx_q->cur_tx)
 255                         return; /* still unfinished work */
 256         }
 257
 258         /* Check and enter in LPI mode */
 259         if (!priv->tx_path_in_lpi_mode)
 260                 priv->hw->mac->set_eee_mode(priv->hw,
 261                                             priv->plat->en_tx_lpi_clockgating);
 262 }
 263
 264 /**
 265  * stmmac_disable_eee_mode - disable and exit from LPI mode
 266  * @priv: driver private structure
 267  * Description: this function is to exit and disable EEE in case of
 268  * LPI state is true. This is called by the xmit.
 269  */
 270 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 271 {
 272         priv->hw->mac->reset_eee_mode(priv->hw);
 273         del_timer_sync(&priv->eee_ctrl_timer);
 274         priv->tx_path_in_lpi_mode = false;
 275 }
 276
 277 /**
 278  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 279  * @arg : data hook
 280  * Description:
 281  *  if there is no data transfer and if we are not in LPI state,
 282  *  then MAC Transmitter can be moved to LPI state.
 283  */
 284 static void stmmac_eee_ctrl_timer(unsigned long arg)
 285 {
 286         struct stmmac_priv *priv = (struct stmmac_priv *)arg;
 287
 288         stmmac_enable_eee_mode(priv);
 289         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 290 }
 291
 292 /**
 293  * stmmac_eee_init - init EEE
 294  * @priv: driver private structure
 295  * Description:
 296  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 297  *  can also manage EEE, this function enable the LPI state and start related
 298  *  timer.
 299  */
 300 bool stmmac_eee_init(struct stmmac_priv *priv)
 301 {
 302         struct net_device *ndev = priv->dev;
 303         unsigned long flags;
 304         bool ret = false;
 305
 306         /* Using PCS we cannot dial with the phy registers at this stage
 307          * so we do not support extra feature like EEE.
 308          */
 309         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 310             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 311             (priv->hw->pcs == STMMAC_PCS_RTBI))
 312                 goto out;
 313
 314         /* MAC core supports the EEE feature. */
 315         if (priv->dma_cap.eee) {
 316                 int tx_lpi_timer = priv->tx_lpi_timer;
 317
 318                 /* Check if the PHY supports EEE */
 319                 if (phy_init_eee(ndev->phydev, 1)) {
 320                         /* To manage at run-time if the EEE cannot be supported
 321                          * anymore (for example because the lp caps have been
 322                          * changed).
 323                          * In that case the driver disable own timers.
 324                          */
 325                         spin_lock_irqsave(&priv->lock, flags);
 326                         if (priv->eee_active) {
 327                                 netdev_dbg(priv->dev, "disable EEE\n");
 328                                 del_timer_sync(&priv->eee_ctrl_timer);
 329                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 330                                                              tx_lpi_timer);
 331                         }
 332                         priv->eee_active = 0;
 333                         spin_unlock_irqrestore(&priv->lock, flags);
 334                         goto out;
 335                 }
 336                 /* Activate the EEE and start timers */
 337                 spin_lock_irqsave(&priv->lock, flags);
 338                 if (!priv->eee_active) {
 339                         priv->eee_active = 1;
 340                         setup_timer(&priv->eee_ctrl_timer,
 341                                     stmmac_eee_ctrl_timer,
 342                                     (unsigned long)priv);
 343                         mod_timer(&priv->eee_ctrl_timer,
 344                                   STMMAC_LPI_T(eee_timer));
 345
 346                         priv->hw->mac->set_eee_timer(priv->hw,
 347                                                      STMMAC_DEFAULT_LIT_LS,
 348                                                      tx_lpi_timer);
 349                 }
 350                 /* Set HW EEE according to the speed */
 351                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 352
 353                 ret = true;
 354                 spin_unlock_irqrestore(&priv->lock, flags);
 355
 356                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 357         }
 358 out:
 359         return ret;
 360 }
 361
 362 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 363  * @priv: driver private structure
 364  * @p : descriptor pointer
 365  * @skb : the socket buffer
 366  * Description :
 367  * This function will read timestamp from the descriptor & pass it to stack.
 368  * and also perform some sanity checks.
 369  */
 370 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 371                                    struct dma_desc *p, struct sk_buff *skb)
 372 {
 373         struct skb_shared_hwtstamps shhwtstamp;
 374         u64 ns;
 375
 376         if (!priv->hwts_tx_en)
 377                 return;
 378
 379         /* exit if skb doesn't support hw tstamp */
 380         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 381                 return;
 382
 383         /* check tx tstamp status */
 384         if (!priv->hw->desc->get_tx_timestamp_status(p)) {
 385                 /* get the valid tstamp */
 386                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 387
 388                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 389                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 390
 391                 netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 392                 /* pass tstamp to stack */
 393                 skb_tstamp_tx(skb, &shhwtstamp);
 394         }
 395
 396         return;
 397 }
 398
 399 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 400  * @priv: driver private structure
 401  * @p : descriptor pointer
 402  * @np : next descriptor pointer
 403  * @skb : the socket buffer
 404  * Description :
 405  * This function will read received packet's timestamp from the descriptor
 406  * and pass it to stack. It also perform some sanity checks.
 407  */
 408 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 409                                    struct dma_desc *np, struct sk_buff *skb)
 410 {
 411         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 412         u64 ns;
 413
 414         if (!priv->hwts_rx_en)
 415                 return;
 416
 417         /* Check if timestamp is available */
 418         if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) {
 419                 /* For GMAC4, the valid timestamp is from CTX next desc. */
 420                 if (priv->plat->has_gmac4)
 421                         ns = priv->hw->desc->get_timestamp(np, priv->adv_ts);
 422                 else
 423                         ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 424
 425                 netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 426                 shhwtstamp = skb_hwtstamps(skb);
 427                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 428                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 429         } else  {
 430                 netdev_err(priv->dev, "cannot get RX hw timestamp\n");
 431         }
 432 }
 433
 434 /**
 435  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 436  *  @dev: device pointer.
 437  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 438  *  a proprietary structure used to pass information to the driver.
 439  *  Description:
 440  *  This function configures the MAC to enable/disable both outgoing(TX)
 441  *  and incoming(RX) packets time stamping based on user input.
 442  *  Return Value:
 443  *  0 on success and an appropriate -ve integer on failure.
 444  */
 445 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 446 {
 447         struct stmmac_priv *priv = netdev_priv(dev);
 448         struct hwtstamp_config config;
 449         struct timespec64 now;
 450         u64 temp = 0;
 451         u32 ptp_v2 = 0;
 452         u32 tstamp_all = 0;
 453         u32 ptp_over_ipv4_udp = 0;
 454         u32 ptp_over_ipv6_udp = 0;
 455         u32 ptp_over_ethernet = 0;
 456         u32 snap_type_sel = 0;
 457         u32 ts_master_en = 0;
 458         u32 ts_event_en = 0;
 459         u32 value = 0;
 460         u32 sec_inc;
 461
 462         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 463                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 464                 priv->hwts_tx_en = 0;
 465                 priv->hwts_rx_en = 0;
 466
 467                 return -EOPNOTSUPP;
 468         }
 469
 470         if (copy_from_user(&config, ifr->ifr_data,
 471                            sizeof(struct hwtstamp_config)))
 472                 return -EFAULT;
 473
 474         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 475                    __func__, config.flags, config.tx_type, config.rx_filter);
 476
 477         /* reserved for future extensions */
 478         if (config.flags)
 479                 return -EINVAL;
 480
 481         if (config.tx_type != HWTSTAMP_TX_OFF &&
 482             config.tx_type != HWTSTAMP_TX_ON)
 483                 return -ERANGE;
 484
 485         if (priv->adv_ts) {
 486                 switch (config.rx_filter) {
 487                 case HWTSTAMP_FILTER_NONE:
 488                         /* time stamp no incoming packet at all */
 489                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 490                         break;
 491
 492                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 493                         /* PTP v1, UDP, any kind of event packet */
 494                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 495                         /* take time stamp for all event messages */
 496                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 497
 498                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 499                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 500                         break;
 501
 502                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 503                         /* PTP v1, UDP, Sync packet */
 504                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 505                         /* take time stamp for SYNC messages only */
 506                         ts_event_en = PTP_TCR_TSEVNTENA;
 507
 508                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 509                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 510                         break;
 511
 512                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 513                         /* PTP v1, UDP, Delay_req packet */
 514                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 515                         /* take time stamp for Delay_Req messages only */
 516                         ts_master_en = PTP_TCR_TSMSTRENA;
 517                         ts_event_en = PTP_TCR_TSEVNTENA;
 518
 519                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 520                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 521                         break;
 522
 523                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 524                         /* PTP v2, UDP, any kind of event packet */
 525                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 526                         ptp_v2 = PTP_TCR_TSVER2ENA;
 527                         /* take time stamp for all event messages */
 528                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 529
 530                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 531                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 532                         break;
 533
 534                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 535                         /* PTP v2, UDP, Sync packet */
 536                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 537                         ptp_v2 = PTP_TCR_TSVER2ENA;
 538                         /* take time stamp for SYNC messages only */
 539                         ts_event_en = PTP_TCR_TSEVNTENA;
 540
 541                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 542                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 543                         break;
 544
 545                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 546                         /* PTP v2, UDP, Delay_req packet */
 547                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 548                         ptp_v2 = PTP_TCR_TSVER2ENA;
 549                         /* take time stamp for Delay_Req messages only */
 550                         ts_master_en = PTP_TCR_TSMSTRENA;
 551                         ts_event_en = PTP_TCR_TSEVNTENA;
 552
 553                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 554                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 555                         break;
 556
 557                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 558                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 559                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 560                         ptp_v2 = PTP_TCR_TSVER2ENA;
 561                         /* take time stamp for all event messages */
 562                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 563
 564                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 565                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 566                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 567                         break;
 568
 569                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 570                         /* PTP v2/802.AS1, any layer, Sync packet */
 571                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 572                         ptp_v2 = PTP_TCR_TSVER2ENA;
 573                         /* take time stamp for SYNC messages only */
 574                         ts_event_en = PTP_TCR_TSEVNTENA;
 575
 576                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 577                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 578                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 579                         break;
 580
 581                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 582                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 583                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 584                         ptp_v2 = PTP_TCR_TSVER2ENA;
 585                         /* take time stamp for Delay_Req messages only */
 586                         ts_master_en = PTP_TCR_TSMSTRENA;
 587                         ts_event_en = PTP_TCR_TSEVNTENA;
 588
 589                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 590                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 591                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 592                         break;
 593
 594                 case HWTSTAMP_FILTER_ALL:
 595                         /* time stamp any incoming packet */
 596                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 597                         tstamp_all = PTP_TCR_TSENALL;
 598                         break;
 599
 600                 default:
 601                         return -ERANGE;
 602                 }
 603         } else {
 604                 switch (config.rx_filter) {
 605                 case HWTSTAMP_FILTER_NONE:
 606                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 607                         break;
 608                 default:
 609                         /* PTP v1, UDP, any kind of event packet */
 610                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 611                         break;
 612                 }
 613         }
 614         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 615         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 616
 617         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 618                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 619         else {
 620                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 621                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 622                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 623                          ts_master_en | snap_type_sel);
 624                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 625
 626                 /* program Sub Second Increment reg */
 627                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 628                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 629                         priv->plat->has_gmac4);
 630                 temp = div_u64(1000000000ULL, sec_inc);
 631
 632                 /* calculate default added value:
 633                  * formula is :
 634                  * addend = (2^32)/freq_div_ratio;
 635                  * where, freq_div_ratio = 1e9ns/sec_inc
 636                  */
 637                 temp = (u64)(temp << 32);
 638                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 639                 priv->hw->ptp->config_addend(priv->ptpaddr,
 640                                              priv->default_addend);
 641
 642                 /* initialize system time */
 643                 ktime_get_real_ts64(&now);
 644
 645                 /* lower 32 bits of tv_sec are safe until y2106 */
 646                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 647                                             now.tv_nsec);
 648         }
 649
 650         return copy_to_user(ifr->ifr_data, &config,
 651                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 652 }
 653
 654 /**
 655  * stmmac_init_ptp - init PTP
 656  * @priv: driver private structure
 657  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 658  * This is done by looking at the HW cap. register.
 659  * This function also registers the ptp driver.
 660  */
 661 static int stmmac_init_ptp(struct stmmac_priv *priv)
 662 {
 663         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 664                 return -EOPNOTSUPP;
 665
 666         priv->adv_ts = 0;
 667         /* Check if adv_ts can be enabled for dwmac 4.x core */
 668         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 669                 priv->adv_ts = 1;
 670         /* Dwmac 3.x core with extend_desc can support adv_ts */
 671         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 672                 priv->adv_ts = 1;
 673
 674         if (priv->dma_cap.time_stamp)
 675                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 676
 677         if (priv->adv_ts)
 678                 netdev_info(priv->dev,
 679                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 680
 681         priv->hw->ptp = &stmmac_ptp;
 682         priv->hwts_tx_en = 0;
 683         priv->hwts_rx_en = 0;
 684
 685         stmmac_ptp_register(priv);
 686
 687         return 0;
 688 }
 689
 690 static void stmmac_release_ptp(struct stmmac_priv *priv)
 691 {
 692         if (priv->plat->clk_ptp_ref)
 693                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 694         stmmac_ptp_unregister(priv);
 695 }
 696
 697 /**
 698  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 699  *  @priv: driver private structure
 700  *  Description: It is used for configuring the flow control in all queues
 701  */
 702 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 703 {
 704         u32 tx_cnt = priv->plat->tx_queues_to_use;
 705
 706         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 707                                  priv->pause, tx_cnt);
 708 }
 709
 710 /**
 711  * stmmac_adjust_link - adjusts the link parameters
 712  * @dev: net device structure
 713  * Description: this is the helper called by the physical abstraction layer
 714  * drivers to communicate the phy link status. According the speed and duplex
 715  * this driver can invoke registered glue-logic as well.
 716  * It also invoke the eee initialization because it could happen when switch
 717  * on different networks (that are eee capable).
 718  */
 719 static void stmmac_adjust_link(struct net_device *dev)
 720 {
 721         struct stmmac_priv *priv = netdev_priv(dev);
 722         struct phy_device *phydev = dev->phydev;
 723         unsigned long flags;
 724         int new_state = 0;
 725
 726         if (!phydev)
 727                 return;
 728
 729         spin_lock_irqsave(&priv->lock, flags);
 730
 731         if (phydev->link) {
 732                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 733
 734                 /* Now we make sure that we can be in full duplex mode.
 735                  * If not, we operate in half-duplex mode. */
 736                 if (phydev->duplex != priv->oldduplex) {
 737                         new_state = 1;
 738                         if (!(phydev->duplex))
 739                                 ctrl &= ~priv->hw->link.duplex;
 740                         else
 741                                 ctrl |= priv->hw->link.duplex;
 742                         priv->oldduplex = phydev->duplex;
 743                 }
 744                 /* Flow Control operation */
 745                 if (phydev->pause)
 746                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 747
 748                 if (phydev->speed != priv->speed) {
 749                         new_state = 1;
 750                         switch (phydev->speed) {
 751                         case 1000:
 752                                 if (priv->plat->has_gmac ||
 753                                     priv->plat->has_gmac4)
 754                                         ctrl &= ~priv->hw->link.port;
 755                                 break;
 756                         case 100:
 757                                 if (priv->plat->has_gmac ||
 758                                     priv->plat->has_gmac4) {
 759                                         ctrl |= priv->hw->link.port;
 760                                         ctrl |= priv->hw->link.speed;
 761                                 } else {
 762                                         ctrl &= ~priv->hw->link.port;
 763                                 }
 764                                 break;
 765                         case 10:
 766                                 if (priv->plat->has_gmac ||
 767                                     priv->plat->has_gmac4) {
 768                                         ctrl |= priv->hw->link.port;
 769                                         ctrl &= ~(priv->hw->link.speed);
 770                                 } else {
 771                                         ctrl &= ~priv->hw->link.port;
 772                                 }
 773                                 break;
 774                         default:
 775                                 netif_warn(priv, link, priv->dev,
 776                                            "broken speed: %d\n", phydev->speed);
 777                                 phydev->speed = SPEED_UNKNOWN;
 778                                 break;
 779                         }
 780                         if (phydev->speed != SPEED_UNKNOWN)
 781                                 stmmac_hw_fix_mac_speed(priv);
 782                         priv->speed = phydev->speed;
 783                 }
 784
 785                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 786
 787                 if (!priv->oldlink) {
 788                         new_state = 1;
 789                         priv->oldlink = 1;
 790                 }
 791         } else if (priv->oldlink) {
 792                 new_state = 1;
 793                 priv->oldlink = 0;
 794                 priv->speed = SPEED_UNKNOWN;
 795                 priv->oldduplex = DUPLEX_UNKNOWN;
 796         }
 797
 798         if (new_state && netif_msg_link(priv))
 799                 phy_print_status(phydev);
 800
 801         spin_unlock_irqrestore(&priv->lock, flags);
 802
 803         if (phydev->is_pseudo_fixed_link)
 804                 /* Stop PHY layer to call the hook to adjust the link in case
 805                  * of a switch is attached to the stmmac driver.
 806                  */
 807                 phydev->irq = PHY_IGNORE_INTERRUPT;
 808         else
 809                 /* At this stage, init the EEE if supported.
 810                  * Never called in case of fixed_link.
 811                  */
 812                 priv->eee_enabled = stmmac_eee_init(priv);
 813 }
 814
 815 /**
 816  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 817  * @priv: driver private structure
 818  * Description: this is to verify if the HW supports the PCS.
 819  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 820  * configured for the TBI, RTBI, or SGMII PHY interface.
 821  */
 822 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 823 {
 824         int interface = priv->plat->interface;
 825
 826         if (priv->dma_cap.pcs) {
 827                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 828                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 829                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 830                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 831                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 832                         priv->hw->pcs = STMMAC_PCS_RGMII;
 833                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 834                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 835                         priv->hw->pcs = STMMAC_PCS_SGMII;
 836                 }
 837         }
 838 }
 839
 840 /**
 841  * stmmac_init_phy - PHY initialization
 842  * @dev: net device structure
 843  * Description: it initializes the driver's PHY state, and attaches the PHY
 844  * to the mac driver.
 845  *  Return value:
 846  *  0 on success
 847  */
 848 static int stmmac_init_phy(struct net_device *dev)
 849 {
 850         struct stmmac_priv *priv = netdev_priv(dev);
 851         struct phy_device *phydev;
 852         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 853         char bus_id[MII_BUS_ID_SIZE];
 854         int interface = priv->plat->interface;
 855         int max_speed = priv->plat->max_speed;
 856         priv->oldlink = 0;
 857         priv->speed = SPEED_UNKNOWN;
 858         priv->oldduplex = DUPLEX_UNKNOWN;
 859
 860         if (priv->plat->phy_node) {
 861                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 862                                         &stmmac_adjust_link, 0, interface);
 863         } else {
 864                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 865                          priv->plat->bus_id);
 866
 867                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 868                          priv->plat->phy_addr);
 869                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 870                            phy_id_fmt);
 871
 872                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 873                                      interface);
 874         }
 875
 876         if (IS_ERR_OR_NULL(phydev)) {
 877                 netdev_err(priv->dev, "Could not attach to PHY\n");
 878                 if (!phydev)
 879                         return -ENODEV;
 880
 881                 return PTR_ERR(phydev);
 882         }
 883
 884         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 885         if ((interface == PHY_INTERFACE_MODE_MII) ||
 886             (interface == PHY_INTERFACE_MODE_RMII) ||
 887                 (max_speed < 1000 && max_speed > 0))
 888                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 889                                          SUPPORTED_1000baseT_Full);
 890
 891         /*
 892          * Broken HW is sometimes missing the pull-up resistor on the
 893          * MDIO line, which results in reads to non-existent devices returning
 894          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 895          * device as well.
 896          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 897          */
 898         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 899                 phy_disconnect(phydev);
 900                 return -ENODEV;
 901         }
 902
 903         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 904          * subsequent PHY polling, make sure we force a link transition if
 905          * we have a UP/DOWN/UP transition
 906          */
 907         if (phydev->is_pseudo_fixed_link)
 908                 phydev->irq = PHY_POLL;
 909
 910         phy_attached_info(phydev);
 911         return 0;
 912 }
 913
 914 static void stmmac_display_rings(struct stmmac_priv *priv)
 915 {
 916         u32 rx_cnt = priv->plat->rx_queues_to_use;
 917         u32 tx_cnt = priv->plat->tx_queues_to_use;
 918         void *head_rx, *head_tx;
 919         u32 queue;
 920
 921         /* Display RX rings */
 922         for (queue = 0; queue < rx_cnt; queue++) {
 923                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 924
 925                 pr_info("\tRX Queue %d rings\n", queue);
 926
 927                 if (priv->extend_desc)
 928                         head_rx = (void *)rx_q->dma_erx;
 929                 else
 930                         head_rx = (void *)rx_q->dma_rx;
 931
 932                 /* Display Rx ring */
 933                 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 934         }
 935
 936         /* Display TX rings */
 937         for (queue = 0; queue < tx_cnt; queue++) {
 938                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 939
 940                 pr_info("\tTX Queue %d rings\n", queue);
 941
 942                 if (priv->extend_desc)
 943                         head_tx = (void *)tx_q->dma_etx;
 944                 else
 945                         head_tx = (void *)tx_q->dma_tx;
 946
 947                 /* Display Tx ring */
 948                 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
 949         }
 950 }
 951
 952 static int stmmac_set_bfsize(int mtu, int bufsize)
 953 {
 954         int ret = bufsize;
 955
 956         if (mtu >= BUF_SIZE_4KiB)
 957                 ret = BUF_SIZE_8KiB;
 958         else if (mtu >= BUF_SIZE_2KiB)
 959                 ret = BUF_SIZE_4KiB;
 960         else if (mtu > DEFAULT_BUFSIZE)
 961                 ret = BUF_SIZE_2KiB;
 962         else
 963                 ret = DEFAULT_BUFSIZE;
 964
 965         return ret;
 966 }
 967
 968 /**
 969  * stmmac_clear_rx_descriptors - clear the descriptors of a RX queue
 970  * @priv: driver private structure
 971  * @queue: RX queue index
 972  * Description: this function is called to clear the RX descriptors
 973  * in case of both basic and extended descriptors are used.
 974  */
 975 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
 976 {
 977         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 978         u32 i = 0;
 979
 980         /* Clear the RX descriptors */
 981         for (i = 0; i < DMA_RX_SIZE; i++)
 982                 if (priv->extend_desc)
 983                         priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
 984                                                      priv->use_riwt, priv->mode,
 985                                                      (i == DMA_RX_SIZE - 1));
 986                 else
 987                         priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
 988                                                      priv->use_riwt, priv->mode,
 989                                                      (i == DMA_RX_SIZE - 1));
 990 }
 991
 992 /**
 993  * stmmac_clear_tx_descriptors - clear the descriptors of a TX queue
 994  * @priv: driver private structure
 995  * @queue: TX queue index
 996  * Description: this function is called to clear the TX descriptors
 997  * in case of both basic and extended descriptors are used.
 998  */
 999 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1000 {
1001         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1002         u32 i = 0;
1003
1004         /* Clear the TX descriptors */
1005         for (i = 0; i < DMA_TX_SIZE; i++)
1006                 if (priv->extend_desc)
1007                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1008                                                      priv->mode,
1009                                                      (i == DMA_TX_SIZE - 1));
1010                 else
1011                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1012                                                      priv->mode,
1013                                                      (i == DMA_TX_SIZE - 1));
1014 }
1015
1016 /**
1017  * stmmac_clear_descriptors - clear descriptors
1018  * @priv: driver private structure
1019  * Description: this function is called to clear the tx and rx descriptors
1020  * in case of both basic and extended descriptors are used.
1021  */
1022 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1023 {
1024         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1025         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1026         u32 queue;
1027
1028         for (queue = 0; queue < rx_queue_cnt; queue++)
1029                 stmmac_clear_rx_descriptors(priv, queue);
1030
1031         for (queue = 0; queue < tx_queue_cnt; queue++)
1032                 stmmac_clear_tx_descriptors(priv, queue);
1033 }
1034
1035 /**
1036  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1037  * @priv: driver private structure
1038  * @p: descriptor pointer
1039  * @i: descriptor index
1040  * @flags: gfp flag.
1041  * @queue: RX queue index
1042  * Description: this function is called to allocate a receive buffer, perform
1043  * the DMA mapping and init the descriptor.
1044  */
1045 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1046                                   int i, gfp_t flags, u32 queue)
1047 {
1048         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1049         struct sk_buff *skb;
1050
1051         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1052         if (!skb) {
1053                 netdev_err(priv->dev,
1054                            "%s: Rx init fails; skb is NULL\n", __func__);
1055                 return -ENOMEM;
1056         }
1057         rx_q->rx_skbuff[i] = skb;
1058         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1059                                                 priv->dma_buf_sz,
1060                                                 DMA_FROM_DEVICE);
1061         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1062                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1063                 dev_kfree_skb_any(skb);
1064                 return -EINVAL;
1065         }
1066
1067         if (priv->synopsys_id >= DWMAC_CORE_4_00)
1068                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1069         else
1070                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1071
1072         if ((priv->hw->mode->init_desc3) &&
1073             (priv->dma_buf_sz == BUF_SIZE_16KiB))
1074                 priv->hw->mode->init_desc3(p);
1075
1076         return 0;
1077 }
1078
1079 /**
1080  * stmmac_free_rx_buffers - free RX buffers.
1081  * @priv: driver private structure
1082  * @queue: RX queue index
1083  * @i: buffer index
1084  */
1085 static void stmmac_free_rx_buffers(struct stmmac_priv *priv, u32 queue, int i)
1086 {
1087         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1088
1089         if (rx_q->rx_skbuff[i]) {
1090                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1091                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1092                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1093         }
1094         rx_q->rx_skbuff[i] = NULL;
1095 }
1096
1097 /**
1098  * stmmac_free_tx_buffers - free RX buffers.
1099  * @priv: driver private structure
1100  * @queue: RX queue index
1101  * @i: buffer index
1102  */
1103 static void stmmac_free_tx_buffers(struct stmmac_priv *priv, u32 queue, u32 i)
1104 {
1105         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1106
1107         if (tx_q->tx_skbuff_dma[i].buf) {
1108                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1109                         dma_unmap_page(priv->device,
1110                                        tx_q->tx_skbuff_dma[i].buf,
1111                                        tx_q->tx_skbuff_dma[i].len,
1112                                        DMA_TO_DEVICE);
1113                 else
1114                         dma_unmap_single(priv->device,
1115                                          tx_q->tx_skbuff_dma[i].buf,
1116                                          tx_q->tx_skbuff_dma[i].len,
1117                                          DMA_TO_DEVICE);
1118         }
1119
1120         if (tx_q->tx_skbuff[i]) {
1121                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1122                 tx_q->tx_skbuff[i] = NULL;
1123                 tx_q->tx_skbuff_dma[i].buf = 0;
1124                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1125         }
1126 }
1127
1128 /**
1129  * init_tx_dma_desc_rings - init the TX descriptor rings
1130  * @dev: net device structure
1131  * Description: this function initializes the DMA TX descriptors
1132  * and allocates the socket buffers. It suppors the chained and ring
1133  * modes.
1134  */
1135 static int init_tx_dma_desc_rings(struct net_device *dev)
1136 {
1137         struct stmmac_priv *priv = netdev_priv(dev);
1138         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1139         u32 queue;
1140         int i = 0;
1141
1142         for (queue = 0; queue < tx_queue_cnt; queue++) {
1143                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1144
1145                 netif_dbg(priv, probe, priv->dev,
1146                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1147                           (u32)tx_q->dma_tx_phy);
1148
1149                 /* Setup the chained descriptor addresses */
1150                 if (priv->mode == STMMAC_CHAIN_MODE) {
1151                         if (priv->extend_desc)
1152                                 priv->hw->mode->init(tx_q->dma_etx,
1153                                                      tx_q->dma_tx_phy,
1154                                                      DMA_TX_SIZE, 1);
1155                         else
1156                                 priv->hw->mode->init(tx_q->dma_tx,
1157                                                      tx_q->dma_tx_phy,
1158                                                      DMA_TX_SIZE, 0);
1159                 }
1160
1161                 for (i = 0; i < DMA_TX_SIZE; i++) {
1162                         struct dma_desc *p;
1163
1164                         if (priv->extend_desc)
1165                                 p = &((tx_q->dma_etx + i)->basic);
1166                         else
1167                                 p = tx_q->dma_tx + i;
1168
1169                         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1170                                 p->des0 = 0;
1171                                 p->des1 = 0;
1172                                 p->des2 = 0;
1173                                 p->des3 = 0;
1174                         } else {
1175                                 p->des2 = 0;
1176                         }
1177
1178                         tx_q->tx_skbuff_dma[i].buf = 0;
1179                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1180                         tx_q->tx_skbuff_dma[i].len = 0;
1181                         tx_q->tx_skbuff_dma[i].last_segment = false;
1182                         tx_q->tx_skbuff[i] = NULL;
1183                 }
1184
1185                 tx_q->dirty_tx = 0;
1186                 tx_q->cur_tx = 0;
1187                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1188         }
1189
1190         return 0;
1191 }
1192
1193 /**
1194  * init_rx_dma_desc_rings - init the RX descriptor rings
1195  * @dev: net device structure
1196  * @flags: gfp flag.
1197  * Description: this function initializes the DMA RX descriptors
1198  * and allocates the socket buffers. It suppors the chained and ring
1199  * modes.
1200  */
1201 static int init_rx_dma_desc_rings(struct net_device *dev, gfp_t flags)
1202 {
1203         struct stmmac_priv *priv = netdev_priv(dev);
1204         u32 rx_count = priv->plat->rx_queues_to_use;
1205         unsigned int bfsize = 0;
1206         int ret = -ENOMEM;
1207         u32 queue;
1208         int i;
1209
1210         if (priv->hw->mode->set_16kib_bfsize)
1211                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1212
1213         if (bfsize < BUF_SIZE_16KiB)
1214                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1215
1216         priv->dma_buf_sz = bfsize;
1217
1218         /* RX INITIALIZATION */
1219         netif_dbg(priv, probe, priv->dev,
1220                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1221
1222         for (queue = 0; queue < rx_count; queue++) {
1223                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1224
1225                 netif_dbg(priv, probe, priv->dev,
1226                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1227                           (u32)rx_q->dma_rx_phy);
1228
1229                 for (i = 0; i < DMA_RX_SIZE; i++) {
1230                         struct dma_desc *p;
1231
1232                         if (priv->extend_desc)
1233                                 p = &((rx_q->dma_erx + i)->basic);
1234                         else
1235                                 p = rx_q->dma_rx + i;
1236
1237                         ret = stmmac_init_rx_buffers(priv, p, i, flags, queue);
1238                         if (ret)
1239                                 goto err_init_rx_buffers;
1240
1241                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1242                                   rx_q->rx_skbuff[i],
1243                                   rx_q->rx_skbuff[i]->data,
1244                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1245                 }
1246
1247                 rx_q->cur_rx = 0;
1248                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1249
1250                 stmmac_clear_rx_descriptors(priv, queue);
1251
1252                 if (priv->mode == STMMAC_CHAIN_MODE) {
1253                         if (priv->extend_desc)
1254                                 priv->hw->mode->init(rx_q->dma_erx,
1255                                                      rx_q->dma_rx_phy,
1256                                                      DMA_RX_SIZE, 1);
1257                         else
1258                                 priv->hw->mode->init(rx_q->dma_rx,
1259                                                      rx_q->dma_rx_phy,
1260                                                      DMA_RX_SIZE, 0);
1261                 }
1262         }
1263
1264         buf_sz = bfsize;
1265
1266         return 0;
1267
1268 err_init_rx_buffers:
1269         while (queue-- >= 0) {
1270                 while (--i >= 0)
1271                         stmmac_free_rx_buffers(priv, queue, i);
1272
1273                 i = DMA_RX_SIZE;
1274         }
1275
1276         return ret;
1277 }
1278
1279 /**
1280  * init_dma_desc_rings - init the RX/TX descriptor rings
1281  * @dev: net device structure
1282  * @flags: gfp flag.
1283  * Description: this function initializes the DMA RX/TX descriptors
1284  * and allocates the socket buffers. It suppors the chained and ring
1285  * modes.
1286  */
1287 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1288 {
1289         struct stmmac_priv *priv = netdev_priv(dev);
1290         int ret = init_rx_dma_desc_rings(dev, flags);
1291
1292         if (ret)
1293                 return ret;
1294
1295         ret = init_tx_dma_desc_rings(dev);
1296
1297         if (netif_msg_hw(priv))
1298                 stmmac_display_rings(priv);
1299
1300         return ret;
1301 }
1302
1303 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1304 {
1305         int i;
1306
1307         for (i = 0; i < DMA_RX_SIZE; i++)
1308                 stmmac_free_rx_buffers(priv, queue, i);
1309 }
1310
1311 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1312 {
1313         int i;
1314
1315         for (i = 0; i < DMA_TX_SIZE; i++)
1316                 stmmac_free_tx_buffers(priv, queue, i);
1317 }
1318
1319 /**
1320  * free_rx_dma_desc_resources - free RX DMA resources
1321  * @priv: driver private structure
1322  */
1323 static void free_rx_dma_desc_resources(struct stmmac_priv *priv)
1324 {
1325         u32 rx_count = priv->plat->rx_queues_to_use;
1326         u32 queue = 0;
1327
1328         if (!priv->rx_queue)
1329                 return;
1330
1331         /* Free RX queue resources */
1332         for (queue = 0; queue < rx_count; queue++) {
1333                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1334
1335                 if (!rx_q)
1336                         break;
1337
1338                 /* Release the DMA RX socket buffers */
1339                 dma_free_rx_skbufs(priv, queue);
1340
1341                 kfree(rx_q->rx_skbuff);
1342
1343                 kfree(rx_q->rx_skbuff_dma);
1344
1345                 if (!priv->extend_desc)
1346                         dma_free_coherent(priv->device,
1347                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1348                                           rx_q->dma_rx,
1349                                           rx_q->dma_rx_phy);
1350                 else
1351                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1352                                           sizeof(struct dma_extended_desc),
1353                                           rx_q->dma_erx,
1354                                           rx_q->dma_rx_phy);
1355         }
1356
1357         kfree(priv->rx_queue);
1358 }
1359
1360 /**
1361  * free_tx_dma_desc_resources - free TX DMA resources
1362  * @priv: driver private structure
1363  */
1364 static void free_tx_dma_desc_resources(struct stmmac_priv *priv)
1365 {
1366         u32 tx_count = priv->plat->tx_queues_to_use;
1367         u32 queue = 0;
1368
1369         if (!priv->tx_queue)
1370                 return;
1371
1372         /* Free TX queue resources */
1373         for (queue = 0; queue < tx_count; queue++) {
1374                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1375
1376                 if (!tx_q)
1377                         break;
1378
1379                 /* Release the DMA TX socket buffers */
1380                 dma_free_tx_skbufs(priv, queue);
1381
1382                 kfree(tx_q->tx_skbuff);
1383
1384                 kfree(tx_q->tx_skbuff_dma);
1385
1386                 if (!priv->extend_desc)
1387                         dma_free_coherent(priv->device,
1388                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1389                                           tx_q->dma_tx,
1390                                           tx_q->dma_tx_phy);
1391                 else
1392                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1393                                           sizeof(struct dma_extended_desc),
1394                                           tx_q->dma_etx,
1395                                           tx_q->dma_tx_phy);
1396         }
1397
1398         kfree(priv->tx_queue);
1399 }
1400
1401 /**
1402  * free_dma_desc_resources - free All DMA resources
1403  * @priv: driver private structure
1404  */
1405 static void free_dma_desc_resources(struct stmmac_priv *priv)
1406 {
1407         free_rx_dma_desc_resources(priv);
1408         free_tx_dma_desc_resources(priv);
1409 }
1410
1411 /**
1412  * alloc_rx_dma_desc_resources - alloc RX resources.
1413  * @priv: private structure
1414  * Description: according to which descriptor can be used (extend or basic)
1415  * this function allocates the resources for RX paths. It pre-allocates the
1416  * RX socket buffer in order to allow zero-copy mechanism.
1417  */
1418 static int alloc_rx_dma_desc_resources(struct stmmac_priv *priv)
1419 {
1420         u32 rx_count = priv->plat->rx_queues_to_use;
1421         int ret = -ENOMEM;
1422         u32 queue = 0;
1423
1424         /* Allocate RX queues array */
1425         priv->rx_queue = kmalloc_array(rx_count,
1426                                        sizeof(struct stmmac_rx_queue),
1427                                        GFP_KERNEL);
1428         if (!priv->rx_queue) {
1429                 kfree(priv->rx_queue);
1430                 return -ENOMEM;
1431         }
1432
1433         /* RX queues buffers and DMA */
1434         for (queue = 0; queue < rx_count; queue++) {
1435                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1436
1437                 rx_q->queue_index = queue;
1438                 rx_q->priv_data = priv;
1439
1440                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1441                                                         sizeof(dma_addr_t),
1442                                                         GFP_KERNEL);
1443                 if (!rx_q->rx_skbuff_dma)
1444                         goto err_dma_buffers;
1445
1446                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1447                                                     sizeof(struct sk_buff *),
1448                                                     GFP_KERNEL);
1449                 if (!rx_q->rx_skbuff)
1450                         goto err_dma_buffers;
1451
1452                 if (priv->extend_desc) {
1453                         rx_q->dma_erx = dma_zalloc_coherent(priv->device,
1454                         (DMA_RX_SIZE * sizeof(struct dma_extended_desc)),
1455                         &rx_q->dma_rx_phy, GFP_KERNEL);
1456
1457                         if (!rx_q->dma_erx)
1458                                 goto err_dma_buffers;
1459                 } else {
1460                         rx_q->dma_rx = dma_zalloc_coherent(priv->device,
1461                         (DMA_RX_SIZE * sizeof(struct dma_desc)),
1462                         &rx_q->dma_rx_phy, GFP_KERNEL);
1463
1464                         if (!rx_q->dma_rx)
1465                                 goto err_dma_buffers;
1466                 }
1467         }
1468
1469         return 0;
1470
1471 err_dma_buffers:
1472         free_rx_dma_desc_resources(priv);
1473
1474         return ret;
1475 }
1476
1477 /**
1478  * alloc_tx_dma_desc_resources - alloc TX resources.
1479  * @priv: private structure
1480  * Description: according to which descriptor can be used (extend or basic)
1481  * this function allocates the resources for TX paths.
1482  */
1483 static int alloc_tx_dma_desc_resources(struct stmmac_priv *priv)
1484 {
1485         u32 tx_count = priv->plat->tx_queues_to_use;
1486         int ret = -ENOMEM;
1487         u32 queue = 0;
1488
1489         /* Allocate TX queues array */
1490         priv->tx_queue = kmalloc_array(tx_count,
1491                                        sizeof(struct stmmac_tx_queue),
1492                                        GFP_KERNEL);
1493         if (!priv->tx_queue)
1494                 return -ENOMEM;
1495
1496         /* TX queues buffers and DMA */
1497         for (queue = 0; queue < tx_count; queue++) {
1498                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1499
1500                 tx_q->queue_index = queue;
1501                 tx_q->priv_data = priv;
1502
1503                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1504                                           sizeof(struct stmmac_tx_info),
1505                                           GFP_KERNEL);
1506
1507                 if (!tx_q->tx_skbuff_dma)
1508                         goto err_dma_buffers;
1509
1510                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1511                                                     sizeof(struct sk_buff *),
1512                                                     GFP_KERNEL);
1513                 if (!tx_q->tx_skbuff)
1514                         goto err_dma_buffers;
1515
1516                 if (priv->extend_desc) {
1517                         tx_q->dma_etx =
1518                         dma_zalloc_coherent(priv->device,
1519                         (DMA_TX_SIZE * sizeof(struct dma_extended_desc)),
1520                         &tx_q->dma_tx_phy, GFP_KERNEL);
1521
1522                         if (!tx_q->dma_etx)
1523                                 goto err_dma_buffers;
1524                 } else {
1525                         tx_q->dma_tx =
1526                         dma_zalloc_coherent(priv->device,
1527                         (DMA_TX_SIZE * sizeof(struct dma_desc)),
1528                         &tx_q->dma_tx_phy, GFP_KERNEL);
1529
1530                         if (!tx_q->dma_tx)
1531                                 goto err_dma_buffers;
1532                 }
1533         }
1534
1535         return 0;
1536
1537 err_dma_buffers:
1538         free_tx_dma_desc_resources(priv);
1539
1540         return ret;
1541 }
1542
1543 /**
1544  * alloc_dma_desc_resources - alloc TX/RX resources.
1545  * @priv: private structure
1546  * Description: according to which descriptor can be used (extend or basic)
1547  * this function allocates the resources for TX and RX paths. In case of
1548  * reception, for example, it pre-allocated the RX socket buffer in order to
1549  * allow zero-copy mechanism.
1550  */
1551 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1552 {
1553         int ret = 0;
1554
1555         ret = alloc_tx_dma_desc_resources(priv);
1556         if (ret)
1557                 return ret;
1558
1559         ret = alloc_rx_dma_desc_resources(priv);
1560
1561         return ret;
1562 }
1563
1564 /**
1565  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1566  *  @priv: driver private structure
1567  *  Description: It is used for enabling the rx queues in the MAC
1568  */
1569 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1570 {
1571         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1572         int queue;
1573         u8 mode;
1574
1575         for (queue = 0; queue < rx_queues_count; queue++) {
1576                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1577                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1578         }
1579 }
1580
1581 /**
1582  * stmmac_start_rx_dma - start RX DMA channel
1583  * @priv: driver private structure
1584  * @chan: RX channel index
1585  * Description:
1586  * This starts a RX DMA channel
1587  */
1588 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1589 {
1590         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1591         priv->hw->dma->start_rx(priv->ioaddr, chan);
1592 }
1593
1594 /**
1595  * stmmac_start_tx_dma - start TX DMA channel
1596  * @priv: driver private structure
1597  * @chan: TX channel index
1598  * Description:
1599  * This starts a TX DMA channel
1600  */
1601 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1602 {
1603         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1604         priv->hw->dma->start_tx(priv->ioaddr, chan);
1605 }
1606
1607 /**
1608  * stmmac_stop_rx_dma - stop RX DMA channel
1609  * @priv: driver private structure
1610  * @chan: RX channel index
1611  * Description:
1612  * This stops a RX DMA channel
1613  */
1614 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1615 {
1616         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1617         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1618 }
1619
1620 /**
1621  * stmmac_stop_tx_dma - stop TX DMA channel
1622  * @priv: driver private structure
1623  * @chan: TX channel index
1624  * Description:
1625  * This stops a TX DMA channel
1626  */
1627 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1628 {
1629         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1630         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1631 }
1632
1633 /**
1634  * stmmac_start_all_dma - start all RX and TX DMA channels
1635  * @priv: driver private structure
1636  * Description:
1637  * This starts all the RX and TX DMA channels
1638  */
1639 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1640 {
1641         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1642         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1643         u32 chan = 0;
1644
1645         for (chan = 0; chan < rx_channels_count; chan++)
1646                 stmmac_start_rx_dma(priv, chan);
1647
1648         for (chan = 0; chan < tx_channels_count; chan++)
1649                 stmmac_start_tx_dma(priv, chan);
1650 }
1651
1652 /**
1653  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1654  * @priv: driver private structure
1655  * Description:
1656  * This stops the RX and TX DMA channels
1657  */
1658 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1659 {
1660         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1661         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1662         u32 chan = 0;
1663
1664         for (chan = 0; chan < rx_channels_count; chan++)
1665                 stmmac_stop_rx_dma(priv, chan);
1666
1667         for (chan = 0; chan < tx_channels_count; chan++)
1668                 stmmac_stop_tx_dma(priv, chan);
1669 }
1670
1671 /**
1672  *  stmmac_dma_operation_mode - HW DMA operation mode
1673  *  @priv: driver private structure
1674  *  Description: it is used for configuring the DMA operation mode register in
1675  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1676  */
1677 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1678 {
1679         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1680         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1681         int rxfifosz = priv->plat->rx_fifo_size;
1682         u32 txmode = 0;
1683         u32 rxmode = 0;
1684         u32 chan = 0;
1685
1686         if (rxfifosz == 0)
1687                 rxfifosz = priv->dma_cap.rx_fifo_size;
1688
1689         if (priv->plat->force_thresh_dma_mode) {
1690                 txmode = tc;
1691                 rxmode = tc;
1692         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1693                 /*
1694                  * In case of GMAC, SF mode can be enabled
1695                  * to perform the TX COE in HW. This depends on:
1696                  * 1) TX COE if actually supported
1697                  * 2) There is no bugged Jumbo frame support
1698                  *    that needs to not insert csum in the TDES.
1699                  */
1700                 txmode = SF_DMA_MODE;
1701                 rxmode = SF_DMA_MODE;
1702                 priv->xstats.threshold = SF_DMA_MODE;
1703         } else {
1704                 txmode = tc;
1705                 rxmode = SF_DMA_MODE;
1706         }
1707
1708         /* configure all channels */
1709         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1710                 for (chan = 0; chan < rx_channels_count; chan++)
1711                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1712                                                    rxfifosz);
1713
1714                 for (chan = 0; chan < tx_channels_count; chan++)
1715                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1716         } else {
1717                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1718                                         rxfifosz);
1719         }
1720 }
1721
1722 /**
1723  * stmmac_tx_clean - to manage the transmission completion
1724  * @priv: driver private structure
1725  * @queue: TX queue index
1726  * Description: it reclaims the transmit resources after transmission completes.
1727  */
1728 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1729 {
1730         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1731         unsigned int bytes_compl = 0, pkts_compl = 0;
1732         unsigned int entry = tx_q->dirty_tx;
1733
1734         netif_tx_lock(priv->dev);
1735
1736         priv->xstats.tx_clean++;
1737
1738         while (entry != tx_q->cur_tx) {
1739                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1740                 struct dma_desc *p;
1741                 int status;
1742
1743                 if (priv->extend_desc)
1744                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1745                 else
1746                         p = tx_q->dma_tx + entry;
1747
1748                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1749                                                       &priv->xstats, p,
1750                                                       priv->ioaddr);
1751                 /* Check if the descriptor is owned by the DMA */
1752                 if (unlikely(status & tx_dma_own))
1753                         break;
1754
1755                 /* Just consider the last segment and ...*/
1756                 if (likely(!(status & tx_not_ls))) {
1757                         /* ... verify the status error condition */
1758                         if (unlikely(status & tx_err)) {
1759                                 priv->dev->stats.tx_errors++;
1760                         } else {
1761                                 priv->dev->stats.tx_packets++;
1762                                 priv->xstats.tx_pkt_n++;
1763                         }
1764                         stmmac_get_tx_hwtstamp(priv, p, skb);
1765                 }
1766
1767                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1768                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1769                                 dma_unmap_page(priv->device,
1770                                                tx_q->tx_skbuff_dma[entry].buf,
1771                                                tx_q->tx_skbuff_dma[entry].len,
1772                                                DMA_TO_DEVICE);
1773                         else
1774                                 dma_unmap_single(priv->device,
1775                                                  tx_q->tx_skbuff_dma[entry].buf,
1776                                                  tx_q->tx_skbuff_dma[entry].len,
1777                                                  DMA_TO_DEVICE);
1778                         tx_q->tx_skbuff_dma[entry].buf = 0;
1779                         tx_q->tx_skbuff_dma[entry].len = 0;
1780                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1781                 }
1782
1783                 if (priv->hw->mode->clean_desc3)
1784                         priv->hw->mode->clean_desc3(tx_q, p);
1785
1786                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1787                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1788
1789                 if (likely(skb != NULL)) {
1790                         pkts_compl++;
1791                         bytes_compl += skb->len;
1792                         dev_consume_skb_any(skb);
1793                         tx_q->tx_skbuff[entry] = NULL;
1794                 }
1795
1796                 priv->hw->desc->release_tx_desc(p, priv->mode);
1797
1798                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1799         }
1800         tx_q->dirty_tx = entry;
1801
1802         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1803                                   pkts_compl, bytes_compl);
1804
1805         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1806                                                                queue))) &&
1807             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1808                 netif_dbg(priv, tx_done, priv->dev,
1809                           "%s: restart transmit\n", __func__);
1810                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1811         }
1812
1813         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1814                 stmmac_enable_eee_mode(priv);
1815                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1816         }
1817         netif_tx_unlock(priv->dev);
1818 }
1819
1820 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1821 {
1822         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1823 }
1824
1825 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1826 {
1827         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1828 }
1829
1830 /**
1831  * stmmac_tx_err - to manage the tx error
1832  * @priv: driver private structure
1833  * @queue: queue index
1834  * Description: it cleans the descriptors and restarts the transmission
1835  * in case of transmission errors.
1836  */
1837 static void stmmac_tx_err(struct stmmac_priv *priv, u32 queue)
1838 {
1839         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1840         u32 chan = queue;
1841         int i;
1842
1843         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
1844
1845         stmmac_stop_tx_dma(priv, chan);
1846         dma_free_tx_skbufs(priv, queue);
1847         for (i = 0; i < DMA_TX_SIZE; i++)
1848                 if (priv->extend_desc)
1849                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1850                                                      priv->mode,
1851                                                      (i == DMA_TX_SIZE - 1));
1852                 else
1853                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1854                                                      priv->mode,
1855                                                      (i == DMA_TX_SIZE - 1));
1856         tx_q->dirty_tx = 0;
1857         tx_q->cur_tx = 0;
1858         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1859         stmmac_start_tx_dma(priv, chan);
1860
1861         priv->dev->stats.tx_errors++;
1862         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1863 }
1864
1865 /**
1866  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1867  *  @priv: driver private structure
1868  *  @txmode: TX operating mode
1869  *  @rxmode: RX operating mode
1870  *  @chan: channel index
1871  *  Description: it is used for configuring of the DMA operation mode in
1872  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1873  *  mode.
1874  */
1875 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1876                                           u32 rxmode, u32 chan)
1877 {
1878         int rxfifosz = priv->plat->rx_fifo_size;
1879
1880         if (rxfifosz == 0)
1881                 rxfifosz = priv->dma_cap.rx_fifo_size;
1882
1883         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1884                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1885                                            rxfifosz);
1886                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1887         } else {
1888                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1889                                         rxfifosz);
1890         }
1891 }
1892
1893 /**
1894  * stmmac_dma_interrupt - DMA ISR
1895  * @priv: driver private structure
1896  * Description: this is the DMA ISR. It is called by the main ISR.
1897  * It calls the dwmac dma routine and schedule poll method in case of some
1898  * work can be done.
1899  */
1900 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
1901 {
1902         u32 tx_channel_count = priv->plat->tx_queues_to_use;
1903         int status;
1904         u32 chan;
1905
1906         for (chan = 0; chan < tx_channel_count; chan++) {
1907                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
1908
1909                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
1910                                                       &priv->xstats, chan);
1911                 if (likely((status & handle_rx)) || (status & handle_tx)) {
1912                         if (likely(napi_schedule_prep(&rx_q->napi))) {
1913                                 stmmac_disable_dma_irq(priv, chan);
1914                                 __napi_schedule(&rx_q->napi);
1915                         }
1916                 }
1917
1918                 if (unlikely(status & tx_hard_error_bump_tc)) {
1919                         /* Try to bump up the dma threshold on this failure */
1920                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
1921                             (tc <= 256)) {
1922                                 tc += 64;
1923                                 if (priv->plat->force_thresh_dma_mode)
1924                                         stmmac_set_dma_operation_mode(priv,
1925                                                                       tc,
1926                                                                       tc,
1927                                                                       chan);
1928                                 else
1929                                         stmmac_set_dma_operation_mode(priv,
1930                                                                     tc,
1931                                                                     SF_DMA_MODE,
1932                                                                     chan);
1933                                 priv->xstats.threshold = tc;
1934                         }
1935                 } else if (unlikely(status == tx_hard_error)) {
1936                         stmmac_tx_err(priv, chan);
1937                 }
1938         }
1939 }
1940
1941 /**
1942  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
1943  * @priv: driver private structure
1944  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
1945  */
1946 static void stmmac_mmc_setup(struct stmmac_priv *priv)
1947 {
1948         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
1949                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
1950
1951         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1952                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
1953                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
1954         } else {
1955                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
1956                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
1957         }
1958
1959         dwmac_mmc_intr_all_mask(priv->mmcaddr);
1960
1961         if (priv->dma_cap.rmon) {
1962                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
1963                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
1964         } else
1965                 netdev_info(priv->dev, "No MAC Management Counters available\n");
1966 }
1967
1968 /**
1969  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
1970  * @priv: driver private structure
1971  * Description: select the Enhanced/Alternate or Normal descriptors.
1972  * In case of Enhanced/Alternate, it checks if the extended descriptors are
1973  * supported by the HW capability register.
1974  */
1975 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
1976 {
1977         if (priv->plat->enh_desc) {
1978                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
1979
1980                 /* GMAC older than 3.50 has no extended descriptors */
1981                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
1982                         dev_info(priv->device, "Enabled extended descriptors\n");
1983                         priv->extend_desc = 1;
1984                 } else
1985                         dev_warn(priv->device, "Extended descriptors not supported\n");
1986
1987                 priv->hw->desc = &enh_desc_ops;
1988         } else {
1989                 dev_info(priv->device, "Normal descriptors\n");
1990                 priv->hw->desc = &ndesc_ops;
1991         }
1992 }
1993
1994 /**
1995  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
1996  * @priv: driver private structure
1997  * Description:
1998  *  new GMAC chip generations have a new register to indicate the
1999  *  presence of the optional feature/functions.
2000  *  This can be also used to override the value passed through the
2001  *  platform and necessary for old MAC10/100 and GMAC chips.
2002  */
2003 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2004 {
2005         u32 ret = 0;
2006
2007         if (priv->hw->dma->get_hw_feature) {
2008                 priv->hw->dma->get_hw_feature(priv->ioaddr,
2009                                               &priv->dma_cap);
2010                 ret = 1;
2011         }
2012
2013         return ret;
2014 }
2015
2016 /**
2017  * stmmac_check_ether_addr - check if the MAC addr is valid
2018  * @priv: driver private structure
2019  * Description:
2020  * it is to verify if the MAC address is valid, in case of failures it
2021  * generates a random MAC address
2022  */
2023 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2024 {
2025         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2026                 priv->hw->mac->get_umac_addr(priv->hw,
2027                                              priv->dev->dev_addr, 0);
2028                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2029                         eth_hw_addr_random(priv->dev);
2030                 netdev_info(priv->dev, "device MAC address %pM\n",
2031                             priv->dev->dev_addr);
2032         }
2033 }
2034
2035 /**
2036  * stmmac_init_dma_engine - DMA init.
2037  * @priv: driver private structure
2038  * Description:
2039  * It inits the DMA invoking the specific MAC/GMAC callback.
2040  * Some DMA parameters can be passed from the platform;
2041  * in case of these are not passed a default is kept for the MAC or GMAC.
2042  */
2043 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2044 {
2045         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2046         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2047         struct stmmac_rx_queue *rx_q;
2048         struct stmmac_tx_queue *tx_q;
2049         u32 dummy_dma_rx_phy = 0;
2050         u32 dummy_dma_tx_phy = 0;
2051         u32 chan = 0;
2052         int atds = 0;
2053         int ret = 0;
2054
2055         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2056                 dev_err(priv->device, "Invalid DMA configuration\n");
2057                 return -EINVAL;
2058         }
2059
2060         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2061                 atds = 1;
2062
2063         ret = priv->hw->dma->reset(priv->ioaddr);
2064         if (ret) {
2065                 dev_err(priv->device, "Failed to reset the dma\n");
2066                 return ret;
2067         }
2068
2069         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2070                 /* DMA Configuration */
2071                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2072                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2073
2074                 /* DMA RX Channel Configuration */
2075                 for (chan = 0; chan < rx_channels_count; chan++) {
2076                         rx_q = &priv->rx_queue[chan];
2077
2078                         priv->hw->dma->init_rx_chan(priv->ioaddr,
2079                                                     priv->plat->dma_cfg,
2080                                                     rx_q->dma_rx_phy, chan);
2081
2082                         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2083                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
2084                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
2085                                                        rx_q->rx_tail_addr,
2086                                                        chan);
2087                 }
2088
2089                 /* DMA TX Channel Configuration */
2090                 for (chan = 0; chan < tx_channels_count; chan++) {
2091                         tx_q = &priv->tx_queue[chan];
2092
2093                         priv->hw->dma->init_chan(priv->ioaddr,
2094                                                  priv->plat->dma_cfg,
2095                                                  chan);
2096
2097                         priv->hw->dma->init_tx_chan(priv->ioaddr,
2098                                                     priv->plat->dma_cfg,
2099                                                     tx_q->dma_tx_phy, chan);
2100
2101                         tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2102                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
2103                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
2104                                                        tx_q->tx_tail_addr,
2105                                                        chan);
2106                 }
2107         } else {
2108                 rx_q = &priv->rx_queue[chan];
2109                 tx_q = &priv->tx_queue[chan];
2110
2111                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2112                                     tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
2113         }
2114
2115         if (priv->plat->axi && priv->hw->dma->axi)
2116                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
2117
2118         return ret;
2119 }
2120
2121 /**
2122  * stmmac_tx_timer - mitigation sw timer for tx.
2123  * @data: data pointer
2124  * Description:
2125  * This is the timer handler to directly invoke the stmmac_tx_clean.
2126  */
2127 static void stmmac_tx_timer(unsigned long data)
2128 {
2129         struct stmmac_priv *priv = (struct stmmac_priv *)data;
2130         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2131         u32 queue;
2132
2133         /* let's scan all the tx queues */
2134         for (queue = 0; queue < tx_queues_count; queue++)
2135                 stmmac_tx_clean(priv, queue);
2136 }
2137
2138 /**
2139  * stmmac_stop_all_queues - Stop all queues
2140  * @priv: driver private structure
2141  */
2142 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
2143 {
2144         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
2145         u32 queue;
2146
2147         for (queue = 0; queue < tx_queues_cnt; queue++)
2148                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2149 }
2150
2151 /**
2152  * stmmac_start_all_queues - Start all queues
2153  * @priv: driver private structure
2154  */
2155 static void stmmac_start_all_queues(struct stmmac_priv *priv)
2156 {
2157         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
2158         u32 queue;
2159
2160         for (queue = 0; queue < tx_queues_cnt; queue++)
2161                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
2162 }
2163
2164 /**
2165  * stmmac_disable_all_queues - Disable all queues
2166  * @priv: driver private structure
2167  */
2168 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
2169 {
2170         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
2171         u32 queue;
2172
2173         for (queue = 0; queue < rx_queues_cnt; queue++) {
2174                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2175
2176                 napi_disable(&rx_q->napi);
2177         }
2178 }
2179
2180 /**
2181  * stmmac_enable_all_queues - Enable all queues
2182  * @priv: driver private structure
2183  */
2184 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
2185 {
2186         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
2187         u32 queue;
2188
2189         for (queue = 0; queue < rx_queues_cnt; queue++) {
2190                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2191
2192                 napi_enable(&rx_q->napi);
2193         }
2194 }
2195
2196 /**
2197  * stmmac_init_tx_coalesce - init tx mitigation options.
2198  * @priv: driver private structure
2199  * Description:
2200  * This inits the transmit coalesce parameters: i.e. timer rate,
2201  * timer handler and default threshold used for enabling the
2202  * interrupt on completion bit.
2203  */
2204 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2205 {
2206         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2207         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2208         init_timer(&priv->txtimer);
2209         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2210         priv->txtimer.data = (unsigned long)priv;
2211         priv->txtimer.function = stmmac_tx_timer;
2212         add_timer(&priv->txtimer);
2213 }
2214
2215 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2216 {
2217         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2218         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2219         u32 chan;
2220
2221         /* set TX ring length */
2222         if (priv->hw->dma->set_tx_ring_len) {
2223                 for (chan = 0; chan < tx_channels_count; chan++)
2224                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
2225                                                        (DMA_TX_SIZE - 1), chan);
2226         }
2227
2228         /* set RX ring length */
2229         if (priv->hw->dma->set_rx_ring_len) {
2230                 for (chan = 0; chan < rx_channels_count; chan++)
2231                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
2232                                                        (DMA_RX_SIZE - 1), chan);
2233         }
2234 }
2235
2236 /**
2237  *  stmmac_set_tx_queue_weight - Set TX queue weight
2238  *  @priv: driver private structure
2239  *  Description: It is used for setting TX queues weight
2240  */
2241 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2242 {
2243         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2244         u32 weight;
2245         u32 queue;
2246
2247         for (queue = 0; queue < tx_queues_count; queue++) {
2248                 weight = priv->plat->tx_queues_cfg[queue].weight;
2249                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
2250         }
2251 }
2252
2253 /**
2254  *  stmmac_configure_cbs - Configure CBS in TX queue
2255  *  @priv: driver private structure
2256  *  Description: It is used for configuring CBS in AVB TX queues
2257  */
2258 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2259 {
2260         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2261         u32 mode_to_use;
2262         u32 queue;
2263
2264         for (queue = 0; queue < tx_queues_count; queue++) {
2265                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2266                 if (mode_to_use == MTL_QUEUE_DCB)
2267                         continue;
2268
2269                 priv->hw->mac->config_cbs(priv->hw,
2270                                 priv->plat->tx_queues_cfg[queue].send_slope,
2271                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2272                                 priv->plat->tx_queues_cfg[queue].high_credit,
2273                                 priv->plat->tx_queues_cfg[queue].low_credit,
2274                                 queue);
2275         }
2276 }
2277
2278 /**
2279  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2280  *  @priv: driver private structure
2281  *  Description: It is used for mapping RX queues to RX dma channels
2282  */
2283 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2284 {
2285         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2286         u32 queue;
2287         u32 chan;
2288
2289         for (queue = 0; queue < rx_queues_count; queue++) {
2290                 chan = priv->plat->rx_queues_cfg[queue].chan;
2291                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
2292         }
2293 }
2294
2295 /**
2296  *  stmmac_mtl_configuration - Configure MTL
2297  *  @priv: driver private structure
2298  *  Description: It is used for configurring MTL
2299  */
2300 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2301 {
2302         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2303         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2304
2305         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
2306                 stmmac_set_tx_queue_weight(priv);
2307
2308         /* Configure MTL RX algorithms */
2309         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
2310                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
2311                                                 priv->plat->rx_sched_algorithm);
2312
2313         /* Configure MTL TX algorithms */
2314         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
2315                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
2316                                                 priv->plat->tx_sched_algorithm);
2317
2318         /* Configure CBS in AVB TX queues */
2319         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2320                 stmmac_configure_cbs(priv);
2321
2322         /* Map RX MTL to DMA channels */
2323         if (rx_queues_count > 1 && priv->hw->mac->map_mtl_to_dma)
2324                 stmmac_rx_queue_dma_chan_map(priv);
2325
2326         /* Enable MAC RX Queues */
2327         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_enable)
2328                 stmmac_mac_enable_rx_queues(priv);
2329
2330         /* Set the HW DMA mode and the COE */
2331         stmmac_dma_operation_mode(priv);
2332 }
2333
2334 /**
2335  * stmmac_hw_setup - setup mac in a usable state.
2336  *  @dev : pointer to the device structure.
2337  *  Description:
2338  *  this is the main function to setup the HW in a usable state because the
2339  *  dma engine is reset, the core registers are configured (e.g. AXI,
2340  *  Checksum features, timers). The DMA is ready to start receiving and
2341  *  transmitting.
2342  *  Return value:
2343  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2344  *  file on failure.
2345  */
2346 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2347 {
2348         struct stmmac_priv *priv = netdev_priv(dev);
2349         u32 rx_cnt = priv->plat->rx_queues_to_use;
2350         u32 tx_cnt = priv->plat->tx_queues_to_use;
2351         u32 chan;
2352         int ret;
2353
2354         /* DMA initialization and SW reset */
2355         ret = stmmac_init_dma_engine(priv);
2356         if (ret < 0) {
2357                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2358                            __func__);
2359                 return ret;
2360         }
2361
2362         /* Copy the MAC addr into the HW  */
2363         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2364
2365         /* PS and related bits will be programmed according to the speed */
2366         if (priv->hw->pcs) {
2367                 int speed = priv->plat->mac_port_sel_speed;
2368
2369                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2370                     (speed == SPEED_1000)) {
2371                         priv->hw->ps = speed;
2372                 } else {
2373                         dev_warn(priv->device, "invalid port speed\n");
2374                         priv->hw->ps = 0;
2375                 }
2376         }
2377
2378         /* Initialize the MAC Core */
2379         priv->hw->mac->core_init(priv->hw, dev->mtu);
2380
2381         /* Initialize MTL*/
2382         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2383                 stmmac_mtl_configuration(priv);
2384
2385         ret = priv->hw->mac->rx_ipc(priv->hw);
2386         if (!ret) {
2387                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2388                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2389                 priv->hw->rx_csum = 0;
2390         }
2391
2392         /* Enable the MAC Rx/Tx */
2393         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2394                 stmmac_dwmac4_set_mac(priv->ioaddr, true);
2395         else
2396                 stmmac_set_mac(priv->ioaddr, true);
2397
2398         stmmac_mmc_setup(priv);
2399
2400         if (init_ptp) {
2401                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2402                 if (ret < 0)
2403                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2404
2405                 ret = stmmac_init_ptp(priv);
2406                 if (ret == -EOPNOTSUPP)
2407                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2408                 else if (ret)
2409                         netdev_warn(priv->dev, "PTP init failed\n");
2410         }
2411
2412 #ifdef CONFIG_DEBUG_FS
2413         ret = stmmac_init_fs(dev);
2414         if (ret < 0)
2415                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2416                             __func__);
2417 #endif
2418         /* Start the ball rolling... */
2419         stmmac_start_all_dma(priv);
2420
2421         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2422
2423         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2424                 priv->rx_riwt = MAX_DMA_RIWT;
2425                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2426         }
2427
2428         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2429                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2430
2431         /* set TX and RX rings length */
2432         stmmac_set_rings_length(priv);
2433
2434         /* Enable TSO */
2435         if (priv->tso) {
2436                 for (chan = 0; chan < tx_cnt; chan++)
2437                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2438         }
2439
2440         return 0;
2441 }
2442
2443 static void stmmac_hw_teardown(struct net_device *dev)
2444 {
2445         struct stmmac_priv *priv = netdev_priv(dev);
2446
2447         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2448 }
2449
2450 /**
2451  *  stmmac_open - open entry point of the driver
2452  *  @dev : pointer to the device structure.
2453  *  Description:
2454  *  This function is the open entry point of the driver.
2455  *  Return value:
2456  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2457  *  file on failure.
2458  */
2459 static int stmmac_open(struct net_device *dev)
2460 {
2461         struct stmmac_priv *priv = netdev_priv(dev);
2462         int ret;
2463
2464         stmmac_check_ether_addr(priv);
2465
2466         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2467             priv->hw->pcs != STMMAC_PCS_TBI &&
2468             priv->hw->pcs != STMMAC_PCS_RTBI) {
2469                 ret = stmmac_init_phy(dev);
2470                 if (ret) {
2471                         netdev_err(priv->dev,
2472                                    "%s: Cannot attach to PHY (error: %d)\n",
2473                                    __func__, ret);
2474                         return ret;
2475                 }
2476         }
2477
2478         /* Extra statistics */
2479         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2480         priv->xstats.threshold = tc;
2481
2482         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2483
2484         ret = stmmac_hw_setup(dev, true);
2485         if (ret < 0) {
2486                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2487                 goto init_error;
2488         }
2489
2490         stmmac_init_tx_coalesce(priv);
2491
2492         if (dev->phydev)
2493                 phy_start(dev->phydev);
2494
2495         /* Request the IRQ lines */
2496         ret = request_irq(dev->irq, stmmac_interrupt,
2497                           IRQF_SHARED, dev->name, dev);
2498         if (unlikely(ret < 0)) {
2499                 netdev_err(priv->dev,
2500                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2501                            __func__, dev->irq, ret);
2502                 goto irq_error;
2503         }
2504
2505         /* Request the Wake IRQ in case of another line is used for WoL */
2506         if (priv->wol_irq != dev->irq) {
2507                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2508                                   IRQF_SHARED, dev->name, dev);
2509                 if (unlikely(ret < 0)) {
2510                         netdev_err(priv->dev,
2511                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2512                                    __func__, priv->wol_irq, ret);
2513                         goto wolirq_error;
2514                 }
2515         }
2516
2517         /* Request the IRQ lines */
2518         if (priv->lpi_irq > 0) {
2519                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2520                                   dev->name, dev);
2521                 if (unlikely(ret < 0)) {
2522                         netdev_err(priv->dev,
2523                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2524                                    __func__, priv->lpi_irq, ret);
2525                         goto lpiirq_error;
2526                 }
2527         }
2528
2529         stmmac_enable_all_queues(priv);
2530         stmmac_start_all_queues(priv);
2531
2532         return 0;
2533
2534 lpiirq_error:
2535         if (priv->wol_irq != dev->irq)
2536                 free_irq(priv->wol_irq, dev);
2537 wolirq_error:
2538         free_irq(dev->irq, dev);
2539 irq_error:
2540         if (dev->phydev)
2541                 phy_stop(dev->phydev);
2542
2543         del_timer_sync(&priv->txtimer);
2544         stmmac_hw_teardown(dev);
2545 init_error:
2546         free_dma_desc_resources(priv);
2547
2548         if (dev->phydev)
2549                 phy_disconnect(dev->phydev);
2550
2551         return ret;
2552 }
2553
2554 /**
2555  *  stmmac_release - close entry point of the driver
2556  *  @dev : device pointer.
2557  *  Description:
2558  *  This is the stop entry point of the driver.
2559  */
2560 static int stmmac_release(struct net_device *dev)
2561 {
2562         struct stmmac_priv *priv = netdev_priv(dev);
2563
2564         if (priv->eee_enabled)
2565                 del_timer_sync(&priv->eee_ctrl_timer);
2566
2567         /* Stop and disconnect the PHY */
2568         if (dev->phydev) {
2569                 phy_stop(dev->phydev);
2570                 phy_disconnect(dev->phydev);
2571         }
2572
2573         stmmac_stop_all_queues(priv);
2574
2575         stmmac_disable_all_queues(priv);
2576
2577         del_timer_sync(&priv->txtimer);
2578
2579         /* Free the IRQ lines */
2580         free_irq(dev->irq, dev);
2581         if (priv->wol_irq != dev->irq)
2582                 free_irq(priv->wol_irq, dev);
2583         if (priv->lpi_irq > 0)
2584                 free_irq(priv->lpi_irq, dev);
2585
2586         /* Stop TX/RX DMA and clear the descriptors */
2587         stmmac_stop_all_dma(priv);
2588
2589         /* Release and free the Rx/Tx resources */
2590         free_dma_desc_resources(priv);
2591
2592         /* Disable the MAC Rx/Tx */
2593         stmmac_set_mac(priv->ioaddr, false);
2594
2595         netif_carrier_off(dev);
2596
2597 #ifdef CONFIG_DEBUG_FS
2598         stmmac_exit_fs(dev);
2599 #endif
2600
2601         stmmac_release_ptp(priv);
2602
2603         return 0;
2604 }
2605
2606 /**
2607  *  stmmac_tso_allocator - close entry point of the driver
2608  *  @priv: driver private structure
2609  *  @des: buffer start address
2610  *  @total_len: total length to fill in descriptors
2611  *  @last_segmant: condition for the last descriptor
2612  *  @queue: TX queue index
2613  *  Description:
2614  *  This function fills descriptor and request new descriptors according to
2615  *  buffer length to fill
2616  */
2617 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2618                                  int total_len, bool last_segment, u32 queue)
2619 {
2620         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2621         struct dma_desc *desc;
2622         u32 buff_size;
2623         int tmp_len;
2624
2625         tmp_len = total_len;
2626
2627         while (tmp_len > 0) {
2628                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2629                 desc = tx_q->dma_tx + tx_q->cur_tx;
2630
2631                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2632                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2633                             TSO_MAX_BUFF_SIZE : tmp_len;
2634
2635                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2636                         0, 1,
2637                         (last_segment) && (buff_size < TSO_MAX_BUFF_SIZE),
2638                         0, 0);
2639
2640                 tmp_len -= TSO_MAX_BUFF_SIZE;
2641         }
2642 }
2643
2644 /**
2645  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2646  *  @skb : the socket buffer
2647  *  @dev : device pointer
2648  *  Description: this is the transmit function that is called on TSO frames
2649  *  (support available on GMAC4 and newer chips).
2650  *  Diagram below show the ring programming in case of TSO frames:
2651  *
2652  *  First Descriptor
2653  *   --------
2654  *   | DES0 |---> buffer1 = L2/L3/L4 header
2655  *   | DES1 |---> TCP Payload (can continue on next descr...)
2656  *   | DES2 |---> buffer 1 and 2 len
2657  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2658  *   --------
2659  *      |
2660  *     ...
2661  *      |
2662  *   --------
2663  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2664  *   | DES1 | --|
2665  *   | DES2 | --> buffer 1 and 2 len
2666  *   | DES3 |
2667  *   --------
2668  *
2669  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2670  */
2671 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2672 {
2673         struct dma_desc *desc, *first, *mss_desc = NULL;
2674         struct stmmac_priv *priv = netdev_priv(dev);
2675         u32 queue = skb_get_queue_mapping(skb);
2676         int nfrags = skb_shinfo(skb)->nr_frags;
2677         unsigned int first_entry, des;
2678         struct stmmac_tx_queue *tx_q;
2679         int tmp_pay_len = 0;
2680         u32 pay_len, mss;
2681         u8 proto_hdr_len;
2682         int i;
2683
2684         tx_q = &priv->tx_queue[queue];
2685
2686         /* Compute header lengths */
2687         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2688
2689         /* Desc availability based on threshold should be enough safe */
2690         if (unlikely(stmmac_tx_avail(priv, queue) <
2691                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2692                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2693                         netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2694                         /* This is a hard error, log it. */
2695                         netdev_err(priv->dev,
2696                                    "%s: Tx Ring full when queue awake\n",
2697                                    __func__);
2698                 }
2699                 return NETDEV_TX_BUSY;
2700         }
2701
2702         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2703
2704         mss = skb_shinfo(skb)->gso_size;
2705
2706         /* set new MSS value if needed */
2707         if (mss != priv->mss) {
2708                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2709                 priv->hw->desc->set_mss(mss_desc, mss);
2710                 priv->mss = mss;
2711                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2712         }
2713
2714         if (netif_msg_tx_queued(priv)) {
2715                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2716                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2717                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2718                         skb->data_len);
2719         }
2720
2721         first_entry = tx_q->cur_tx;
2722
2723         desc = tx_q->dma_tx + first_entry;
2724         first = desc;
2725
2726         /* first descriptor: fill Headers on Buf1 */
2727         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2728                              DMA_TO_DEVICE);
2729         if (dma_mapping_error(priv->device, des))
2730                 goto dma_map_err;
2731
2732         tx_q->tx_skbuff_dma[first_entry].buf = des;
2733         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2734         tx_q->tx_skbuff[first_entry] = skb;
2735
2736         first->des0 = cpu_to_le32(des);
2737
2738         /* Fill start of payload in buff2 of first descriptor */
2739         if (pay_len)
2740                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2741
2742         /* If needed take extra descriptors to fill the remaining payload */
2743         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2744
2745         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2746
2747         /* Prepare fragments */
2748         for (i = 0; i < nfrags; i++) {
2749                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2750
2751                 des = skb_frag_dma_map(priv->device, frag, 0,
2752                                        skb_frag_size(frag),
2753                                        DMA_TO_DEVICE);
2754                 if (dma_mapping_error(priv->device, des))
2755                         goto dma_map_err;
2756
2757                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2758                                      (i == nfrags - 1), queue);
2759
2760                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2761                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2762                 tx_q->tx_skbuff[tx_q->cur_tx] = NULL;
2763                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2764         }
2765
2766         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2767
2768         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2769
2770         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2771                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2772                           __func__);
2773                 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2774         }
2775
2776         dev->stats.tx_bytes += skb->len;
2777         priv->xstats.tx_tso_frames++;
2778         priv->xstats.tx_tso_nfrags += nfrags;
2779
2780         /* Manage tx mitigation */
2781         priv->tx_count_frames += nfrags + 1;
2782         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2783                 mod_timer(&priv->txtimer,
2784                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2785         } else {
2786                 priv->tx_count_frames = 0;
2787                 priv->hw->desc->set_tx_ic(desc);
2788                 priv->xstats.tx_set_ic_bit++;
2789         }
2790
2791         if (!priv->hwts_tx_en)
2792                 skb_tx_timestamp(skb);
2793
2794         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2795                      priv->hwts_tx_en)) {
2796                 /* declare that device is doing timestamping */
2797                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2798                 priv->hw->desc->enable_tx_timestamp(first);
2799         }
2800
2801         /* Complete the first descriptor before granting the DMA */
2802         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2803                         proto_hdr_len,
2804                         pay_len,
2805                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2806                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2807
2808         /* If context desc is used to change MSS */
2809         if (mss_desc)
2810                 priv->hw->desc->set_tx_owner(mss_desc);
2811
2812         /* The own bit must be the latest setting done when prepare the
2813          * descriptor and then barrier is needed to make sure that
2814          * all is coherent before granting the DMA engine.
2815          */
2816         dma_wmb();
2817
2818         if (netif_msg_pktdata(priv)) {
2819                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2820                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2821                         tx_q->cur_tx, first, nfrags);
2822
2823                 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
2824                                              0);
2825
2826                 pr_info(">>> frame to be transmitted: ");
2827                 print_pkt(skb->data, skb_headlen(skb));
2828         }
2829
2830         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2831
2832         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
2833                                        queue);
2834
2835         return NETDEV_TX_OK;
2836
2837 dma_map_err:
2838         dev_err(priv->device, "Tx dma map failed\n");
2839         dev_kfree_skb(skb);
2840         priv->dev->stats.tx_dropped++;
2841         return NETDEV_TX_OK;
2842 }
2843
2844 /**
2845  *  stmmac_xmit - Tx entry point of the driver
2846  *  @skb : the socket buffer
2847  *  @dev : device pointer
2848  *  Description : this is the tx entry point of the driver.
2849  *  It programs the chain or the ring and supports oversized frames
2850  *  and SG feature.
2851  */
2852 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2853 {
2854         struct stmmac_priv *priv = netdev_priv(dev);
2855         unsigned int nopaged_len = skb_headlen(skb);
2856         int i, csum_insertion = 0, is_jumbo = 0;
2857         u32 queue = skb_get_queue_mapping(skb);
2858         int nfrags = skb_shinfo(skb)->nr_frags;
2859         unsigned int entry, first_entry;
2860         struct dma_desc *desc, *first;
2861         struct stmmac_tx_queue *tx_q;
2862         unsigned int enh_desc;
2863         unsigned int des;
2864
2865         tx_q = &priv->tx_queue[queue];
2866
2867         /* Manage oversized TCP frames for GMAC4 device */
2868         if (skb_is_gso(skb) && priv->tso) {
2869                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2870                         return stmmac_tso_xmit(skb, dev);
2871         }
2872
2873         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
2874                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2875                         netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2876                         /* This is a hard error, log it. */
2877                         netdev_err(priv->dev,
2878                                    "%s: Tx Ring full when queue awake\n",
2879                                    __func__);
2880                 }
2881                 return NETDEV_TX_BUSY;
2882         }
2883
2884         if (priv->tx_path_in_lpi_mode)
2885                 stmmac_disable_eee_mode(priv);
2886
2887         entry = tx_q->cur_tx;
2888         first_entry = entry;
2889
2890         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
2891
2892         if (likely(priv->extend_desc))
2893                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2894         else
2895                 desc = tx_q->dma_tx + entry;
2896
2897         first = desc;
2898
2899         tx_q->tx_skbuff[first_entry] = skb;
2900
2901         enh_desc = priv->plat->enh_desc;
2902         /* To program the descriptors according to the size of the frame */
2903         if (enh_desc)
2904                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
2905
2906         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
2907                                          DWMAC_CORE_4_00)) {
2908                 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
2909                 if (unlikely(entry < 0))
2910                         goto dma_map_err;
2911         }
2912
2913         for (i = 0; i < nfrags; i++) {
2914                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2915                 int len = skb_frag_size(frag);
2916                 bool last_segment = (i == (nfrags - 1));
2917
2918                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
2919
2920                 if (likely(priv->extend_desc))
2921                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2922                 else
2923                         desc = tx_q->dma_tx + entry;
2924
2925                 des = skb_frag_dma_map(priv->device, frag, 0, len,
2926                                        DMA_TO_DEVICE);
2927                 if (dma_mapping_error(priv->device, des))
2928                         goto dma_map_err; /* should reuse desc w/o issues */
2929
2930                 tx_q->tx_skbuff[entry] = NULL;
2931
2932                 tx_q->tx_skbuff_dma[entry].buf = des;
2933                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
2934                         desc->des0 = cpu_to_le32(des);
2935                 else
2936                         desc->des2 = cpu_to_le32(des);
2937
2938                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
2939                 tx_q->tx_skbuff_dma[entry].len = len;
2940                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
2941
2942                 /* Prepare the descriptor and set the own bit too */
2943                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
2944                                                 priv->mode, 1, last_segment);
2945         }
2946
2947         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
2948
2949         tx_q->cur_tx = entry;
2950
2951         if (netif_msg_pktdata(priv)) {
2952                 void *tx_head;
2953
2954                 netdev_dbg(priv->dev,
2955                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
2956                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2957                            entry, first, nfrags);
2958
2959                 if (priv->extend_desc)
2960                         tx_head = (void *)tx_q->dma_etx;
2961                 else
2962                         tx_head = (void *)tx_q->dma_tx;
2963
2964                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
2965
2966                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
2967                 print_pkt(skb->data, skb->len);
2968         }
2969
2970         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2971                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2972                           __func__);
2973                 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue));
2974         }
2975
2976         dev->stats.tx_bytes += skb->len;
2977
2978         /* According to the coalesce parameter the IC bit for the latest
2979          * segment is reset and the timer re-started to clean the tx status.
2980          * This approach takes care about the fragments: desc is the first
2981          * element in case of no SG.
2982          */
2983         priv->tx_count_frames += nfrags + 1;
2984         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2985                 mod_timer(&priv->txtimer,
2986                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2987         } else {
2988                 priv->tx_count_frames = 0;
2989                 priv->hw->desc->set_tx_ic(desc);
2990                 priv->xstats.tx_set_ic_bit++;
2991         }
2992
2993         if (!priv->hwts_tx_en)
2994                 skb_tx_timestamp(skb);
2995
2996         /* Ready to fill the first descriptor and set the OWN bit w/o any
2997          * problems because all the descriptors are actually ready to be
2998          * passed to the DMA engine.
2999          */
3000         if (likely(!is_jumbo)) {
3001                 bool last_segment = (nfrags == 0);
3002
3003                 des = dma_map_single(priv->device, skb->data,
3004                                      nopaged_len, DMA_TO_DEVICE);
3005                 if (dma_mapping_error(priv->device, des))
3006                         goto dma_map_err;
3007
3008                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3009                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3010                         first->des0 = cpu_to_le32(des);
3011                 else
3012                         first->des2 = cpu_to_le32(des);
3013
3014                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3015                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3016
3017                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3018                              priv->hwts_tx_en)) {
3019                         /* declare that device is doing timestamping */
3020                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3021                         priv->hw->desc->enable_tx_timestamp(first);
3022                 }
3023
3024                 /* Prepare the first descriptor setting the OWN bit too */
3025                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
3026                                                 csum_insertion, priv->mode, 1,
3027                                                 last_segment);
3028
3029                 /* The own bit must be the latest setting done when prepare the
3030                  * descriptor and then barrier is needed to make sure that
3031                  * all is coherent before granting the DMA engine.
3032                  */
3033                 dma_wmb();
3034         }
3035
3036         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3037
3038         if (priv->synopsys_id < DWMAC_CORE_4_00)
3039                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
3040         else
3041                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
3042                                                queue);
3043
3044         return NETDEV_TX_OK;
3045
3046 dma_map_err:
3047         netdev_err(priv->dev, "Tx DMA map failed\n");
3048         dev_kfree_skb(skb);
3049         priv->dev->stats.tx_dropped++;
3050         return NETDEV_TX_OK;
3051 }
3052
3053 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3054 {
3055         struct ethhdr *ehdr;
3056         u16 vlanid;
3057
3058         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
3059             NETIF_F_HW_VLAN_CTAG_RX &&
3060             !__vlan_get_tag(skb, &vlanid)) {
3061                 /* pop the vlan tag */
3062                 ehdr = (struct ethhdr *)skb->data;
3063                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
3064                 skb_pull(skb, VLAN_HLEN);
3065                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
3066         }
3067 }
3068
3069
3070 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3071 {
3072         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3073                 return 0;
3074
3075         return 1;
3076 }
3077
3078 /**
3079  * stmmac_rx_refill - refill used skb preallocated buffers
3080  * @priv: driver private structure
3081  * @queue: RX queue index
3082  * Description : this is to reallocate the skb for the reception process
3083  * that is based on zero-copy.
3084  */
3085 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3086 {
3087         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3088         int dirty = stmmac_rx_dirty(priv, queue);
3089         unsigned int entry = rx_q->dirty_rx;
3090         int bfsize = priv->dma_buf_sz;
3091
3092         while (dirty-- > 0) {
3093                 struct dma_desc *p;
3094
3095                 if (priv->extend_desc)
3096                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3097                 else
3098                         p = rx_q->dma_rx + entry;
3099
3100                 if (!rx_q->rx_skbuff[entry]) {
3101                         struct sk_buff *skb;
3102
3103                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3104                         if (unlikely(!skb)) {
3105                                 /* so for a while no zero-copy! */
3106                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3107                                 if (unlikely(net_ratelimit()))
3108                                         dev_err(priv->device,
3109                                                 "fail to alloc skb entry %d\n",
3110                                                 entry);
3111                                 break;
3112                         }
3113
3114                         rx_q->rx_skbuff[entry] = skb;
3115                         rx_q->rx_skbuff_dma[entry] =
3116                             dma_map_single(priv->device, skb->data, bfsize,
3117                                            DMA_FROM_DEVICE);
3118                         if (dma_mapping_error(priv->device,
3119                                               rx_q->rx_skbuff_dma[entry])) {
3120                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3121                                 dev_kfree_skb(skb);
3122                                 break;
3123                         }
3124
3125                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3126                                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3127                                 p->des1 = 0;
3128                         } else {
3129                                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3130                         }
3131                         if (priv->hw->mode->refill_desc3)
3132                                 priv->hw->mode->refill_desc3(rx_q, p);
3133
3134                         if (rx_q->rx_zeroc_thresh > 0)
3135                                 rx_q->rx_zeroc_thresh--;
3136
3137                         netif_dbg(priv, rx_status, priv->dev,
3138                                   "refill entry #%d\n", entry);
3139                 }
3140                 dma_wmb();
3141
3142                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3143                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
3144                 else
3145                         priv->hw->desc->set_rx_owner(p);
3146
3147                 dma_wmb();
3148
3149                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3150         }
3151         rx_q->dirty_rx = entry;
3152 }
3153
3154 /**
3155  * stmmac_rx - manage the receive process
3156  * @priv: driver private structure
3157  * @limit: napi bugget.
3158  * Description :  this the function called by the napi poll method.
3159  * It gets all the frames inside the ring.
3160  */
3161 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3162 {
3163         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3164         unsigned int entry = rx_q->cur_rx;
3165         int coe = priv->hw->rx_csum;
3166         unsigned int next_entry;
3167         unsigned int count = 0;
3168
3169         if (netif_msg_rx_status(priv)) {
3170                 void *rx_head;
3171
3172                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3173                 if (priv->extend_desc)
3174                         rx_head = (void *)rx_q->dma_erx;
3175                 else
3176                         rx_head = (void *)rx_q->dma_rx;
3177
3178                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
3179         }
3180         while (count < limit) {
3181                 int status;
3182                 struct dma_desc *p;
3183                 struct dma_desc *np;
3184
3185                 if (priv->extend_desc)
3186                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3187                 else
3188                         p = rx_q->dma_rx + entry;
3189
3190                 /* read the status of the incoming frame */
3191                 status = priv->hw->desc->rx_status(&priv->dev->stats,
3192                                                    &priv->xstats, p);
3193                 /* check if managed by the DMA otherwise go ahead */
3194                 if (unlikely(status & dma_own))
3195                         break;
3196
3197                 count++;
3198
3199                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3200                 next_entry = rx_q->cur_rx;
3201
3202                 if (priv->extend_desc)
3203                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3204                 else
3205                         np = rx_q->dma_rx + next_entry;
3206
3207                 prefetch(np);
3208
3209                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
3210                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
3211                                                            &priv->xstats,
3212                                                            rx_q->dma_erx +
3213                                                            entry);
3214                 if (unlikely(status == discard_frame)) {
3215                         priv->dev->stats.rx_errors++;
3216                         if (priv->hwts_rx_en && !priv->extend_desc) {
3217                                 /* DESC2 & DESC3 will be overwritten by device
3218                                  * with timestamp value, hence reinitialize
3219                                  * them in stmmac_rx_refill() function so that
3220                                  * device can reuse it.
3221                                  */
3222                                 rx_q->rx_skbuff[entry] = NULL;
3223                                 dma_unmap_single(priv->device,
3224                                                  rx_q->rx_skbuff_dma[entry],
3225                                                  priv->dma_buf_sz,
3226                                                  DMA_FROM_DEVICE);
3227                         }
3228                 } else {
3229                         struct sk_buff *skb;
3230                         int frame_len;
3231                         unsigned int des;
3232
3233                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3234                                 des = le32_to_cpu(p->des0);
3235                         else
3236                                 des = le32_to_cpu(p->des2);
3237
3238                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
3239
3240                         /*  If frame length is greater than skb buffer size
3241                          *  (preallocated during init) then the packet is
3242                          *  ignored
3243                          */
3244                         if (frame_len > priv->dma_buf_sz) {
3245                                 netdev_err(priv->dev,
3246                                            "len %d larger than size (%d)\n",
3247                                            frame_len, priv->dma_buf_sz);
3248                                 priv->dev->stats.rx_length_errors++;
3249                                 break;
3250                         }
3251
3252                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3253                          * Type frames (LLC/LLC-SNAP)
3254                          */
3255                         if (unlikely(status != llc_snap))
3256                                 frame_len -= ETH_FCS_LEN;
3257
3258                         if (netif_msg_rx_status(priv)) {
3259                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3260                                            p, entry, des);
3261                                 if (frame_len > ETH_FRAME_LEN)
3262                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3263                                                    frame_len, status);
3264                         }
3265
3266                         /* The zero-copy is always used for all the sizes
3267                          * in case of GMAC4 because it needs
3268                          * to refill the used descriptors, always.
3269                          */
3270                         if (unlikely(!priv->plat->has_gmac4 &&
3271                                      ((frame_len < priv->rx_copybreak) ||
3272                                      stmmac_rx_threshold_count(rx_q)))) {
3273                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3274                                                                 frame_len);
3275                                 if (unlikely(!skb)) {
3276                                         if (net_ratelimit())
3277                                                 dev_warn(priv->device,
3278                                                          "packet dropped\n");
3279                                         priv->dev->stats.rx_dropped++;
3280                                         break;
3281                                 }
3282
3283                                 dma_sync_single_for_cpu(priv->device,
3284                                                         rx_q->rx_skbuff_dma
3285                                                         [entry], frame_len,
3286                                                         DMA_FROM_DEVICE);
3287                                 skb_copy_to_linear_data(skb,
3288                                                         rx_q->
3289                                                         rx_skbuff[entry]->data,
3290                                                         frame_len);
3291
3292                                 skb_put(skb, frame_len);
3293                                 dma_sync_single_for_device(priv->device,
3294                                                            rx_q->rx_skbuff_dma
3295                                                            [entry], frame_len,
3296                                                            DMA_FROM_DEVICE);
3297                         } else {
3298                                 skb = rx_q->rx_skbuff[entry];
3299                                 if (unlikely(!skb)) {
3300                                         netdev_err(priv->dev,
3301                                                    "%s: Inconsistent Rx chain\n",
3302                                                    priv->dev->name);
3303                                         priv->dev->stats.rx_dropped++;
3304                                         break;
3305                                 }
3306                                 prefetch(skb->data - NET_IP_ALIGN);
3307                                 rx_q->rx_skbuff[entry] = NULL;
3308                                 rx_q->rx_zeroc_thresh++;
3309
3310                                 skb_put(skb, frame_len);
3311                                 dma_unmap_single(priv->device,
3312                                                  rx_q->rx_skbuff_dma[entry],
3313                                                  priv->dma_buf_sz,
3314                                                  DMA_FROM_DEVICE);
3315                         }
3316
3317                         if (netif_msg_pktdata(priv)) {
3318                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3319                                            frame_len);
3320                                 print_pkt(skb->data, frame_len);
3321                         }
3322
3323                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3324
3325                         stmmac_rx_vlan(priv->dev, skb);
3326
3327                         skb->protocol = eth_type_trans(skb, priv->dev);
3328
3329                         if (unlikely(!coe))
3330                                 skb_checksum_none_assert(skb);
3331                         else
3332                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3333
3334                         napi_gro_receive(&rx_q->napi, skb);
3335
3336                         priv->dev->stats.rx_packets++;
3337                         priv->dev->stats.rx_bytes += frame_len;
3338                 }
3339                 entry = next_entry;
3340         }
3341
3342         stmmac_rx_refill(priv, queue);
3343
3344         priv->xstats.rx_pkt_n += count;
3345
3346         return count;
3347 }
3348
3349 /**
3350  *  stmmac_poll - stmmac poll method (NAPI)
3351  *  @napi : pointer to the napi structure.
3352  *  @budget : maximum number of packets that the current CPU can receive from
3353  *            all interfaces.
3354  *  Description :
3355  *  To look at the incoming frames and clear the tx resources.
3356  */
3357 static int stmmac_poll(struct napi_struct *napi, int budget)
3358 {
3359         struct stmmac_rx_queue *rx_q =
3360                 container_of(napi, struct stmmac_rx_queue, napi);
3361         struct stmmac_priv *priv = rx_q->priv_data;
3362         u32 tx_count = priv->dma_cap.number_tx_queues;
3363         u32 chan = rx_q->queue_index;
3364         u32 work_done = 0;
3365         u32 queue = 0;
3366
3367         priv->xstats.napi_poll++;
3368         /* check all the queues */
3369         for (queue = 0; queue < tx_count; queue++)
3370                 stmmac_tx_clean(priv, queue);
3371
3372         /* Process RX packets from this queue */
3373         work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3374
3375         if (work_done < budget) {
3376                 napi_complete_done(napi, work_done);
3377                 stmmac_enable_dma_irq(priv, chan);
3378         }
3379         return work_done;
3380 }
3381
3382 /**
3383  *  stmmac_tx_timeout
3384  *  @dev : Pointer to net device structure
3385  *  Description: this function is called when a packet transmission fails to
3386  *   complete within a reasonable time. The driver will mark the error in the
3387  *   netdev structure and arrange for the device to be reset to a sane state
3388  *   in order to transmit a new packet.
3389  */
3390 static void stmmac_tx_timeout(struct net_device *dev)
3391 {
3392         struct stmmac_priv *priv = netdev_priv(dev);
3393         u32 tx_count = priv->plat->tx_queues_to_use;
3394         u32 chan;
3395
3396         /* Clear Tx resources and restart transmitting again */
3397         for (chan = 0; chan < tx_count; chan++)
3398                 stmmac_tx_err(priv, chan);
3399 }
3400
3401 /**
3402  *  stmmac_set_rx_mode - entry point for multicast addressing
3403  *  @dev : pointer to the device structure
3404  *  Description:
3405  *  This function is a driver entry point which gets called by the kernel
3406  *  whenever multicast addresses must be enabled/disabled.
3407  *  Return value:
3408  *  void.
3409  */
3410 static void stmmac_set_rx_mode(struct net_device *dev)
3411 {
3412         struct stmmac_priv *priv = netdev_priv(dev);
3413
3414         priv->hw->mac->set_filter(priv->hw, dev);
3415 }
3416
3417 /**
3418  *  stmmac_change_mtu - entry point to change MTU size for the device.
3419  *  @dev : device pointer.
3420  *  @new_mtu : the new MTU size for the device.
3421  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3422  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3423  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3424  *  Return value:
3425  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3426  *  file on failure.
3427  */
3428 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3429 {
3430         struct stmmac_priv *priv = netdev_priv(dev);
3431
3432         if (netif_running(dev)) {
3433                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3434                 return -EBUSY;
3435         }
3436
3437         dev->mtu = new_mtu;
3438
3439         netdev_update_features(dev);
3440
3441         return 0;
3442 }
3443
3444 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3445                                              netdev_features_t features)
3446 {
3447         struct stmmac_priv *priv = netdev_priv(dev);
3448
3449         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3450                 features &= ~NETIF_F_RXCSUM;
3451
3452         if (!priv->plat->tx_coe)
3453                 features &= ~NETIF_F_CSUM_MASK;
3454
3455         /* Some GMAC devices have a bugged Jumbo frame support that
3456          * needs to have the Tx COE disabled for oversized frames
3457          * (due to limited buffer sizes). In this case we disable
3458          * the TX csum insertion in the TDES and not use SF.
3459          */
3460         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3461                 features &= ~NETIF_F_CSUM_MASK;
3462
3463         /* Disable tso if asked by ethtool */
3464         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3465                 if (features & NETIF_F_TSO)
3466                         priv->tso = true;
3467                 else
3468                         priv->tso = false;
3469         }
3470
3471         return features;
3472 }
3473
3474 static int stmmac_set_features(struct net_device *netdev,
3475                                netdev_features_t features)
3476 {
3477         struct stmmac_priv *priv = netdev_priv(netdev);
3478
3479         /* Keep the COE Type in case of csum is supporting */
3480         if (features & NETIF_F_RXCSUM)
3481                 priv->hw->rx_csum = priv->plat->rx_coe;
3482         else
3483                 priv->hw->rx_csum = 0;
3484         /* No check needed because rx_coe has been set before and it will be
3485          * fixed in case of issue.
3486          */
3487         priv->hw->mac->rx_ipc(priv->hw);
3488
3489         return 0;
3490 }
3491
3492 /**
3493  *  stmmac_interrupt - main ISR
3494  *  @irq: interrupt number.
3495  *  @dev_id: to pass the net device pointer.
3496  *  Description: this is the main driver interrupt service routine.
3497  *  It can call:
3498  *  o DMA service routine (to manage incoming frame reception and transmission
3499  *    status)
3500  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3501  *    interrupts.
3502  */
3503 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3504 {
3505         struct net_device *dev = (struct net_device *)dev_id;
3506         struct stmmac_priv *priv = netdev_priv(dev);
3507         u32 rx_cnt = priv->plat->rx_queues_to_use;
3508         u32 tx_cnt = priv->plat->tx_queues_to_use;
3509         u32 queues_count;
3510         u32 queue;
3511
3512         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3513
3514         if (priv->irq_wake)
3515                 pm_wakeup_event(priv->device, 0);
3516
3517         if (unlikely(!dev)) {
3518                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3519                 return IRQ_NONE;
3520         }
3521
3522         /* To handle GMAC own interrupts */
3523         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3524                 int status = priv->hw->mac->host_irq_status(priv->hw,
3525                                                             &priv->xstats);
3526
3527                 if (unlikely(status)) {
3528                         /* For LPI we need to save the tx status */
3529                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3530                                 priv->tx_path_in_lpi_mode = true;
3531                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3532                                 priv->tx_path_in_lpi_mode = false;
3533                 }
3534
3535                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3536                         for (queue = 0; queue < queues_count; queue++) {
3537                                 struct stmmac_rx_queue *rx_q =
3538                                 &priv->rx_queue[queue];
3539
3540                                 status |=
3541                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3542                                                                    queue);
3543
3544                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3545                                     priv->hw->dma->set_rx_tail_ptr)
3546                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3547                                                                 rx_q->rx_tail_addr,
3548                                                                 queue);
3549                         }
3550                 }
3551
3552                 /* PCS link status */
3553                 if (priv->hw->pcs) {
3554                         if (priv->xstats.pcs_link)
3555                                 netif_carrier_on(dev);
3556                         else
3557                                 netif_carrier_off(dev);
3558                 }
3559         }
3560
3561         /* To handle DMA interrupts */
3562         stmmac_dma_interrupt(priv);
3563
3564         return IRQ_HANDLED;
3565 }
3566
3567 #ifdef CONFIG_NET_POLL_CONTROLLER
3568 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3569  * to allow network I/O with interrupts disabled.
3570  */
3571 static void stmmac_poll_controller(struct net_device *dev)
3572 {
3573         disable_irq(dev->irq);
3574         stmmac_interrupt(dev->irq, dev);
3575         enable_irq(dev->irq);
3576 }
3577 #endif
3578
3579 /**
3580  *  stmmac_ioctl - Entry point for the Ioctl
3581  *  @dev: Device pointer.
3582  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3583  *  a proprietary structure used to pass information to the driver.
3584  *  @cmd: IOCTL command
3585  *  Description:
3586  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3587  */
3588 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3589 {
3590         int ret = -EOPNOTSUPP;
3591
3592         if (!netif_running(dev))
3593                 return -EINVAL;
3594
3595         switch (cmd) {
3596         case SIOCGMIIPHY:
3597         case SIOCGMIIREG:
3598         case SIOCSMIIREG:
3599                 if (!dev->phydev)
3600                         return -EINVAL;
3601                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3602                 break;
3603         case SIOCSHWTSTAMP:
3604                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3605                 break;
3606         default:
3607                 break;
3608         }
3609
3610         return ret;
3611 }
3612
3613 #ifdef CONFIG_DEBUG_FS
3614 static struct dentry *stmmac_fs_dir;
3615
3616 static void sysfs_display_ring(void *head, int size, int extend_desc,
3617                                struct seq_file *seq)
3618 {
3619         int i;
3620         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3621         struct dma_desc *p = (struct dma_desc *)head;
3622
3623         for (i = 0; i < size; i++) {
3624                 if (extend_desc) {
3625                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3626                                    i, (unsigned int)virt_to_phys(ep),
3627                                    le32_to_cpu(ep->basic.des0),
3628                                    le32_to_cpu(ep->basic.des1),
3629                                    le32_to_cpu(ep->basic.des2),
3630                                    le32_to_cpu(ep->basic.des3));
3631                         ep++;
3632                 } else {
3633                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3634                                    i, (unsigned int)virt_to_phys(ep),
3635                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3636                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3637                         p++;
3638                 }
3639                 seq_printf(seq, "\n");
3640         }
3641 }
3642
3643 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3644 {
3645         struct net_device *dev = seq->private;
3646         struct stmmac_priv *priv = netdev_priv(dev);
3647         u32 rx_count = priv->plat->rx_queues_to_use;
3648         u32 tx_count = priv->plat->tx_queues_to_use;
3649         u32 queue;
3650
3651         for (queue = 0; queue < rx_count; queue++) {
3652                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3653
3654                 seq_printf(seq, "RX Queue %d:\n", queue);
3655
3656                 if (priv->extend_desc) {
3657                         seq_printf(seq, "Extended descriptor ring:\n");
3658                         sysfs_display_ring((void *)rx_q->dma_erx,
3659                                            DMA_RX_SIZE, 1, seq);
3660                 } else {
3661                         seq_printf(seq, "Descriptor ring:\n");
3662                         sysfs_display_ring((void *)rx_q->dma_rx,
3663                                            DMA_RX_SIZE, 0, seq);
3664                 }
3665         }
3666
3667         for (queue = 0; queue < tx_count; queue++) {
3668                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3669
3670                 seq_printf(seq, "TX Queue %d:\n", queue);
3671
3672                 if (priv->extend_desc) {
3673                         seq_printf(seq, "Extended descriptor ring:\n");
3674                         sysfs_display_ring((void *)tx_q->dma_etx,
3675                                            DMA_TX_SIZE, 1, seq);
3676                 } else {
3677                         seq_printf(seq, "Descriptor ring:\n");
3678                         sysfs_display_ring((void *)tx_q->dma_tx,
3679                                            DMA_TX_SIZE, 0, seq);
3680                 }
3681         }
3682
3683         return 0;
3684 }
3685
3686 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3687 {
3688         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3689 }
3690
3691 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3692
3693 static const struct file_operations stmmac_rings_status_fops = {
3694         .owner = THIS_MODULE,
3695         .open = stmmac_sysfs_ring_open,
3696         .read = seq_read,
3697         .llseek = seq_lseek,
3698         .release = single_release,
3699 };
3700
3701 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3702 {
3703         struct net_device *dev = seq->private;
3704         struct stmmac_priv *priv = netdev_priv(dev);
3705
3706         if (!priv->hw_cap_support) {
3707                 seq_printf(seq, "DMA HW features not supported\n");
3708                 return 0;
3709         }
3710
3711         seq_printf(seq, "==============================\n");
3712         seq_printf(seq, "\tDMA HW features\n");
3713         seq_printf(seq, "==============================\n");
3714
3715         seq_printf(seq, "\t10/100 Mbps: %s\n",
3716                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3717         seq_printf(seq, "\t1000 Mbps: %s\n",
3718                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3719         seq_printf(seq, "\tHalf duplex: %s\n",
3720                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3721         seq_printf(seq, "\tHash Filter: %s\n",
3722                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3723         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3724                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3725         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3726                    (priv->dma_cap.pcs) ? "Y" : "N");
3727         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3728                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3729         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3730                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3731         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3732                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3733         seq_printf(seq, "\tRMON module: %s\n",
3734                    (priv->dma_cap.rmon) ? "Y" : "N");
3735         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3736                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3737         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3738                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3739         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3740                    (priv->dma_cap.eee) ? "Y" : "N");
3741         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3742         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3743                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3744         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3745                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3746                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3747         } else {
3748                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3749                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3750                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3751                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3752         }
3753         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3754                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3755         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3756                    priv->dma_cap.number_rx_channel);
3757         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3758                    priv->dma_cap.number_tx_channel);
3759         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3760                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3761
3762         return 0;
3763 }
3764
3765 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3766 {
3767         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3768 }
3769
3770 static const struct file_operations stmmac_dma_cap_fops = {
3771         .owner = THIS_MODULE,
3772         .open = stmmac_sysfs_dma_cap_open,
3773         .read = seq_read,
3774         .llseek = seq_lseek,
3775         .release = single_release,
3776 };
3777
3778 static int stmmac_init_fs(struct net_device *dev)
3779 {
3780         struct stmmac_priv *priv = netdev_priv(dev);
3781
3782         /* Create per netdev entries */
3783         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3784
3785         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3786                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3787
3788                 return -ENOMEM;
3789         }
3790
3791         /* Entry to report DMA RX/TX rings */
3792         priv->dbgfs_rings_status =
3793                 debugfs_create_file("descriptors_status", S_IRUGO,
3794                                     priv->dbgfs_dir, dev,
3795                                     &stmmac_rings_status_fops);
3796
3797         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3798                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3799                 debugfs_remove_recursive(priv->dbgfs_dir);
3800
3801                 return -ENOMEM;
3802         }
3803
3804         /* Entry to report the DMA HW features */
3805         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3806                                             priv->dbgfs_dir,
3807                                             dev, &stmmac_dma_cap_fops);
3808
3809         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3810                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3811                 debugfs_remove_recursive(priv->dbgfs_dir);
3812
3813                 return -ENOMEM;
3814         }
3815
3816         return 0;
3817 }
3818
3819 static void stmmac_exit_fs(struct net_device *dev)
3820 {
3821         struct stmmac_priv *priv = netdev_priv(dev);
3822
3823         debugfs_remove_recursive(priv->dbgfs_dir);
3824 }
3825 #endif /* CONFIG_DEBUG_FS */
3826
3827 static const struct net_device_ops stmmac_netdev_ops = {
3828         .ndo_open = stmmac_open,
3829         .ndo_start_xmit = stmmac_xmit,
3830         .ndo_stop = stmmac_release,
3831         .ndo_change_mtu = stmmac_change_mtu,
3832         .ndo_fix_features = stmmac_fix_features,
3833         .ndo_set_features = stmmac_set_features,
3834         .ndo_set_rx_mode = stmmac_set_rx_mode,
3835         .ndo_tx_timeout = stmmac_tx_timeout,
3836         .ndo_do_ioctl = stmmac_ioctl,
3837 #ifdef CONFIG_NET_POLL_CONTROLLER
3838         .ndo_poll_controller = stmmac_poll_controller,
3839 #endif
3840         .ndo_set_mac_address = eth_mac_addr,
3841 };
3842
3843 /**
3844  *  stmmac_hw_init - Init the MAC device
3845  *  @priv: driver private structure
3846  *  Description: this function is to configure the MAC device according to
3847  *  some platform parameters or the HW capability register. It prepares the
3848  *  driver to use either ring or chain modes and to setup either enhanced or
3849  *  normal descriptors.
3850  */
3851 static int stmmac_hw_init(struct stmmac_priv *priv)
3852 {
3853         struct mac_device_info *mac;
3854
3855         /* Identify the MAC HW device */
3856         if (priv->plat->has_gmac) {
3857                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3858                 mac = dwmac1000_setup(priv->ioaddr,
3859                                       priv->plat->multicast_filter_bins,
3860                                       priv->plat->unicast_filter_entries,
3861                                       &priv->synopsys_id);
3862         } else if (priv->plat->has_gmac4) {
3863                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3864                 mac = dwmac4_setup(priv->ioaddr,
3865                                    priv->plat->multicast_filter_bins,
3866                                    priv->plat->unicast_filter_entries,
3867                                    &priv->synopsys_id);
3868         } else {
3869                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
3870         }
3871         if (!mac)
3872                 return -ENOMEM;
3873
3874         priv->hw = mac;
3875
3876         /* To use the chained or ring mode */
3877         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3878                 priv->hw->mode = &dwmac4_ring_mode_ops;
3879         } else {
3880                 if (chain_mode) {
3881                         priv->hw->mode = &chain_mode_ops;
3882                         dev_info(priv->device, "Chain mode enabled\n");
3883                         priv->mode = STMMAC_CHAIN_MODE;
3884                 } else {
3885                         priv->hw->mode = &ring_mode_ops;
3886                         dev_info(priv->device, "Ring mode enabled\n");
3887                         priv->mode = STMMAC_RING_MODE;
3888                 }
3889         }
3890
3891         /* Get the HW capability (new GMAC newer than 3.50a) */
3892         priv->hw_cap_support = stmmac_get_hw_features(priv);
3893         if (priv->hw_cap_support) {
3894                 dev_info(priv->device, "DMA HW capability register supported\n");
3895
3896                 /* We can override some gmac/dma configuration fields: e.g.
3897                  * enh_desc, tx_coe (e.g. that are passed through the
3898                  * platform) with the values from the HW capability
3899                  * register (if supported).
3900                  */
3901                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
3902                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
3903                 priv->hw->pmt = priv->plat->pmt;
3904
3905                 /* TXCOE doesn't work in thresh DMA mode */
3906                 if (priv->plat->force_thresh_dma_mode)
3907                         priv->plat->tx_coe = 0;
3908                 else
3909                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
3910
3911                 /* In case of GMAC4 rx_coe is from HW cap register. */
3912                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
3913
3914                 if (priv->dma_cap.rx_coe_type2)
3915                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
3916                 else if (priv->dma_cap.rx_coe_type1)
3917                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
3918
3919         } else {
3920                 dev_info(priv->device, "No HW DMA feature register supported\n");
3921         }
3922
3923         /* To use alternate (extended), normal or GMAC4 descriptor structures */
3924         if (priv->synopsys_id >= DWMAC_CORE_4_00)
3925                 priv->hw->desc = &dwmac4_desc_ops;
3926         else
3927                 stmmac_selec_desc_mode(priv);
3928
3929         if (priv->plat->rx_coe) {
3930                 priv->hw->rx_csum = priv->plat->rx_coe;
3931                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
3932                 if (priv->synopsys_id < DWMAC_CORE_4_00)
3933                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
3934         }
3935         if (priv->plat->tx_coe)
3936                 dev_info(priv->device, "TX Checksum insertion supported\n");
3937
3938         if (priv->plat->pmt) {
3939                 dev_info(priv->device, "Wake-Up On Lan supported\n");
3940                 device_set_wakeup_capable(priv->device, 1);
3941         }
3942
3943         if (priv->dma_cap.tsoen)
3944                 dev_info(priv->device, "TSO supported\n");
3945
3946         return 0;
3947 }
3948
3949 /**
3950  * stmmac_dvr_probe
3951  * @device: device pointer
3952  * @plat_dat: platform data pointer
3953  * @res: stmmac resource pointer
3954  * Description: this is the main probe function used to
3955  * call the alloc_etherdev, allocate the priv structure.
3956  * Return:
3957  * returns 0 on success, otherwise errno.
3958  */
3959 int stmmac_dvr_probe(struct device *device,
3960                      struct plat_stmmacenet_data *plat_dat,
3961                      struct stmmac_resources *res)
3962 {
3963         struct net_device *ndev = NULL;
3964         struct stmmac_priv *priv;
3965         int ret = 0;
3966         u32 queue;
3967
3968         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
3969                                   MTL_MAX_TX_QUEUES,
3970                                   MTL_MAX_RX_QUEUES);
3971         if (!ndev)
3972                 return -ENOMEM;
3973
3974         SET_NETDEV_DEV(ndev, device);
3975
3976         priv = netdev_priv(ndev);
3977         priv->device = device;
3978         priv->dev = ndev;
3979
3980         stmmac_set_ethtool_ops(ndev);
3981         priv->pause = pause;
3982         priv->plat = plat_dat;
3983         priv->ioaddr = res->addr;
3984         priv->dev->base_addr = (unsigned long)res->addr;
3985
3986         priv->dev->irq = res->irq;
3987         priv->wol_irq = res->wol_irq;
3988         priv->lpi_irq = res->lpi_irq;
3989
3990         if (res->mac)
3991                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
3992
3993         dev_set_drvdata(device, priv->dev);
3994
3995         /* Verify driver arguments */
3996         stmmac_verify_args();
3997
3998         /* Override with kernel parameters if supplied XXX CRS XXX
3999          * this needs to have multiple instances
4000          */
4001         if ((phyaddr >= 0) && (phyaddr <= 31))
4002                 priv->plat->phy_addr = phyaddr;
4003
4004         if (priv->plat->stmmac_rst)
4005                 reset_control_deassert(priv->plat->stmmac_rst);
4006
4007         /* Init MAC and get the capabilities */
4008         ret = stmmac_hw_init(priv);
4009         if (ret)
4010                 goto error_hw_init;
4011
4012         /* Configure real RX and TX queues */
4013         ndev->real_num_rx_queues = priv->plat->rx_queues_to_use;
4014         ndev->real_num_tx_queues = priv->plat->tx_queues_to_use;
4015
4016         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
4017
4018         ndev->netdev_ops = &stmmac_netdev_ops;
4019
4020         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4021                             NETIF_F_RXCSUM;
4022
4023         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4024                 ndev->hw_features |= NETIF_F_TSO;
4025                 priv->tso = true;
4026                 dev_info(priv->device, "TSO feature enabled\n");
4027         }
4028         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4029         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4030 #ifdef STMMAC_VLAN_TAG_USED
4031         /* Both mac100 and gmac support receive VLAN tag detection */
4032         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4033 #endif
4034         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4035
4036         /* MTU range: 46 - hw-specific max */
4037         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4038         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4039                 ndev->max_mtu = JUMBO_LEN;
4040         else
4041                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4042         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4043          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4044          */
4045         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4046             (priv->plat->maxmtu >= ndev->min_mtu))
4047                 ndev->max_mtu = priv->plat->maxmtu;
4048         else if (priv->plat->maxmtu < ndev->min_mtu)
4049                 dev_warn(priv->device,
4050                          "%s: warning: maxmtu having invalid value (%d)\n",
4051                          __func__, priv->plat->maxmtu);
4052
4053         if (flow_ctrl)
4054                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4055
4056         /* Rx Watchdog is available in the COREs newer than the 3.40.
4057          * In some case, for example on bugged HW this feature
4058          * has to be disable and this can be done by passing the
4059          * riwt_off field from the platform.
4060          */
4061         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
4062                 priv->use_riwt = 1;
4063                 dev_info(priv->device,
4064                          "Enable RX Mitigation via HW Watchdog Timer\n");
4065         }
4066
4067         ret = alloc_dma_desc_resources(priv);
4068         if (ret < 0) {
4069                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
4070                            __func__);
4071                 goto init_dma_error;
4072         }
4073
4074         ret = init_dma_desc_rings(priv->dev, GFP_KERNEL);
4075         if (ret < 0) {
4076                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
4077                            __func__);
4078                 goto init_dma_error;
4079         }
4080
4081         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4082                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4083
4084                 netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4085                                (64 * priv->plat->rx_queues_to_use));
4086         }
4087
4088         spin_lock_init(&priv->lock);
4089
4090         /* If a specific clk_csr value is passed from the platform
4091          * this means that the CSR Clock Range selection cannot be
4092          * changed at run-time and it is fixed. Viceversa the driver'll try to
4093          * set the MDC clock dynamically according to the csr actual
4094          * clock input.
4095          */
4096         if (!priv->plat->clk_csr)
4097                 stmmac_clk_csr_set(priv);
4098         else
4099                 priv->clk_csr = priv->plat->clk_csr;
4100
4101         stmmac_check_pcs_mode(priv);
4102
4103         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4104             priv->hw->pcs != STMMAC_PCS_TBI &&
4105             priv->hw->pcs != STMMAC_PCS_RTBI) {
4106                 /* MDIO bus Registration */
4107                 ret = stmmac_mdio_register(ndev);
4108                 if (ret < 0) {
4109                         dev_err(priv->device,
4110                                 "%s: MDIO bus (id: %d) registration failed",
4111                                 __func__, priv->plat->bus_id);
4112                         goto error_mdio_register;
4113                 }
4114         }
4115
4116         ret = register_netdev(ndev);
4117         if (ret) {
4118                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4119                         __func__, ret);
4120                 goto error_netdev_register;
4121         }
4122
4123         return ret;
4124
4125 error_netdev_register:
4126         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4127             priv->hw->pcs != STMMAC_PCS_TBI &&
4128             priv->hw->pcs != STMMAC_PCS_RTBI)
4129                 stmmac_mdio_unregister(ndev);
4130 error_mdio_register:
4131         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4132                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4133
4134                 netif_napi_del(&rx_q->napi);
4135         }
4136 init_dma_error:
4137         free_dma_desc_resources(priv);
4138 error_hw_init:
4139         free_netdev(ndev);
4140
4141         return ret;
4142 }
4143 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4144
4145 /**
4146  * stmmac_dvr_remove
4147  * @dev: device pointer
4148  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4149  * changes the link status, releases the DMA descriptor rings.
4150  */
4151 int stmmac_dvr_remove(struct device *dev)
4152 {
4153         struct net_device *ndev = dev_get_drvdata(dev);
4154         struct stmmac_priv *priv = netdev_priv(ndev);
4155
4156         netdev_info(priv->dev, "%s: removing driver", __func__);
4157
4158         stmmac_stop_all_dma(priv);
4159
4160         stmmac_set_mac(priv->ioaddr, false);
4161         netif_carrier_off(ndev);
4162         unregister_netdev(ndev);
4163         if (priv->plat->stmmac_rst)
4164                 reset_control_assert(priv->plat->stmmac_rst);
4165         clk_disable_unprepare(priv->plat->pclk);
4166         clk_disable_unprepare(priv->plat->stmmac_clk);
4167         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4168             priv->hw->pcs != STMMAC_PCS_TBI &&
4169             priv->hw->pcs != STMMAC_PCS_RTBI)
4170                 stmmac_mdio_unregister(ndev);
4171         free_netdev(ndev);
4172
4173         return 0;
4174 }
4175 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4176
4177 /**
4178  * stmmac_suspend - suspend callback
4179  * @dev: device pointer
4180  * Description: this is the function to suspend the device and it is called
4181  * by the platform driver to stop the network queue, release the resources,
4182  * program the PMT register (for WoL), clean and release driver resources.
4183  */
4184 int stmmac_suspend(struct device *dev)
4185 {
4186         struct net_device *ndev = dev_get_drvdata(dev);
4187         struct stmmac_priv *priv = netdev_priv(ndev);
4188         unsigned long flags;
4189
4190         if (!ndev || !netif_running(ndev))
4191                 return 0;
4192
4193         if (ndev->phydev)
4194                 phy_stop(ndev->phydev);
4195
4196         spin_lock_irqsave(&priv->lock, flags);
4197
4198         netif_device_detach(ndev);
4199         stmmac_stop_all_queues(priv);
4200
4201         stmmac_disable_all_queues(priv);
4202
4203         /* Stop TX/RX DMA */
4204         stmmac_stop_all_dma(priv);
4205
4206         /* Enable Power down mode by programming the PMT regs */
4207         if (device_may_wakeup(priv->device)) {
4208                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
4209                 priv->irq_wake = 1;
4210         } else {
4211                 stmmac_set_mac(priv->ioaddr, false);
4212                 pinctrl_pm_select_sleep_state(priv->device);
4213                 /* Disable clock in case of PWM is off */
4214                 clk_disable(priv->plat->pclk);
4215                 clk_disable(priv->plat->stmmac_clk);
4216         }
4217         spin_unlock_irqrestore(&priv->lock, flags);
4218
4219         priv->oldlink = 0;
4220         priv->speed = SPEED_UNKNOWN;
4221         priv->oldduplex = DUPLEX_UNKNOWN;
4222         return 0;
4223 }
4224 EXPORT_SYMBOL_GPL(stmmac_suspend);
4225
4226 /**
4227  * stmmac_reset_queues_param - reset queue parameters
4228  * @dev: device pointer
4229  */
4230 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4231 {
4232         u32 rx_cnt = priv->plat->rx_queues_to_use;
4233         u32 tx_cnt = priv->plat->tx_queues_to_use;
4234         u32 queue;
4235
4236         for (queue = 0; queue < rx_cnt; queue++) {
4237                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4238
4239                 rx_q->cur_rx = 0;
4240                 rx_q->dirty_rx = 0;
4241         }
4242
4243         for (queue = 0; queue < tx_cnt; queue++) {
4244                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4245
4246                 tx_q->cur_tx = 0;
4247                 tx_q->dirty_tx = 0;
4248         }
4249 }
4250
4251 /**
4252  * stmmac_resume - resume callback
4253  * @dev: device pointer
4254  * Description: when resume this function is invoked to setup the DMA and CORE
4255  * in a usable state.
4256  */
4257 int stmmac_resume(struct device *dev)
4258 {
4259         struct net_device *ndev = dev_get_drvdata(dev);
4260         struct stmmac_priv *priv = netdev_priv(ndev);
4261         unsigned long flags;
4262
4263         if (!netif_running(ndev))
4264                 return 0;
4265
4266         /* Power Down bit, into the PM register, is cleared
4267          * automatically as soon as a magic packet or a Wake-up frame
4268          * is received. Anyway, it's better to manually clear
4269          * this bit because it can generate problems while resuming
4270          * from another devices (e.g. serial console).
4271          */
4272         if (device_may_wakeup(priv->device)) {
4273                 spin_lock_irqsave(&priv->lock, flags);
4274                 priv->hw->mac->pmt(priv->hw, 0);
4275                 spin_unlock_irqrestore(&priv->lock, flags);
4276                 priv->irq_wake = 0;
4277         } else {
4278                 pinctrl_pm_select_default_state(priv->device);
4279                 /* enable the clk previously disabled */
4280                 clk_enable(priv->plat->stmmac_clk);
4281                 clk_enable(priv->plat->pclk);
4282                 /* reset the phy so that it's ready */
4283                 if (priv->mii)
4284                         stmmac_mdio_reset(priv->mii);
4285         }
4286
4287         netif_device_attach(ndev);
4288
4289         spin_lock_irqsave(&priv->lock, flags);
4290
4291         stmmac_reset_queues_param(priv);
4292
4293         /* reset private mss value to force mss context settings at
4294          * next tso xmit (only used for gmac4).
4295          */
4296         priv->mss = 0;
4297
4298         stmmac_clear_descriptors(priv);
4299
4300         stmmac_hw_setup(ndev, false);
4301         stmmac_init_tx_coalesce(priv);
4302         stmmac_set_rx_mode(ndev);
4303
4304         stmmac_enable_all_queues(priv);
4305
4306         stmmac_start_all_queues(priv);
4307
4308         spin_unlock_irqrestore(&priv->lock, flags);
4309
4310         if (ndev->phydev)
4311                 phy_start(ndev->phydev);
4312
4313         return 0;
4314 }
4315 EXPORT_SYMBOL_GPL(stmmac_resume);
4316
4317 #ifndef MODULE
4318 static int __init stmmac_cmdline_opt(char *str)
4319 {
4320         char *opt;
4321
4322         if (!str || !*str)
4323                 return -EINVAL;
4324         while ((opt = strsep(&str, ",")) != NULL) {
4325                 if (!strncmp(opt, "debug:", 6)) {
4326                         if (kstrtoint(opt + 6, 0, &debug))
4327                                 goto err;
4328                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4329                         if (kstrtoint(opt + 8, 0, &phyaddr))
4330                                 goto err;
4331                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4332                         if (kstrtoint(opt + 7, 0, &buf_sz))
4333                                 goto err;
4334                 } else if (!strncmp(opt, "tc:", 3)) {
4335                         if (kstrtoint(opt + 3, 0, &tc))
4336                                 goto err;
4337                 } else if (!strncmp(opt, "watchdog:", 9)) {
4338                         if (kstrtoint(opt + 9, 0, &watchdog))
4339                                 goto err;
4340                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4341                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4342                                 goto err;
4343                 } else if (!strncmp(opt, "pause:", 6)) {
4344                         if (kstrtoint(opt + 6, 0, &pause))
4345                                 goto err;
4346                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4347                         if (kstrtoint(opt + 10, 0, &eee_timer))
4348                                 goto err;
4349                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4350                         if (kstrtoint(opt + 11, 0, &chain_mode))
4351                                 goto err;
4352                 }
4353         }
4354         return 0;
4355
4356 err:
4357         pr_err("%s: ERROR broken module parameter conversion", __func__);
4358         return -EINVAL;
4359 }
4360
4361 __setup("stmmaceth=", stmmac_cmdline_opt);
4362 #endif /* MODULE */
4363
4364 static int __init stmmac_init(void)
4365 {
4366 #ifdef CONFIG_DEBUG_FS
4367         /* Create debugfs main directory if it doesn't exist yet */
4368         if (!stmmac_fs_dir) {
4369                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4370
4371                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4372                         pr_err("ERROR %s, debugfs create directory failed\n",
4373                                STMMAC_RESOURCE_NAME);
4374
4375                         return -ENOMEM;
4376                 }
4377         }
4378 #endif
4379
4380         return 0;
4381 }
4382
4383 static void __exit stmmac_exit(void)
4384 {
4385 #ifdef CONFIG_DEBUG_FS
4386         debugfs_remove_recursive(stmmac_fs_dir);
4387 #endif
4388 }
4389
4390 module_init(stmmac_init)
4391 module_exit(stmmac_exit)
4392
4393 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4394 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4395 MODULE_LICENSE("GPL");