drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_disable_all_queues - Disable all queues
 143  * @priv: driver private structure
 144  */
 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 146 {
 147         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 148         u32 queue;
 149
 150         for (queue = 0; queue < rx_queues_cnt; queue++) {
 151                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 152
 153                 napi_disable(&rx_q->napi);
 154         }
 155 }
 156
 157 /**
 158  * stmmac_enable_all_queues - Enable all queues
 159  * @priv: driver private structure
 160  */
 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 162 {
 163         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 164         u32 queue;
 165
 166         for (queue = 0; queue < rx_queues_cnt; queue++) {
 167                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 168
 169                 napi_enable(&rx_q->napi);
 170         }
 171 }
 172
 173 /**
 174  * stmmac_stop_all_queues - Stop all queues
 175  * @priv: driver private structure
 176  */
 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
 178 {
 179         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 180         u32 queue;
 181
 182         for (queue = 0; queue < tx_queues_cnt; queue++)
 183                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
 184 }
 185
 186 /**
 187  * stmmac_start_all_queues - Start all queues
 188  * @priv: driver private structure
 189  */
 190 static void stmmac_start_all_queues(struct stmmac_priv *priv)
 191 {
 192         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 193         u32 queue;
 194
 195         for (queue = 0; queue < tx_queues_cnt; queue++)
 196                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
 197 }
 198
 199 /**
 200  * stmmac_clk_csr_set - dynamically set the MDC clock
 201  * @priv: driver private structure
 202  * Description: this is to dynamically set the MDC clock according to the csr
 203  * clock input.
 204  * Note:
 205  *      If a specific clk_csr value is passed from the platform
 206  *      this means that the CSR Clock Range selection cannot be
 207  *      changed at run-time and it is fixed (as reported in the driver
 208  *      documentation). Viceversa the driver will try to set the MDC
 209  *      clock dynamically according to the actual clock input.
 210  */
 211 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 212 {
 213         u32 clk_rate;
 214
 215         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 216
 217         /* Platform provided default clk_csr would be assumed valid
 218          * for all other cases except for the below mentioned ones.
 219          * For values higher than the IEEE 802.3 specified frequency
 220          * we can not estimate the proper divider as it is not known
 221          * the frequency of clk_csr_i. So we do not change the default
 222          * divider.
 223          */
 224         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 225                 if (clk_rate < CSR_F_35M)
 226                         priv->clk_csr = STMMAC_CSR_20_35M;
 227                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 228                         priv->clk_csr = STMMAC_CSR_35_60M;
 229                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 230                         priv->clk_csr = STMMAC_CSR_60_100M;
 231                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 232                         priv->clk_csr = STMMAC_CSR_100_150M;
 233                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 234                         priv->clk_csr = STMMAC_CSR_150_250M;
 235                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 236                         priv->clk_csr = STMMAC_CSR_250_300M;
 237         }
 238 }
 239
 240 static void print_pkt(unsigned char *buf, int len)
 241 {
 242         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 243         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 244 }
 245
 246 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 247 {
 248         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 249         u32 avail;
 250
 251         if (tx_q->dirty_tx > tx_q->cur_tx)
 252                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 253         else
 254                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 255
 256         return avail;
 257 }
 258
 259 /**
 260  * stmmac_rx_dirty - Get RX queue dirty
 261  * @priv: driver private structure
 262  * @queue: RX queue index
 263  */
 264 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 265 {
 266         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 267         u32 dirty;
 268
 269         if (rx_q->dirty_rx <= rx_q->cur_rx)
 270                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 271         else
 272                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 273
 274         return dirty;
 275 }
 276
 277 /**
 278  * stmmac_hw_fix_mac_speed - callback for speed selection
 279  * @priv: driver private structure
 280  * Description: on some platforms (e.g. ST), some HW system configuration
 281  * registers have to be set according to the link speed negotiated.
 282  */
 283 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 284 {
 285         struct net_device *ndev = priv->dev;
 286         struct phy_device *phydev = ndev->phydev;
 287
 288         if (likely(priv->plat->fix_mac_speed))
 289                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 290 }
 291
 292 /**
 293  * stmmac_enable_eee_mode - check and enter in LPI mode
 294  * @priv: driver private structure
 295  * Description: this function is to verify and enter in LPI mode in case of
 296  * EEE.
 297  */
 298 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 299 {
 300         u32 tx_cnt = priv->plat->tx_queues_to_use;
 301         u32 queue;
 302
 303         /* check if all TX queues have the work finished */
 304         for (queue = 0; queue < tx_cnt; queue++) {
 305                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 306
 307                 if (tx_q->dirty_tx != tx_q->cur_tx)
 308                         return; /* still unfinished work */
 309         }
 310
 311         /* Check and enter in LPI mode */
 312         if (!priv->tx_path_in_lpi_mode)
 313                 priv->hw->mac->set_eee_mode(priv->hw,
 314                                             priv->plat->en_tx_lpi_clockgating);
 315 }
 316
 317 /**
 318  * stmmac_disable_eee_mode - disable and exit from LPI mode
 319  * @priv: driver private structure
 320  * Description: this function is to exit and disable EEE in case of
 321  * LPI state is true. This is called by the xmit.
 322  */
 323 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 324 {
 325         priv->hw->mac->reset_eee_mode(priv->hw);
 326         del_timer_sync(&priv->eee_ctrl_timer);
 327         priv->tx_path_in_lpi_mode = false;
 328 }
 329
 330 /**
 331  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 332  * @arg : data hook
 333  * Description:
 334  *  if there is no data transfer and if we are not in LPI state,
 335  *  then MAC Transmitter can be moved to LPI state.
 336  */
 337 static void stmmac_eee_ctrl_timer(unsigned long arg)
 338 {
 339         struct stmmac_priv *priv = (struct stmmac_priv *)arg;
 340
 341         stmmac_enable_eee_mode(priv);
 342         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 343 }
 344
 345 /**
 346  * stmmac_eee_init - init EEE
 347  * @priv: driver private structure
 348  * Description:
 349  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 350  *  can also manage EEE, this function enable the LPI state and start related
 351  *  timer.
 352  */
 353 bool stmmac_eee_init(struct stmmac_priv *priv)
 354 {
 355         struct net_device *ndev = priv->dev;
 356         unsigned long flags;
 357         bool ret = false;
 358
 359         /* Using PCS we cannot dial with the phy registers at this stage
 360          * so we do not support extra feature like EEE.
 361          */
 362         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 363             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 364             (priv->hw->pcs == STMMAC_PCS_RTBI))
 365                 goto out;
 366
 367         /* MAC core supports the EEE feature. */
 368         if (priv->dma_cap.eee) {
 369                 int tx_lpi_timer = priv->tx_lpi_timer;
 370
 371                 /* Check if the PHY supports EEE */
 372                 if (phy_init_eee(ndev->phydev, 1)) {
 373                         /* To manage at run-time if the EEE cannot be supported
 374                          * anymore (for example because the lp caps have been
 375                          * changed).
 376                          * In that case the driver disable own timers.
 377                          */
 378                         spin_lock_irqsave(&priv->lock, flags);
 379                         if (priv->eee_active) {
 380                                 netdev_dbg(priv->dev, "disable EEE\n");
 381                                 del_timer_sync(&priv->eee_ctrl_timer);
 382                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 383                                                              tx_lpi_timer);
 384                         }
 385                         priv->eee_active = 0;
 386                         spin_unlock_irqrestore(&priv->lock, flags);
 387                         goto out;
 388                 }
 389                 /* Activate the EEE and start timers */
 390                 spin_lock_irqsave(&priv->lock, flags);
 391                 if (!priv->eee_active) {
 392                         priv->eee_active = 1;
 393                         setup_timer(&priv->eee_ctrl_timer,
 394                                     stmmac_eee_ctrl_timer,
 395                                     (unsigned long)priv);
 396                         mod_timer(&priv->eee_ctrl_timer,
 397                                   STMMAC_LPI_T(eee_timer));
 398
 399                         priv->hw->mac->set_eee_timer(priv->hw,
 400                                                      STMMAC_DEFAULT_LIT_LS,
 401                                                      tx_lpi_timer);
 402                 }
 403                 /* Set HW EEE according to the speed */
 404                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 405
 406                 ret = true;
 407                 spin_unlock_irqrestore(&priv->lock, flags);
 408
 409                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 410         }
 411 out:
 412         return ret;
 413 }
 414
 415 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 416  * @priv: driver private structure
 417  * @p : descriptor pointer
 418  * @skb : the socket buffer
 419  * Description :
 420  * This function will read timestamp from the descriptor & pass it to stack.
 421  * and also perform some sanity checks.
 422  */
 423 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 424                                    struct dma_desc *p, struct sk_buff *skb)
 425 {
 426         struct skb_shared_hwtstamps shhwtstamp;
 427         u64 ns;
 428
 429         if (!priv->hwts_tx_en)
 430                 return;
 431
 432         /* exit if skb doesn't support hw tstamp */
 433         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 434                 return;
 435
 436         /* check tx tstamp status */
 437         if (!priv->hw->desc->get_tx_timestamp_status(p)) {
 438                 /* get the valid tstamp */
 439                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 440
 441                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 442                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 443
 444                 netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 445                 /* pass tstamp to stack */
 446                 skb_tstamp_tx(skb, &shhwtstamp);
 447         }
 448
 449         return;
 450 }
 451
 452 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 453  * @priv: driver private structure
 454  * @p : descriptor pointer
 455  * @np : next descriptor pointer
 456  * @skb : the socket buffer
 457  * Description :
 458  * This function will read received packet's timestamp from the descriptor
 459  * and pass it to stack. It also perform some sanity checks.
 460  */
 461 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 462                                    struct dma_desc *np, struct sk_buff *skb)
 463 {
 464         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 465         u64 ns;
 466
 467         if (!priv->hwts_rx_en)
 468                 return;
 469
 470         /* Check if timestamp is available */
 471         if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) {
 472                 /* For GMAC4, the valid timestamp is from CTX next desc. */
 473                 if (priv->plat->has_gmac4)
 474                         ns = priv->hw->desc->get_timestamp(np, priv->adv_ts);
 475                 else
 476                         ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 477
 478                 netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 479                 shhwtstamp = skb_hwtstamps(skb);
 480                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 481                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 482         } else  {
 483                 netdev_err(priv->dev, "cannot get RX hw timestamp\n");
 484         }
 485 }
 486
 487 /**
 488  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 489  *  @dev: device pointer.
 490  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 491  *  a proprietary structure used to pass information to the driver.
 492  *  Description:
 493  *  This function configures the MAC to enable/disable both outgoing(TX)
 494  *  and incoming(RX) packets time stamping based on user input.
 495  *  Return Value:
 496  *  0 on success and an appropriate -ve integer on failure.
 497  */
 498 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 499 {
 500         struct stmmac_priv *priv = netdev_priv(dev);
 501         struct hwtstamp_config config;
 502         struct timespec64 now;
 503         u64 temp = 0;
 504         u32 ptp_v2 = 0;
 505         u32 tstamp_all = 0;
 506         u32 ptp_over_ipv4_udp = 0;
 507         u32 ptp_over_ipv6_udp = 0;
 508         u32 ptp_over_ethernet = 0;
 509         u32 snap_type_sel = 0;
 510         u32 ts_master_en = 0;
 511         u32 ts_event_en = 0;
 512         u32 value = 0;
 513         u32 sec_inc;
 514
 515         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 516                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 517                 priv->hwts_tx_en = 0;
 518                 priv->hwts_rx_en = 0;
 519
 520                 return -EOPNOTSUPP;
 521         }
 522
 523         if (copy_from_user(&config, ifr->ifr_data,
 524                            sizeof(struct hwtstamp_config)))
 525                 return -EFAULT;
 526
 527         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 528                    __func__, config.flags, config.tx_type, config.rx_filter);
 529
 530         /* reserved for future extensions */
 531         if (config.flags)
 532                 return -EINVAL;
 533
 534         if (config.tx_type != HWTSTAMP_TX_OFF &&
 535             config.tx_type != HWTSTAMP_TX_ON)
 536                 return -ERANGE;
 537
 538         if (priv->adv_ts) {
 539                 switch (config.rx_filter) {
 540                 case HWTSTAMP_FILTER_NONE:
 541                         /* time stamp no incoming packet at all */
 542                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 543                         break;
 544
 545                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 546                         /* PTP v1, UDP, any kind of event packet */
 547                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 548                         /* take time stamp for all event messages */
 549                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 550
 551                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 552                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 553                         break;
 554
 555                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 556                         /* PTP v1, UDP, Sync packet */
 557                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 558                         /* take time stamp for SYNC messages only */
 559                         ts_event_en = PTP_TCR_TSEVNTENA;
 560
 561                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 562                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 563                         break;
 564
 565                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 566                         /* PTP v1, UDP, Delay_req packet */
 567                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 568                         /* take time stamp for Delay_Req messages only */
 569                         ts_master_en = PTP_TCR_TSMSTRENA;
 570                         ts_event_en = PTP_TCR_TSEVNTENA;
 571
 572                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 573                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 574                         break;
 575
 576                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 577                         /* PTP v2, UDP, any kind of event packet */
 578                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 579                         ptp_v2 = PTP_TCR_TSVER2ENA;
 580                         /* take time stamp for all event messages */
 581                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 582
 583                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 584                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 585                         break;
 586
 587                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 588                         /* PTP v2, UDP, Sync packet */
 589                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 590                         ptp_v2 = PTP_TCR_TSVER2ENA;
 591                         /* take time stamp for SYNC messages only */
 592                         ts_event_en = PTP_TCR_TSEVNTENA;
 593
 594                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 595                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 596                         break;
 597
 598                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 599                         /* PTP v2, UDP, Delay_req packet */
 600                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 601                         ptp_v2 = PTP_TCR_TSVER2ENA;
 602                         /* take time stamp for Delay_Req messages only */
 603                         ts_master_en = PTP_TCR_TSMSTRENA;
 604                         ts_event_en = PTP_TCR_TSEVNTENA;
 605
 606                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 607                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 608                         break;
 609
 610                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 611                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 612                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 613                         ptp_v2 = PTP_TCR_TSVER2ENA;
 614                         /* take time stamp for all event messages */
 615                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 616
 617                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 618                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 619                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 620                         break;
 621
 622                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 623                         /* PTP v2/802.AS1, any layer, Sync packet */
 624                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 625                         ptp_v2 = PTP_TCR_TSVER2ENA;
 626                         /* take time stamp for SYNC messages only */
 627                         ts_event_en = PTP_TCR_TSEVNTENA;
 628
 629                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 630                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 631                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 632                         break;
 633
 634                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 635                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 636                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 637                         ptp_v2 = PTP_TCR_TSVER2ENA;
 638                         /* take time stamp for Delay_Req messages only */
 639                         ts_master_en = PTP_TCR_TSMSTRENA;
 640                         ts_event_en = PTP_TCR_TSEVNTENA;
 641
 642                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 643                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 644                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 645                         break;
 646
 647                 case HWTSTAMP_FILTER_ALL:
 648                         /* time stamp any incoming packet */
 649                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 650                         tstamp_all = PTP_TCR_TSENALL;
 651                         break;
 652
 653                 default:
 654                         return -ERANGE;
 655                 }
 656         } else {
 657                 switch (config.rx_filter) {
 658                 case HWTSTAMP_FILTER_NONE:
 659                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 660                         break;
 661                 default:
 662                         /* PTP v1, UDP, any kind of event packet */
 663                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 664                         break;
 665                 }
 666         }
 667         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 668         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 669
 670         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 671                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 672         else {
 673                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 674                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 675                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 676                          ts_master_en | snap_type_sel);
 677                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 678
 679                 /* program Sub Second Increment reg */
 680                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 681                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 682                         priv->plat->has_gmac4);
 683                 temp = div_u64(1000000000ULL, sec_inc);
 684
 685                 /* calculate default added value:
 686                  * formula is :
 687                  * addend = (2^32)/freq_div_ratio;
 688                  * where, freq_div_ratio = 1e9ns/sec_inc
 689                  */
 690                 temp = (u64)(temp << 32);
 691                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 692                 priv->hw->ptp->config_addend(priv->ptpaddr,
 693                                              priv->default_addend);
 694
 695                 /* initialize system time */
 696                 ktime_get_real_ts64(&now);
 697
 698                 /* lower 32 bits of tv_sec are safe until y2106 */
 699                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 700                                             now.tv_nsec);
 701         }
 702
 703         return copy_to_user(ifr->ifr_data, &config,
 704                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 705 }
 706
 707 /**
 708  * stmmac_init_ptp - init PTP
 709  * @priv: driver private structure
 710  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 711  * This is done by looking at the HW cap. register.
 712  * This function also registers the ptp driver.
 713  */
 714 static int stmmac_init_ptp(struct stmmac_priv *priv)
 715 {
 716         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 717                 return -EOPNOTSUPP;
 718
 719         priv->adv_ts = 0;
 720         /* Check if adv_ts can be enabled for dwmac 4.x core */
 721         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 722                 priv->adv_ts = 1;
 723         /* Dwmac 3.x core with extend_desc can support adv_ts */
 724         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 725                 priv->adv_ts = 1;
 726
 727         if (priv->dma_cap.time_stamp)
 728                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 729
 730         if (priv->adv_ts)
 731                 netdev_info(priv->dev,
 732                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 733
 734         priv->hw->ptp = &stmmac_ptp;
 735         priv->hwts_tx_en = 0;
 736         priv->hwts_rx_en = 0;
 737
 738         stmmac_ptp_register(priv);
 739
 740         return 0;
 741 }
 742
 743 static void stmmac_release_ptp(struct stmmac_priv *priv)
 744 {
 745         if (priv->plat->clk_ptp_ref)
 746                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 747         stmmac_ptp_unregister(priv);
 748 }
 749
 750 /**
 751  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 752  *  @priv: driver private structure
 753  *  Description: It is used for configuring the flow control in all queues
 754  */
 755 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 756 {
 757         u32 tx_cnt = priv->plat->tx_queues_to_use;
 758
 759         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 760                                  priv->pause, tx_cnt);
 761 }
 762
 763 /**
 764  * stmmac_adjust_link - adjusts the link parameters
 765  * @dev: net device structure
 766  * Description: this is the helper called by the physical abstraction layer
 767  * drivers to communicate the phy link status. According the speed and duplex
 768  * this driver can invoke registered glue-logic as well.
 769  * It also invoke the eee initialization because it could happen when switch
 770  * on different networks (that are eee capable).
 771  */
 772 static void stmmac_adjust_link(struct net_device *dev)
 773 {
 774         struct stmmac_priv *priv = netdev_priv(dev);
 775         struct phy_device *phydev = dev->phydev;
 776         unsigned long flags;
 777         int new_state = 0;
 778
 779         if (!phydev)
 780                 return;
 781
 782         spin_lock_irqsave(&priv->lock, flags);
 783
 784         if (phydev->link) {
 785                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 786
 787                 /* Now we make sure that we can be in full duplex mode.
 788                  * If not, we operate in half-duplex mode. */
 789                 if (phydev->duplex != priv->oldduplex) {
 790                         new_state = 1;
 791                         if (!(phydev->duplex))
 792                                 ctrl &= ~priv->hw->link.duplex;
 793                         else
 794                                 ctrl |= priv->hw->link.duplex;
 795                         priv->oldduplex = phydev->duplex;
 796                 }
 797                 /* Flow Control operation */
 798                 if (phydev->pause)
 799                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 800
 801                 if (phydev->speed != priv->speed) {
 802                         new_state = 1;
 803                         switch (phydev->speed) {
 804                         case 1000:
 805                                 if (priv->plat->has_gmac ||
 806                                     priv->plat->has_gmac4)
 807                                         ctrl &= ~priv->hw->link.port;
 808                                 break;
 809                         case 100:
 810                                 if (priv->plat->has_gmac ||
 811                                     priv->plat->has_gmac4) {
 812                                         ctrl |= priv->hw->link.port;
 813                                         ctrl |= priv->hw->link.speed;
 814                                 } else {
 815                                         ctrl &= ~priv->hw->link.port;
 816                                 }
 817                                 break;
 818                         case 10:
 819                                 if (priv->plat->has_gmac ||
 820                                     priv->plat->has_gmac4) {
 821                                         ctrl |= priv->hw->link.port;
 822                                         ctrl &= ~(priv->hw->link.speed);
 823                                 } else {
 824                                         ctrl &= ~priv->hw->link.port;
 825                                 }
 826                                 break;
 827                         default:
 828                                 netif_warn(priv, link, priv->dev,
 829                                            "broken speed: %d\n", phydev->speed);
 830                                 phydev->speed = SPEED_UNKNOWN;
 831                                 break;
 832                         }
 833                         if (phydev->speed != SPEED_UNKNOWN)
 834                                 stmmac_hw_fix_mac_speed(priv);
 835                         priv->speed = phydev->speed;
 836                 }
 837
 838                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 839
 840                 if (!priv->oldlink) {
 841                         new_state = 1;
 842                         priv->oldlink = 1;
 843                 }
 844         } else if (priv->oldlink) {
 845                 new_state = 1;
 846                 priv->oldlink = 0;
 847                 priv->speed = SPEED_UNKNOWN;
 848                 priv->oldduplex = DUPLEX_UNKNOWN;
 849         }
 850
 851         if (new_state && netif_msg_link(priv))
 852                 phy_print_status(phydev);
 853
 854         spin_unlock_irqrestore(&priv->lock, flags);
 855
 856         if (phydev->is_pseudo_fixed_link)
 857                 /* Stop PHY layer to call the hook to adjust the link in case
 858                  * of a switch is attached to the stmmac driver.
 859                  */
 860                 phydev->irq = PHY_IGNORE_INTERRUPT;
 861         else
 862                 /* At this stage, init the EEE if supported.
 863                  * Never called in case of fixed_link.
 864                  */
 865                 priv->eee_enabled = stmmac_eee_init(priv);
 866 }
 867
 868 /**
 869  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 870  * @priv: driver private structure
 871  * Description: this is to verify if the HW supports the PCS.
 872  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 873  * configured for the TBI, RTBI, or SGMII PHY interface.
 874  */
 875 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 876 {
 877         int interface = priv->plat->interface;
 878
 879         if (priv->dma_cap.pcs) {
 880                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 881                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 882                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 883                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 884                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 885                         priv->hw->pcs = STMMAC_PCS_RGMII;
 886                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 887                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 888                         priv->hw->pcs = STMMAC_PCS_SGMII;
 889                 }
 890         }
 891 }
 892
 893 /**
 894  * stmmac_init_phy - PHY initialization
 895  * @dev: net device structure
 896  * Description: it initializes the driver's PHY state, and attaches the PHY
 897  * to the mac driver.
 898  *  Return value:
 899  *  0 on success
 900  */
 901 static int stmmac_init_phy(struct net_device *dev)
 902 {
 903         struct stmmac_priv *priv = netdev_priv(dev);
 904         struct phy_device *phydev;
 905         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 906         char bus_id[MII_BUS_ID_SIZE];
 907         int interface = priv->plat->interface;
 908         int max_speed = priv->plat->max_speed;
 909         priv->oldlink = 0;
 910         priv->speed = SPEED_UNKNOWN;
 911         priv->oldduplex = DUPLEX_UNKNOWN;
 912
 913         if (priv->plat->phy_node) {
 914                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 915                                         &stmmac_adjust_link, 0, interface);
 916         } else {
 917                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 918                          priv->plat->bus_id);
 919
 920                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 921                          priv->plat->phy_addr);
 922                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 923                            phy_id_fmt);
 924
 925                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 926                                      interface);
 927         }
 928
 929         if (IS_ERR_OR_NULL(phydev)) {
 930                 netdev_err(priv->dev, "Could not attach to PHY\n");
 931                 if (!phydev)
 932                         return -ENODEV;
 933
 934                 return PTR_ERR(phydev);
 935         }
 936
 937         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 938         if ((interface == PHY_INTERFACE_MODE_MII) ||
 939             (interface == PHY_INTERFACE_MODE_RMII) ||
 940                 (max_speed < 1000 && max_speed > 0))
 941                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 942                                          SUPPORTED_1000baseT_Full);
 943
 944         /*
 945          * Broken HW is sometimes missing the pull-up resistor on the
 946          * MDIO line, which results in reads to non-existent devices returning
 947          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 948          * device as well.
 949          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 950          */
 951         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 952                 phy_disconnect(phydev);
 953                 return -ENODEV;
 954         }
 955
 956         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 957          * subsequent PHY polling, make sure we force a link transition if
 958          * we have a UP/DOWN/UP transition
 959          */
 960         if (phydev->is_pseudo_fixed_link)
 961                 phydev->irq = PHY_POLL;
 962
 963         phy_attached_info(phydev);
 964         return 0;
 965 }
 966
 967 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
 968 {
 969         u32 rx_cnt = priv->plat->rx_queues_to_use;
 970         void *head_rx;
 971         u32 queue;
 972
 973         /* Display RX rings */
 974         for (queue = 0; queue < rx_cnt; queue++) {
 975                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 976
 977                 pr_info("\tRX Queue %u rings\n", queue);
 978
 979                 if (priv->extend_desc)
 980                         head_rx = (void *)rx_q->dma_erx;
 981                 else
 982                         head_rx = (void *)rx_q->dma_rx;
 983
 984                 /* Display RX ring */
 985                 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 986         }
 987 }
 988
 989 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
 990 {
 991         u32 tx_cnt = priv->plat->tx_queues_to_use;
 992         void *head_tx;
 993         u32 queue;
 994
 995         /* Display TX rings */
 996         for (queue = 0; queue < tx_cnt; queue++) {
 997                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 998
 999                 pr_info("\tTX Queue %d rings\n", queue);
1000
1001                 if (priv->extend_desc)
1002                         head_tx = (void *)tx_q->dma_etx;
1003                 else
1004                         head_tx = (void *)tx_q->dma_tx;
1005
1006                 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
1007         }
1008 }
1009
1010 static void stmmac_display_rings(struct stmmac_priv *priv)
1011 {
1012         /* Display RX ring */
1013         stmmac_display_rx_rings(priv);
1014
1015         /* Display TX ring */
1016         stmmac_display_tx_rings(priv);
1017 }
1018
1019 static int stmmac_set_bfsize(int mtu, int bufsize)
1020 {
1021         int ret = bufsize;
1022
1023         if (mtu >= BUF_SIZE_4KiB)
1024                 ret = BUF_SIZE_8KiB;
1025         else if (mtu >= BUF_SIZE_2KiB)
1026                 ret = BUF_SIZE_4KiB;
1027         else if (mtu > DEFAULT_BUFSIZE)
1028                 ret = BUF_SIZE_2KiB;
1029         else
1030                 ret = DEFAULT_BUFSIZE;
1031
1032         return ret;
1033 }
1034
1035 /**
1036  * stmmac_clear_rx_descriptors - clear RX descriptors
1037  * @priv: driver private structure
1038  * @queue: RX queue index
1039  * Description: this function is called to clear the RX descriptors
1040  * in case of both basic and extended descriptors are used.
1041  */
1042 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1043 {
1044         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1045         int i;
1046
1047         /* Clear the RX descriptors */
1048         for (i = 0; i < DMA_RX_SIZE; i++)
1049                 if (priv->extend_desc)
1050                         priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
1051                                                      priv->use_riwt, priv->mode,
1052                                                      (i == DMA_RX_SIZE - 1));
1053                 else
1054                         priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
1055                                                      priv->use_riwt, priv->mode,
1056                                                      (i == DMA_RX_SIZE - 1));
1057 }
1058
1059 /**
1060  * stmmac_clear_tx_descriptors - clear tx descriptors
1061  * @priv: driver private structure
1062  * @queue: TX queue index.
1063  * Description: this function is called to clear the TX descriptors
1064  * in case of both basic and extended descriptors are used.
1065  */
1066 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1067 {
1068         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1069         int i;
1070
1071         /* Clear the TX descriptors */
1072         for (i = 0; i < DMA_TX_SIZE; i++)
1073                 if (priv->extend_desc)
1074                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1075                                                      priv->mode,
1076                                                      (i == DMA_TX_SIZE - 1));
1077                 else
1078                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1079                                                      priv->mode,
1080                                                      (i == DMA_TX_SIZE - 1));
1081 }
1082
1083 /**
1084  * stmmac_clear_descriptors - clear descriptors
1085  * @priv: driver private structure
1086  * Description: this function is called to clear the TX and RX descriptors
1087  * in case of both basic and extended descriptors are used.
1088  */
1089 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1090 {
1091         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1092         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1093         u32 queue;
1094
1095         /* Clear the RX descriptors */
1096         for (queue = 0; queue < rx_queue_cnt; queue++)
1097                 stmmac_clear_rx_descriptors(priv, queue);
1098
1099         /* Clear the TX descriptors */
1100         for (queue = 0; queue < tx_queue_cnt; queue++)
1101                 stmmac_clear_tx_descriptors(priv, queue);
1102 }
1103
1104 /**
1105  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1106  * @priv: driver private structure
1107  * @p: descriptor pointer
1108  * @i: descriptor index
1109  * @flags: gfp flag
1110  * @queue: RX queue index
1111  * Description: this function is called to allocate a receive buffer, perform
1112  * the DMA mapping and init the descriptor.
1113  */
1114 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1115                                   int i, gfp_t flags, u32 queue)
1116 {
1117         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1118         struct sk_buff *skb;
1119
1120         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1121         if (!skb) {
1122                 netdev_err(priv->dev,
1123                            "%s: Rx init fails; skb is NULL\n", __func__);
1124                 return -ENOMEM;
1125         }
1126         rx_q->rx_skbuff[i] = skb;
1127         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1128                                                 priv->dma_buf_sz,
1129                                                 DMA_FROM_DEVICE);
1130         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1131                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1132                 dev_kfree_skb_any(skb);
1133                 return -EINVAL;
1134         }
1135
1136         if (priv->synopsys_id >= DWMAC_CORE_4_00)
1137                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1138         else
1139                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1140
1141         if ((priv->hw->mode->init_desc3) &&
1142             (priv->dma_buf_sz == BUF_SIZE_16KiB))
1143                 priv->hw->mode->init_desc3(p);
1144
1145         return 0;
1146 }
1147
1148 /**
1149  * stmmac_free_rx_buffer - free RX dma buffers
1150  * @priv: private structure
1151  * @queue: RX queue index
1152  * @i: buffer index.
1153  */
1154 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1155 {
1156         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1157
1158         if (rx_q->rx_skbuff[i]) {
1159                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1160                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1161                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1162         }
1163         rx_q->rx_skbuff[i] = NULL;
1164 }
1165
1166 /**
1167  * stmmac_free_tx_buffer - free RX dma buffers
1168  * @priv: private structure
1169  * @queue: RX queue index
1170  * @i: buffer index.
1171  */
1172 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1173 {
1174         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1175
1176         if (tx_q->tx_skbuff_dma[i].buf) {
1177                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1178                         dma_unmap_page(priv->device,
1179                                        tx_q->tx_skbuff_dma[i].buf,
1180                                        tx_q->tx_skbuff_dma[i].len,
1181                                        DMA_TO_DEVICE);
1182                 else
1183                         dma_unmap_single(priv->device,
1184                                          tx_q->tx_skbuff_dma[i].buf,
1185                                          tx_q->tx_skbuff_dma[i].len,
1186                                          DMA_TO_DEVICE);
1187         }
1188
1189         if (tx_q->tx_skbuff[i]) {
1190                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1191                 tx_q->tx_skbuff[i] = NULL;
1192                 tx_q->tx_skbuff_dma[i].buf = 0;
1193                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1194         }
1195 }
1196
1197 /**
1198  * init_dma_rx_desc_rings - init the RX descriptor rings
1199  * @dev: net device structure
1200  * @flags: gfp flag.
1201  * Description: this function initializes the DMA RX descriptors
1202  * and allocates the socket buffers. It supports the chained and ring
1203  * modes.
1204  */
1205 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1206 {
1207         struct stmmac_priv *priv = netdev_priv(dev);
1208         u32 rx_count = priv->plat->rx_queues_to_use;
1209         unsigned int bfsize = 0;
1210         int ret = -ENOMEM;
1211         int queue;
1212         int i;
1213
1214         if (priv->hw->mode->set_16kib_bfsize)
1215                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1216
1217         if (bfsize < BUF_SIZE_16KiB)
1218                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1219
1220         priv->dma_buf_sz = bfsize;
1221
1222         /* RX INITIALIZATION */
1223         netif_dbg(priv, probe, priv->dev,
1224                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1225
1226         for (queue = 0; queue < rx_count; queue++) {
1227                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1228
1229                 netif_dbg(priv, probe, priv->dev,
1230                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1231                           (u32)rx_q->dma_rx_phy);
1232
1233                 for (i = 0; i < DMA_RX_SIZE; i++) {
1234                         struct dma_desc *p;
1235
1236                         if (priv->extend_desc)
1237                                 p = &((rx_q->dma_erx + i)->basic);
1238                         else
1239                                 p = rx_q->dma_rx + i;
1240
1241                         ret = stmmac_init_rx_buffers(priv, p, i, flags,
1242                                                      queue);
1243                         if (ret)
1244                                 goto err_init_rx_buffers;
1245
1246                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1247                                   rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data,
1248                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1249                 }
1250
1251                 rx_q->cur_rx = 0;
1252                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1253
1254                 stmmac_clear_rx_descriptors(priv, queue);
1255
1256                 /* Setup the chained descriptor addresses */
1257                 if (priv->mode == STMMAC_CHAIN_MODE) {
1258                         if (priv->extend_desc)
1259                                 priv->hw->mode->init(rx_q->dma_erx,
1260                                                      rx_q->dma_rx_phy,
1261                                                      DMA_RX_SIZE, 1);
1262                         else
1263                                 priv->hw->mode->init(rx_q->dma_rx,
1264                                                      rx_q->dma_rx_phy,
1265                                                      DMA_RX_SIZE, 0);
1266                 }
1267         }
1268
1269         buf_sz = bfsize;
1270
1271         return 0;
1272
1273 err_init_rx_buffers:
1274         while (queue >= 0) {
1275                 while (--i >= 0)
1276                         stmmac_free_rx_buffer(priv, queue, i);
1277
1278                 if (queue == 0)
1279                         break;
1280
1281                 i = DMA_RX_SIZE;
1282                 queue--;
1283         }
1284
1285         return ret;
1286 }
1287
1288 /**
1289  * init_dma_tx_desc_rings - init the TX descriptor rings
1290  * @dev: net device structure.
1291  * Description: this function initializes the DMA TX descriptors
1292  * and allocates the socket buffers. It supports the chained and ring
1293  * modes.
1294  */
1295 static int init_dma_tx_desc_rings(struct net_device *dev)
1296 {
1297         struct stmmac_priv *priv = netdev_priv(dev);
1298         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1299         u32 queue;
1300         int i;
1301
1302         for (queue = 0; queue < tx_queue_cnt; queue++) {
1303                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1304
1305                 netif_dbg(priv, probe, priv->dev,
1306                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1307                          (u32)tx_q->dma_tx_phy);
1308
1309                 /* Setup the chained descriptor addresses */
1310                 if (priv->mode == STMMAC_CHAIN_MODE) {
1311                         if (priv->extend_desc)
1312                                 priv->hw->mode->init(tx_q->dma_etx,
1313                                                      tx_q->dma_tx_phy,
1314                                                      DMA_TX_SIZE, 1);
1315                         else
1316                                 priv->hw->mode->init(tx_q->dma_tx,
1317                                                      tx_q->dma_tx_phy,
1318                                                      DMA_TX_SIZE, 0);
1319                 }
1320
1321                 for (i = 0; i < DMA_TX_SIZE; i++) {
1322                         struct dma_desc *p;
1323                         if (priv->extend_desc)
1324                                 p = &((tx_q->dma_etx + i)->basic);
1325                         else
1326                                 p = tx_q->dma_tx + i;
1327
1328                         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1329                                 p->des0 = 0;
1330                                 p->des1 = 0;
1331                                 p->des2 = 0;
1332                                 p->des3 = 0;
1333                         } else {
1334                                 p->des2 = 0;
1335                         }
1336
1337                         tx_q->tx_skbuff_dma[i].buf = 0;
1338                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1339                         tx_q->tx_skbuff_dma[i].len = 0;
1340                         tx_q->tx_skbuff_dma[i].last_segment = false;
1341                         tx_q->tx_skbuff[i] = NULL;
1342                 }
1343
1344                 tx_q->dirty_tx = 0;
1345                 tx_q->cur_tx = 0;
1346
1347                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1348         }
1349
1350         return 0;
1351 }
1352
1353 /**
1354  * init_dma_desc_rings - init the RX/TX descriptor rings
1355  * @dev: net device structure
1356  * @flags: gfp flag.
1357  * Description: this function initializes the DMA RX/TX descriptors
1358  * and allocates the socket buffers. It supports the chained and ring
1359  * modes.
1360  */
1361 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1362 {
1363         struct stmmac_priv *priv = netdev_priv(dev);
1364         int ret;
1365
1366         ret = init_dma_rx_desc_rings(dev, flags);
1367         if (ret)
1368                 return ret;
1369
1370         ret = init_dma_tx_desc_rings(dev);
1371
1372         stmmac_clear_descriptors(priv);
1373
1374         if (netif_msg_hw(priv))
1375                 stmmac_display_rings(priv);
1376
1377         return ret;
1378 }
1379
1380 /**
1381  * dma_free_rx_skbufs - free RX dma buffers
1382  * @priv: private structure
1383  * @queue: RX queue index
1384  */
1385 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1386 {
1387         int i;
1388
1389         for (i = 0; i < DMA_RX_SIZE; i++)
1390                 stmmac_free_rx_buffer(priv, queue, i);
1391 }
1392
1393 /**
1394  * dma_free_tx_skbufs - free TX dma buffers
1395  * @priv: private structure
1396  * @queue: TX queue index
1397  */
1398 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1399 {
1400         int i;
1401
1402         for (i = 0; i < DMA_TX_SIZE; i++)
1403                 stmmac_free_tx_buffer(priv, queue, i);
1404 }
1405
1406 /**
1407  * free_dma_rx_desc_resources - free RX dma desc resources
1408  * @priv: private structure
1409  */
1410 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1411 {
1412         u32 rx_count = priv->plat->rx_queues_to_use;
1413         u32 queue;
1414
1415         /* Free RX queue resources */
1416         for (queue = 0; queue < rx_count; queue++) {
1417                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1418
1419                 /* Release the DMA RX socket buffers */
1420                 dma_free_rx_skbufs(priv, queue);
1421
1422                 /* Free DMA regions of consistent memory previously allocated */
1423                 if (!priv->extend_desc)
1424                         dma_free_coherent(priv->device,
1425                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1426                                           rx_q->dma_rx, rx_q->dma_rx_phy);
1427                 else
1428                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1429                                           sizeof(struct dma_extended_desc),
1430                                           rx_q->dma_erx, rx_q->dma_rx_phy);
1431
1432                 kfree(rx_q->rx_skbuff_dma);
1433                 kfree(rx_q->rx_skbuff);
1434         }
1435 }
1436
1437 /**
1438  * free_dma_tx_desc_resources - free TX dma desc resources
1439  * @priv: private structure
1440  */
1441 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1442 {
1443         u32 tx_count = priv->plat->tx_queues_to_use;
1444         u32 queue = 0;
1445
1446         /* Free TX queue resources */
1447         for (queue = 0; queue < tx_count; queue++) {
1448                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1449
1450                 /* Release the DMA TX socket buffers */
1451                 dma_free_tx_skbufs(priv, queue);
1452
1453                 /* Free DMA regions of consistent memory previously allocated */
1454                 if (!priv->extend_desc)
1455                         dma_free_coherent(priv->device,
1456                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1457                                           tx_q->dma_tx, tx_q->dma_tx_phy);
1458                 else
1459                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1460                                           sizeof(struct dma_extended_desc),
1461                                           tx_q->dma_etx, tx_q->dma_tx_phy);
1462
1463                 kfree(tx_q->tx_skbuff_dma);
1464                 kfree(tx_q->tx_skbuff);
1465         }
1466 }
1467
1468 /**
1469  * alloc_dma_rx_desc_resources - alloc RX resources.
1470  * @priv: private structure
1471  * Description: according to which descriptor can be used (extend or basic)
1472  * this function allocates the resources for TX and RX paths. In case of
1473  * reception, for example, it pre-allocated the RX socket buffer in order to
1474  * allow zero-copy mechanism.
1475  */
1476 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1477 {
1478         u32 rx_count = priv->plat->rx_queues_to_use;
1479         int ret = -ENOMEM;
1480         u32 queue;
1481
1482         /* RX queues buffers and DMA */
1483         for (queue = 0; queue < rx_count; queue++) {
1484                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1485
1486                 rx_q->queue_index = queue;
1487                 rx_q->priv_data = priv;
1488
1489                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1490                                                     sizeof(dma_addr_t),
1491                                                     GFP_KERNEL);
1492                 if (!rx_q->rx_skbuff_dma)
1493                         return -ENOMEM;
1494
1495                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1496                                                 sizeof(struct sk_buff *),
1497                                                 GFP_KERNEL);
1498                 if (!rx_q->rx_skbuff)
1499                         goto err_dma;
1500
1501                 if (priv->extend_desc) {
1502                         rx_q->dma_erx = dma_zalloc_coherent(priv->device,
1503                                                             DMA_RX_SIZE *
1504                                                             sizeof(struct
1505                                                             dma_extended_desc),
1506                                                             &rx_q->dma_rx_phy,
1507                                                             GFP_KERNEL);
1508                         if (!rx_q->dma_erx)
1509                                 goto err_dma;
1510
1511                 } else {
1512                         rx_q->dma_rx = dma_zalloc_coherent(priv->device,
1513                                                            DMA_RX_SIZE *
1514                                                            sizeof(struct
1515                                                            dma_desc),
1516                                                            &rx_q->dma_rx_phy,
1517                                                            GFP_KERNEL);
1518                         if (!rx_q->dma_rx)
1519                                 goto err_dma;
1520                 }
1521         }
1522
1523         return 0;
1524
1525 err_dma:
1526         free_dma_rx_desc_resources(priv);
1527
1528         return ret;
1529 }
1530
1531 /**
1532  * alloc_dma_tx_desc_resources - alloc TX resources.
1533  * @priv: private structure
1534  * Description: according to which descriptor can be used (extend or basic)
1535  * this function allocates the resources for TX and RX paths. In case of
1536  * reception, for example, it pre-allocated the RX socket buffer in order to
1537  * allow zero-copy mechanism.
1538  */
1539 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1540 {
1541         u32 tx_count = priv->plat->tx_queues_to_use;
1542         int ret = -ENOMEM;
1543         u32 queue;
1544
1545         /* TX queues buffers and DMA */
1546         for (queue = 0; queue < tx_count; queue++) {
1547                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1548
1549                 tx_q->queue_index = queue;
1550                 tx_q->priv_data = priv;
1551
1552                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1553                                                     sizeof(*tx_q->tx_skbuff_dma),
1554                                                     GFP_KERNEL);
1555                 if (!tx_q->tx_skbuff_dma)
1556                         return -ENOMEM;
1557
1558                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1559                                                 sizeof(struct sk_buff *),
1560                                                 GFP_KERNEL);
1561                 if (!tx_q->tx_skbuff)
1562                         goto err_dma_buffers;
1563
1564                 if (priv->extend_desc) {
1565                         tx_q->dma_etx = dma_zalloc_coherent(priv->device,
1566                                                             DMA_TX_SIZE *
1567                                                             sizeof(struct
1568                                                             dma_extended_desc),
1569                                                             &tx_q->dma_tx_phy,
1570                                                             GFP_KERNEL);
1571                         if (!tx_q->dma_etx)
1572                                 goto err_dma_buffers;
1573                 } else {
1574                         tx_q->dma_tx = dma_zalloc_coherent(priv->device,
1575                                                            DMA_TX_SIZE *
1576                                                            sizeof(struct
1577                                                                   dma_desc),
1578                                                            &tx_q->dma_tx_phy,
1579                                                            GFP_KERNEL);
1580                         if (!tx_q->dma_tx)
1581                                 goto err_dma_buffers;
1582                 }
1583         }
1584
1585         return 0;
1586
1587 err_dma_buffers:
1588         free_dma_tx_desc_resources(priv);
1589
1590         return ret;
1591 }
1592
1593 /**
1594  * alloc_dma_desc_resources - alloc TX/RX resources.
1595  * @priv: private structure
1596  * Description: according to which descriptor can be used (extend or basic)
1597  * this function allocates the resources for TX and RX paths. In case of
1598  * reception, for example, it pre-allocated the RX socket buffer in order to
1599  * allow zero-copy mechanism.
1600  */
1601 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1602 {
1603         /* RX Allocation */
1604         int ret = alloc_dma_rx_desc_resources(priv);
1605
1606         if (ret)
1607                 return ret;
1608
1609         ret = alloc_dma_tx_desc_resources(priv);
1610
1611         return ret;
1612 }
1613
1614 /**
1615  * free_dma_desc_resources - free dma desc resources
1616  * @priv: private structure
1617  */
1618 static void free_dma_desc_resources(struct stmmac_priv *priv)
1619 {
1620         /* Release the DMA RX socket buffers */
1621         free_dma_rx_desc_resources(priv);
1622
1623         /* Release the DMA TX socket buffers */
1624         free_dma_tx_desc_resources(priv);
1625 }
1626
1627 /**
1628  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1629  *  @priv: driver private structure
1630  *  Description: It is used for enabling the rx queues in the MAC
1631  */
1632 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1633 {
1634         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1635         int queue;
1636         u8 mode;
1637
1638         for (queue = 0; queue < rx_queues_count; queue++) {
1639                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1640                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1641         }
1642 }
1643
1644 /**
1645  * stmmac_start_rx_dma - start RX DMA channel
1646  * @priv: driver private structure
1647  * @chan: RX channel index
1648  * Description:
1649  * This starts a RX DMA channel
1650  */
1651 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1652 {
1653         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1654         priv->hw->dma->start_rx(priv->ioaddr, chan);
1655 }
1656
1657 /**
1658  * stmmac_start_tx_dma - start TX DMA channel
1659  * @priv: driver private structure
1660  * @chan: TX channel index
1661  * Description:
1662  * This starts a TX DMA channel
1663  */
1664 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1665 {
1666         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1667         priv->hw->dma->start_tx(priv->ioaddr, chan);
1668 }
1669
1670 /**
1671  * stmmac_stop_rx_dma - stop RX DMA channel
1672  * @priv: driver private structure
1673  * @chan: RX channel index
1674  * Description:
1675  * This stops a RX DMA channel
1676  */
1677 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1678 {
1679         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1680         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1681 }
1682
1683 /**
1684  * stmmac_stop_tx_dma - stop TX DMA channel
1685  * @priv: driver private structure
1686  * @chan: TX channel index
1687  * Description:
1688  * This stops a TX DMA channel
1689  */
1690 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1691 {
1692         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1693         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1694 }
1695
1696 /**
1697  * stmmac_start_all_dma - start all RX and TX DMA channels
1698  * @priv: driver private structure
1699  * Description:
1700  * This starts all the RX and TX DMA channels
1701  */
1702 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1703 {
1704         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1705         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1706         u32 chan = 0;
1707
1708         for (chan = 0; chan < rx_channels_count; chan++)
1709                 stmmac_start_rx_dma(priv, chan);
1710
1711         for (chan = 0; chan < tx_channels_count; chan++)
1712                 stmmac_start_tx_dma(priv, chan);
1713 }
1714
1715 /**
1716  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1717  * @priv: driver private structure
1718  * Description:
1719  * This stops the RX and TX DMA channels
1720  */
1721 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1722 {
1723         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1724         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1725         u32 chan = 0;
1726
1727         for (chan = 0; chan < rx_channels_count; chan++)
1728                 stmmac_stop_rx_dma(priv, chan);
1729
1730         for (chan = 0; chan < tx_channels_count; chan++)
1731                 stmmac_stop_tx_dma(priv, chan);
1732 }
1733
1734 /**
1735  *  stmmac_dma_operation_mode - HW DMA operation mode
1736  *  @priv: driver private structure
1737  *  Description: it is used for configuring the DMA operation mode register in
1738  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1739  */
1740 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1741 {
1742         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1743         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1744         int rxfifosz = priv->plat->rx_fifo_size;
1745         u32 txmode = 0;
1746         u32 rxmode = 0;
1747         u32 chan = 0;
1748
1749         if (rxfifosz == 0)
1750                 rxfifosz = priv->dma_cap.rx_fifo_size;
1751
1752         if (priv->plat->force_thresh_dma_mode) {
1753                 txmode = tc;
1754                 rxmode = tc;
1755         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1756                 /*
1757                  * In case of GMAC, SF mode can be enabled
1758                  * to perform the TX COE in HW. This depends on:
1759                  * 1) TX COE if actually supported
1760                  * 2) There is no bugged Jumbo frame support
1761                  *    that needs to not insert csum in the TDES.
1762                  */
1763                 txmode = SF_DMA_MODE;
1764                 rxmode = SF_DMA_MODE;
1765                 priv->xstats.threshold = SF_DMA_MODE;
1766         } else {
1767                 txmode = tc;
1768                 rxmode = SF_DMA_MODE;
1769         }
1770
1771         /* configure all channels */
1772         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1773                 for (chan = 0; chan < rx_channels_count; chan++)
1774                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1775                                                    rxfifosz);
1776
1777                 for (chan = 0; chan < tx_channels_count; chan++)
1778                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1779         } else {
1780                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1781                                         rxfifosz);
1782         }
1783 }
1784
1785 /**
1786  * stmmac_tx_clean - to manage the transmission completion
1787  * @priv: driver private structure
1788  * @queue: TX queue index
1789  * Description: it reclaims the transmit resources after transmission completes.
1790  */
1791 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1792 {
1793         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1794         unsigned int bytes_compl = 0, pkts_compl = 0;
1795         unsigned int entry = tx_q->dirty_tx;
1796
1797         netif_tx_lock(priv->dev);
1798
1799         priv->xstats.tx_clean++;
1800
1801         while (entry != tx_q->cur_tx) {
1802                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1803                 struct dma_desc *p;
1804                 int status;
1805
1806                 if (priv->extend_desc)
1807                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1808                 else
1809                         p = tx_q->dma_tx + entry;
1810
1811                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1812                                                       &priv->xstats, p,
1813                                                       priv->ioaddr);
1814                 /* Check if the descriptor is owned by the DMA */
1815                 if (unlikely(status & tx_dma_own))
1816                         break;
1817
1818                 /* Just consider the last segment and ...*/
1819                 if (likely(!(status & tx_not_ls))) {
1820                         /* ... verify the status error condition */
1821                         if (unlikely(status & tx_err)) {
1822                                 priv->dev->stats.tx_errors++;
1823                         } else {
1824                                 priv->dev->stats.tx_packets++;
1825                                 priv->xstats.tx_pkt_n++;
1826                         }
1827                         stmmac_get_tx_hwtstamp(priv, p, skb);
1828                 }
1829
1830                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1831                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1832                                 dma_unmap_page(priv->device,
1833                                                tx_q->tx_skbuff_dma[entry].buf,
1834                                                tx_q->tx_skbuff_dma[entry].len,
1835                                                DMA_TO_DEVICE);
1836                         else
1837                                 dma_unmap_single(priv->device,
1838                                                  tx_q->tx_skbuff_dma[entry].buf,
1839                                                  tx_q->tx_skbuff_dma[entry].len,
1840                                                  DMA_TO_DEVICE);
1841                         tx_q->tx_skbuff_dma[entry].buf = 0;
1842                         tx_q->tx_skbuff_dma[entry].len = 0;
1843                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1844                 }
1845
1846                 if (priv->hw->mode->clean_desc3)
1847                         priv->hw->mode->clean_desc3(tx_q, p);
1848
1849                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1850                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1851
1852                 if (likely(skb != NULL)) {
1853                         pkts_compl++;
1854                         bytes_compl += skb->len;
1855                         dev_consume_skb_any(skb);
1856                         tx_q->tx_skbuff[entry] = NULL;
1857                 }
1858
1859                 priv->hw->desc->release_tx_desc(p, priv->mode);
1860
1861                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1862         }
1863         tx_q->dirty_tx = entry;
1864
1865         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1866                                   pkts_compl, bytes_compl);
1867
1868         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1869                                                                 queue))) &&
1870             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1871
1872                 netif_dbg(priv, tx_done, priv->dev,
1873                           "%s: restart transmit\n", __func__);
1874                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1875         }
1876
1877         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1878                 stmmac_enable_eee_mode(priv);
1879                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1880         }
1881         netif_tx_unlock(priv->dev);
1882 }
1883
1884 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1885 {
1886         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1887 }
1888
1889 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1890 {
1891         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1892 }
1893
1894 /**
1895  * stmmac_tx_err - to manage the tx error
1896  * @priv: driver private structure
1897  * @chan: channel index
1898  * Description: it cleans the descriptors and restarts the transmission
1899  * in case of transmission errors.
1900  */
1901 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1902 {
1903         struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1904         int i;
1905
1906         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1907
1908         stmmac_stop_tx_dma(priv, chan);
1909         dma_free_tx_skbufs(priv, chan);
1910         for (i = 0; i < DMA_TX_SIZE; i++)
1911                 if (priv->extend_desc)
1912                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1913                                                      priv->mode,
1914                                                      (i == DMA_TX_SIZE - 1));
1915                 else
1916                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1917                                                      priv->mode,
1918                                                      (i == DMA_TX_SIZE - 1));
1919         tx_q->dirty_tx = 0;
1920         tx_q->cur_tx = 0;
1921         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
1922         stmmac_start_tx_dma(priv, chan);
1923
1924         priv->dev->stats.tx_errors++;
1925         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
1926 }
1927
1928 /**
1929  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1930  *  @priv: driver private structure
1931  *  @txmode: TX operating mode
1932  *  @rxmode: RX operating mode
1933  *  @chan: channel index
1934  *  Description: it is used for configuring of the DMA operation mode in
1935  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1936  *  mode.
1937  */
1938 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1939                                           u32 rxmode, u32 chan)
1940 {
1941         int rxfifosz = priv->plat->rx_fifo_size;
1942
1943         if (rxfifosz == 0)
1944                 rxfifosz = priv->dma_cap.rx_fifo_size;
1945
1946         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1947                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1948                                            rxfifosz);
1949                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1950         } else {
1951                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1952                                         rxfifosz);
1953         }
1954 }
1955
1956 /**
1957  * stmmac_dma_interrupt - DMA ISR
1958  * @priv: driver private structure
1959  * Description: this is the DMA ISR. It is called by the main ISR.
1960  * It calls the dwmac dma routine and schedule poll method in case of some
1961  * work can be done.
1962  */
1963 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
1964 {
1965         u32 tx_channel_count = priv->plat->tx_queues_to_use;
1966         int status;
1967         u32 chan;
1968
1969         for (chan = 0; chan < tx_channel_count; chan++) {
1970                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
1971
1972                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
1973                                                       &priv->xstats, chan);
1974                 if (likely((status & handle_rx)) || (status & handle_tx)) {
1975                         if (likely(napi_schedule_prep(&rx_q->napi))) {
1976                                 stmmac_disable_dma_irq(priv, chan);
1977                                 __napi_schedule(&rx_q->napi);
1978                         }
1979                 }
1980
1981                 if (unlikely(status & tx_hard_error_bump_tc)) {
1982                         /* Try to bump up the dma threshold on this failure */
1983                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
1984                             (tc <= 256)) {
1985                                 tc += 64;
1986                                 if (priv->plat->force_thresh_dma_mode)
1987                                         stmmac_set_dma_operation_mode(priv,
1988                                                                       tc,
1989                                                                       tc,
1990                                                                       chan);
1991                                 else
1992                                         stmmac_set_dma_operation_mode(priv,
1993                                                                     tc,
1994                                                                     SF_DMA_MODE,
1995                                                                     chan);
1996                                 priv->xstats.threshold = tc;
1997                         }
1998                 } else if (unlikely(status == tx_hard_error)) {
1999                         stmmac_tx_err(priv, chan);
2000                 }
2001         }
2002 }
2003
2004 /**
2005  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2006  * @priv: driver private structure
2007  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2008  */
2009 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2010 {
2011         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2012                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2013
2014         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2015                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
2016                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
2017         } else {
2018                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
2019                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
2020         }
2021
2022         dwmac_mmc_intr_all_mask(priv->mmcaddr);
2023
2024         if (priv->dma_cap.rmon) {
2025                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
2026                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2027         } else
2028                 netdev_info(priv->dev, "No MAC Management Counters available\n");
2029 }
2030
2031 /**
2032  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
2033  * @priv: driver private structure
2034  * Description: select the Enhanced/Alternate or Normal descriptors.
2035  * In case of Enhanced/Alternate, it checks if the extended descriptors are
2036  * supported by the HW capability register.
2037  */
2038 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
2039 {
2040         if (priv->plat->enh_desc) {
2041                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
2042
2043                 /* GMAC older than 3.50 has no extended descriptors */
2044                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
2045                         dev_info(priv->device, "Enabled extended descriptors\n");
2046                         priv->extend_desc = 1;
2047                 } else
2048                         dev_warn(priv->device, "Extended descriptors not supported\n");
2049
2050                 priv->hw->desc = &enh_desc_ops;
2051         } else {
2052                 dev_info(priv->device, "Normal descriptors\n");
2053                 priv->hw->desc = &ndesc_ops;
2054         }
2055 }
2056
2057 /**
2058  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2059  * @priv: driver private structure
2060  * Description:
2061  *  new GMAC chip generations have a new register to indicate the
2062  *  presence of the optional feature/functions.
2063  *  This can be also used to override the value passed through the
2064  *  platform and necessary for old MAC10/100 and GMAC chips.
2065  */
2066 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2067 {
2068         u32 ret = 0;
2069
2070         if (priv->hw->dma->get_hw_feature) {
2071                 priv->hw->dma->get_hw_feature(priv->ioaddr,
2072                                               &priv->dma_cap);
2073                 ret = 1;
2074         }
2075
2076         return ret;
2077 }
2078
2079 /**
2080  * stmmac_check_ether_addr - check if the MAC addr is valid
2081  * @priv: driver private structure
2082  * Description:
2083  * it is to verify if the MAC address is valid, in case of failures it
2084  * generates a random MAC address
2085  */
2086 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2087 {
2088         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2089                 priv->hw->mac->get_umac_addr(priv->hw,
2090                                              priv->dev->dev_addr, 0);
2091                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2092                         eth_hw_addr_random(priv->dev);
2093                 netdev_info(priv->dev, "device MAC address %pM\n",
2094                             priv->dev->dev_addr);
2095         }
2096 }
2097
2098 /**
2099  * stmmac_init_dma_engine - DMA init.
2100  * @priv: driver private structure
2101  * Description:
2102  * It inits the DMA invoking the specific MAC/GMAC callback.
2103  * Some DMA parameters can be passed from the platform;
2104  * in case of these are not passed a default is kept for the MAC or GMAC.
2105  */
2106 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2107 {
2108         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2109         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2110         struct stmmac_rx_queue *rx_q;
2111         struct stmmac_tx_queue *tx_q;
2112         u32 dummy_dma_rx_phy = 0;
2113         u32 dummy_dma_tx_phy = 0;
2114         u32 chan = 0;
2115         int atds = 0;
2116         int ret = 0;
2117
2118         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2119                 dev_err(priv->device, "Invalid DMA configuration\n");
2120                 return -EINVAL;
2121         }
2122
2123         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2124                 atds = 1;
2125
2126         ret = priv->hw->dma->reset(priv->ioaddr);
2127         if (ret) {
2128                 dev_err(priv->device, "Failed to reset the dma\n");
2129                 return ret;
2130         }
2131
2132         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2133                 /* DMA Configuration */
2134                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2135                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2136
2137                 /* DMA RX Channel Configuration */
2138                 for (chan = 0; chan < rx_channels_count; chan++) {
2139                         rx_q = &priv->rx_queue[chan];
2140
2141                         priv->hw->dma->init_rx_chan(priv->ioaddr,
2142                                                     priv->plat->dma_cfg,
2143                                                     rx_q->dma_rx_phy, chan);
2144
2145                         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2146                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
2147                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
2148                                                        rx_q->rx_tail_addr,
2149                                                        chan);
2150                 }
2151
2152                 /* DMA TX Channel Configuration */
2153                 for (chan = 0; chan < tx_channels_count; chan++) {
2154                         tx_q = &priv->tx_queue[chan];
2155
2156                         priv->hw->dma->init_chan(priv->ioaddr,
2157                                                  priv->plat->dma_cfg,
2158                                                  chan);
2159
2160                         priv->hw->dma->init_tx_chan(priv->ioaddr,
2161                                                     priv->plat->dma_cfg,
2162                                                     tx_q->dma_tx_phy, chan);
2163
2164                         tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2165                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
2166                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
2167                                                        tx_q->tx_tail_addr,
2168                                                        chan);
2169                 }
2170         } else {
2171                 rx_q = &priv->rx_queue[chan];
2172                 tx_q = &priv->tx_queue[chan];
2173                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2174                                     tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
2175         }
2176
2177         if (priv->plat->axi && priv->hw->dma->axi)
2178                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
2179
2180         return ret;
2181 }
2182
2183 /**
2184  * stmmac_tx_timer - mitigation sw timer for tx.
2185  * @data: data pointer
2186  * Description:
2187  * This is the timer handler to directly invoke the stmmac_tx_clean.
2188  */
2189 static void stmmac_tx_timer(unsigned long data)
2190 {
2191         struct stmmac_priv *priv = (struct stmmac_priv *)data;
2192         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2193         u32 queue;
2194
2195         /* let's scan all the tx queues */
2196         for (queue = 0; queue < tx_queues_count; queue++)
2197                 stmmac_tx_clean(priv, queue);
2198 }
2199
2200 /**
2201  * stmmac_init_tx_coalesce - init tx mitigation options.
2202  * @priv: driver private structure
2203  * Description:
2204  * This inits the transmit coalesce parameters: i.e. timer rate,
2205  * timer handler and default threshold used for enabling the
2206  * interrupt on completion bit.
2207  */
2208 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2209 {
2210         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2211         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2212         init_timer(&priv->txtimer);
2213         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2214         priv->txtimer.data = (unsigned long)priv;
2215         priv->txtimer.function = stmmac_tx_timer;
2216         add_timer(&priv->txtimer);
2217 }
2218
2219 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2220 {
2221         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2222         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2223         u32 chan;
2224
2225         /* set TX ring length */
2226         if (priv->hw->dma->set_tx_ring_len) {
2227                 for (chan = 0; chan < tx_channels_count; chan++)
2228                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
2229                                                        (DMA_TX_SIZE - 1), chan);
2230         }
2231
2232         /* set RX ring length */
2233         if (priv->hw->dma->set_rx_ring_len) {
2234                 for (chan = 0; chan < rx_channels_count; chan++)
2235                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
2236                                                        (DMA_RX_SIZE - 1), chan);
2237         }
2238 }
2239
2240 /**
2241  *  stmmac_set_tx_queue_weight - Set TX queue weight
2242  *  @priv: driver private structure
2243  *  Description: It is used for setting TX queues weight
2244  */
2245 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2246 {
2247         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2248         u32 weight;
2249         u32 queue;
2250
2251         for (queue = 0; queue < tx_queues_count; queue++) {
2252                 weight = priv->plat->tx_queues_cfg[queue].weight;
2253                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
2254         }
2255 }
2256
2257 /**
2258  *  stmmac_configure_cbs - Configure CBS in TX queue
2259  *  @priv: driver private structure
2260  *  Description: It is used for configuring CBS in AVB TX queues
2261  */
2262 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2263 {
2264         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2265         u32 mode_to_use;
2266         u32 queue;
2267
2268         /* queue 0 is reserved for legacy traffic */
2269         for (queue = 1; queue < tx_queues_count; queue++) {
2270                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2271                 if (mode_to_use == MTL_QUEUE_DCB)
2272                         continue;
2273
2274                 priv->hw->mac->config_cbs(priv->hw,
2275                                 priv->plat->tx_queues_cfg[queue].send_slope,
2276                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2277                                 priv->plat->tx_queues_cfg[queue].high_credit,
2278                                 priv->plat->tx_queues_cfg[queue].low_credit,
2279                                 queue);
2280         }
2281 }
2282
2283 /**
2284  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2285  *  @priv: driver private structure
2286  *  Description: It is used for mapping RX queues to RX dma channels
2287  */
2288 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2289 {
2290         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2291         u32 queue;
2292         u32 chan;
2293
2294         for (queue = 0; queue < rx_queues_count; queue++) {
2295                 chan = priv->plat->rx_queues_cfg[queue].chan;
2296                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
2297         }
2298 }
2299
2300 /**
2301  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2302  *  @priv: driver private structure
2303  *  Description: It is used for configuring the RX Queue Priority
2304  */
2305 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2306 {
2307         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2308         u32 queue;
2309         u32 prio;
2310
2311         for (queue = 0; queue < rx_queues_count; queue++) {
2312                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2313                         continue;
2314
2315                 prio = priv->plat->rx_queues_cfg[queue].prio;
2316                 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
2317         }
2318 }
2319
2320 /**
2321  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2322  *  @priv: driver private structure
2323  *  Description: It is used for configuring the TX Queue Priority
2324  */
2325 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2326 {
2327         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2328         u32 queue;
2329         u32 prio;
2330
2331         for (queue = 0; queue < tx_queues_count; queue++) {
2332                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2333                         continue;
2334
2335                 prio = priv->plat->tx_queues_cfg[queue].prio;
2336                 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
2337         }
2338 }
2339
2340 /**
2341  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2342  *  @priv: driver private structure
2343  *  Description: It is used for configuring the RX queue routing
2344  */
2345 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2346 {
2347         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2348         u32 queue;
2349         u8 packet;
2350
2351         for (queue = 0; queue < rx_queues_count; queue++) {
2352                 /* no specific packet type routing specified for the queue */
2353                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2354                         continue;
2355
2356                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2357                 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue);
2358         }
2359 }
2360
2361 /**
2362  *  stmmac_mtl_configuration - Configure MTL
2363  *  @priv: driver private structure
2364  *  Description: It is used for configurring MTL
2365  */
2366 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2367 {
2368         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2369         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2370
2371         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
2372                 stmmac_set_tx_queue_weight(priv);
2373
2374         /* Configure MTL RX algorithms */
2375         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
2376                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
2377                                                 priv->plat->rx_sched_algorithm);
2378
2379         /* Configure MTL TX algorithms */
2380         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
2381                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
2382                                                 priv->plat->tx_sched_algorithm);
2383
2384         /* Configure CBS in AVB TX queues */
2385         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2386                 stmmac_configure_cbs(priv);
2387
2388         /* Map RX MTL to DMA channels */
2389         if (priv->hw->mac->map_mtl_to_dma)
2390                 stmmac_rx_queue_dma_chan_map(priv);
2391
2392         /* Enable MAC RX Queues */
2393         if (priv->hw->mac->rx_queue_enable)
2394                 stmmac_mac_enable_rx_queues(priv);
2395
2396         /* Set RX priorities */
2397         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
2398                 stmmac_mac_config_rx_queues_prio(priv);
2399
2400         /* Set TX priorities */
2401         if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
2402                 stmmac_mac_config_tx_queues_prio(priv);
2403
2404         /* Set RX routing */
2405         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
2406                 stmmac_mac_config_rx_queues_routing(priv);
2407 }
2408
2409 /**
2410  * stmmac_hw_setup - setup mac in a usable state.
2411  *  @dev : pointer to the device structure.
2412  *  Description:
2413  *  this is the main function to setup the HW in a usable state because the
2414  *  dma engine is reset, the core registers are configured (e.g. AXI,
2415  *  Checksum features, timers). The DMA is ready to start receiving and
2416  *  transmitting.
2417  *  Return value:
2418  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2419  *  file on failure.
2420  */
2421 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2422 {
2423         struct stmmac_priv *priv = netdev_priv(dev);
2424         u32 rx_cnt = priv->plat->rx_queues_to_use;
2425         u32 tx_cnt = priv->plat->tx_queues_to_use;
2426         u32 chan;
2427         int ret;
2428
2429         /* DMA initialization and SW reset */
2430         ret = stmmac_init_dma_engine(priv);
2431         if (ret < 0) {
2432                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2433                            __func__);
2434                 return ret;
2435         }
2436
2437         /* Copy the MAC addr into the HW  */
2438         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2439
2440         /* PS and related bits will be programmed according to the speed */
2441         if (priv->hw->pcs) {
2442                 int speed = priv->plat->mac_port_sel_speed;
2443
2444                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2445                     (speed == SPEED_1000)) {
2446                         priv->hw->ps = speed;
2447                 } else {
2448                         dev_warn(priv->device, "invalid port speed\n");
2449                         priv->hw->ps = 0;
2450                 }
2451         }
2452
2453         /* Initialize the MAC Core */
2454         priv->hw->mac->core_init(priv->hw, dev->mtu);
2455
2456         /* Initialize MTL*/
2457         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2458                 stmmac_mtl_configuration(priv);
2459
2460         ret = priv->hw->mac->rx_ipc(priv->hw);
2461         if (!ret) {
2462                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2463                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2464                 priv->hw->rx_csum = 0;
2465         }
2466
2467         /* Enable the MAC Rx/Tx */
2468         priv->hw->mac->set_mac(priv->ioaddr, true);
2469
2470         /* Set the HW DMA mode and the COE */
2471         stmmac_dma_operation_mode(priv);
2472
2473         stmmac_mmc_setup(priv);
2474
2475         if (init_ptp) {
2476                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2477                 if (ret < 0)
2478                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2479
2480                 ret = stmmac_init_ptp(priv);
2481                 if (ret == -EOPNOTSUPP)
2482                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2483                 else if (ret)
2484                         netdev_warn(priv->dev, "PTP init failed\n");
2485         }
2486
2487 #ifdef CONFIG_DEBUG_FS
2488         ret = stmmac_init_fs(dev);
2489         if (ret < 0)
2490                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2491                             __func__);
2492 #endif
2493         /* Start the ball rolling... */
2494         stmmac_start_all_dma(priv);
2495
2496         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2497
2498         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2499                 priv->rx_riwt = MAX_DMA_RIWT;
2500                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2501         }
2502
2503         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2504                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2505
2506         /* set TX and RX rings length */
2507         stmmac_set_rings_length(priv);
2508
2509         /* Enable TSO */
2510         if (priv->tso) {
2511                 for (chan = 0; chan < tx_cnt; chan++)
2512                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2513         }
2514
2515         return 0;
2516 }
2517
2518 static void stmmac_hw_teardown(struct net_device *dev)
2519 {
2520         struct stmmac_priv *priv = netdev_priv(dev);
2521
2522         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2523 }
2524
2525 /**
2526  *  stmmac_open - open entry point of the driver
2527  *  @dev : pointer to the device structure.
2528  *  Description:
2529  *  This function is the open entry point of the driver.
2530  *  Return value:
2531  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2532  *  file on failure.
2533  */
2534 static int stmmac_open(struct net_device *dev)
2535 {
2536         struct stmmac_priv *priv = netdev_priv(dev);
2537         int ret;
2538
2539         stmmac_check_ether_addr(priv);
2540
2541         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2542             priv->hw->pcs != STMMAC_PCS_TBI &&
2543             priv->hw->pcs != STMMAC_PCS_RTBI) {
2544                 ret = stmmac_init_phy(dev);
2545                 if (ret) {
2546                         netdev_err(priv->dev,
2547                                    "%s: Cannot attach to PHY (error: %d)\n",
2548                                    __func__, ret);
2549                         return ret;
2550                 }
2551         }
2552
2553         /* Extra statistics */
2554         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2555         priv->xstats.threshold = tc;
2556
2557         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2558         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2559
2560         ret = alloc_dma_desc_resources(priv);
2561         if (ret < 0) {
2562                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2563                            __func__);
2564                 goto dma_desc_error;
2565         }
2566
2567         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2568         if (ret < 0) {
2569                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2570                            __func__);
2571                 goto init_error;
2572         }
2573
2574         ret = stmmac_hw_setup(dev, true);
2575         if (ret < 0) {
2576                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2577                 goto init_error;
2578         }
2579
2580         stmmac_init_tx_coalesce(priv);
2581
2582         if (dev->phydev)
2583                 phy_start(dev->phydev);
2584
2585         /* Request the IRQ lines */
2586         ret = request_irq(dev->irq, stmmac_interrupt,
2587                           IRQF_SHARED, dev->name, dev);
2588         if (unlikely(ret < 0)) {
2589                 netdev_err(priv->dev,
2590                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2591                            __func__, dev->irq, ret);
2592                 goto irq_error;
2593         }
2594
2595         /* Request the Wake IRQ in case of another line is used for WoL */
2596         if (priv->wol_irq != dev->irq) {
2597                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2598                                   IRQF_SHARED, dev->name, dev);
2599                 if (unlikely(ret < 0)) {
2600                         netdev_err(priv->dev,
2601                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2602                                    __func__, priv->wol_irq, ret);
2603                         goto wolirq_error;
2604                 }
2605         }
2606
2607         /* Request the IRQ lines */
2608         if (priv->lpi_irq > 0) {
2609                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2610                                   dev->name, dev);
2611                 if (unlikely(ret < 0)) {
2612                         netdev_err(priv->dev,
2613                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2614                                    __func__, priv->lpi_irq, ret);
2615                         goto lpiirq_error;
2616                 }
2617         }
2618
2619         stmmac_enable_all_queues(priv);
2620         stmmac_start_all_queues(priv);
2621
2622         return 0;
2623
2624 lpiirq_error:
2625         if (priv->wol_irq != dev->irq)
2626                 free_irq(priv->wol_irq, dev);
2627 wolirq_error:
2628         free_irq(dev->irq, dev);
2629 irq_error:
2630         if (dev->phydev)
2631                 phy_stop(dev->phydev);
2632
2633         del_timer_sync(&priv->txtimer);
2634         stmmac_hw_teardown(dev);
2635 init_error:
2636         free_dma_desc_resources(priv);
2637 dma_desc_error:
2638         if (dev->phydev)
2639                 phy_disconnect(dev->phydev);
2640
2641         return ret;
2642 }
2643
2644 /**
2645  *  stmmac_release - close entry point of the driver
2646  *  @dev : device pointer.
2647  *  Description:
2648  *  This is the stop entry point of the driver.
2649  */
2650 static int stmmac_release(struct net_device *dev)
2651 {
2652         struct stmmac_priv *priv = netdev_priv(dev);
2653
2654         if (priv->eee_enabled)
2655                 del_timer_sync(&priv->eee_ctrl_timer);
2656
2657         /* Stop and disconnect the PHY */
2658         if (dev->phydev) {
2659                 phy_stop(dev->phydev);
2660                 phy_disconnect(dev->phydev);
2661         }
2662
2663         stmmac_stop_all_queues(priv);
2664
2665         stmmac_disable_all_queues(priv);
2666
2667         del_timer_sync(&priv->txtimer);
2668
2669         /* Free the IRQ lines */
2670         free_irq(dev->irq, dev);
2671         if (priv->wol_irq != dev->irq)
2672                 free_irq(priv->wol_irq, dev);
2673         if (priv->lpi_irq > 0)
2674                 free_irq(priv->lpi_irq, dev);
2675
2676         /* Stop TX/RX DMA and clear the descriptors */
2677         stmmac_stop_all_dma(priv);
2678
2679         /* Release and free the Rx/Tx resources */
2680         free_dma_desc_resources(priv);
2681
2682         /* Disable the MAC Rx/Tx */
2683         priv->hw->mac->set_mac(priv->ioaddr, false);
2684
2685         netif_carrier_off(dev);
2686
2687 #ifdef CONFIG_DEBUG_FS
2688         stmmac_exit_fs(dev);
2689 #endif
2690
2691         stmmac_release_ptp(priv);
2692
2693         return 0;
2694 }
2695
2696 /**
2697  *  stmmac_tso_allocator - close entry point of the driver
2698  *  @priv: driver private structure
2699  *  @des: buffer start address
2700  *  @total_len: total length to fill in descriptors
2701  *  @last_segmant: condition for the last descriptor
2702  *  @queue: TX queue index
2703  *  Description:
2704  *  This function fills descriptor and request new descriptors according to
2705  *  buffer length to fill
2706  */
2707 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2708                                  int total_len, bool last_segment, u32 queue)
2709 {
2710         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2711         struct dma_desc *desc;
2712         u32 buff_size;
2713         int tmp_len;
2714
2715         tmp_len = total_len;
2716
2717         while (tmp_len > 0) {
2718                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2719                 desc = tx_q->dma_tx + tx_q->cur_tx;
2720
2721                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2722                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2723                             TSO_MAX_BUFF_SIZE : tmp_len;
2724
2725                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2726                         0, 1,
2727                         (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2728                         0, 0);
2729
2730                 tmp_len -= TSO_MAX_BUFF_SIZE;
2731         }
2732 }
2733
2734 /**
2735  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2736  *  @skb : the socket buffer
2737  *  @dev : device pointer
2738  *  Description: this is the transmit function that is called on TSO frames
2739  *  (support available on GMAC4 and newer chips).
2740  *  Diagram below show the ring programming in case of TSO frames:
2741  *
2742  *  First Descriptor
2743  *   --------
2744  *   | DES0 |---> buffer1 = L2/L3/L4 header
2745  *   | DES1 |---> TCP Payload (can continue on next descr...)
2746  *   | DES2 |---> buffer 1 and 2 len
2747  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2748  *   --------
2749  *      |
2750  *     ...
2751  *      |
2752  *   --------
2753  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2754  *   | DES1 | --|
2755  *   | DES2 | --> buffer 1 and 2 len
2756  *   | DES3 |
2757  *   --------
2758  *
2759  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2760  */
2761 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2762 {
2763         struct dma_desc *desc, *first, *mss_desc = NULL;
2764         struct stmmac_priv *priv = netdev_priv(dev);
2765         int nfrags = skb_shinfo(skb)->nr_frags;
2766         u32 queue = skb_get_queue_mapping(skb);
2767         unsigned int first_entry, des;
2768         struct stmmac_tx_queue *tx_q;
2769         int tmp_pay_len = 0;
2770         u32 pay_len, mss;
2771         u8 proto_hdr_len;
2772         int i;
2773
2774         tx_q = &priv->tx_queue[queue];
2775
2776         /* Compute header lengths */
2777         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2778
2779         /* Desc availability based on threshold should be enough safe */
2780         if (unlikely(stmmac_tx_avail(priv, queue) <
2781                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2782                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2783                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2784                                                                 queue));
2785                         /* This is a hard error, log it. */
2786                         netdev_err(priv->dev,
2787                                    "%s: Tx Ring full when queue awake\n",
2788                                    __func__);
2789                 }
2790                 return NETDEV_TX_BUSY;
2791         }
2792
2793         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2794
2795         mss = skb_shinfo(skb)->gso_size;
2796
2797         /* set new MSS value if needed */
2798         if (mss != priv->mss) {
2799                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2800                 priv->hw->desc->set_mss(mss_desc, mss);
2801                 priv->mss = mss;
2802                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2803         }
2804
2805         if (netif_msg_tx_queued(priv)) {
2806                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2807                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2808                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2809                         skb->data_len);
2810         }
2811
2812         first_entry = tx_q->cur_tx;
2813
2814         desc = tx_q->dma_tx + first_entry;
2815         first = desc;
2816
2817         /* first descriptor: fill Headers on Buf1 */
2818         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2819                              DMA_TO_DEVICE);
2820         if (dma_mapping_error(priv->device, des))
2821                 goto dma_map_err;
2822
2823         tx_q->tx_skbuff_dma[first_entry].buf = des;
2824         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2825         tx_q->tx_skbuff[first_entry] = skb;
2826
2827         first->des0 = cpu_to_le32(des);
2828
2829         /* Fill start of payload in buff2 of first descriptor */
2830         if (pay_len)
2831                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2832
2833         /* If needed take extra descriptors to fill the remaining payload */
2834         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2835
2836         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2837
2838         /* Prepare fragments */
2839         for (i = 0; i < nfrags; i++) {
2840                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2841
2842                 des = skb_frag_dma_map(priv->device, frag, 0,
2843                                        skb_frag_size(frag),
2844                                        DMA_TO_DEVICE);
2845                 if (dma_mapping_error(priv->device, des))
2846                         goto dma_map_err;
2847
2848                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2849                                      (i == nfrags - 1), queue);
2850
2851                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2852                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2853                 tx_q->tx_skbuff[tx_q->cur_tx] = NULL;
2854                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2855         }
2856
2857         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2858
2859         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2860
2861         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2862                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2863                           __func__);
2864                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2865         }
2866
2867         dev->stats.tx_bytes += skb->len;
2868         priv->xstats.tx_tso_frames++;
2869         priv->xstats.tx_tso_nfrags += nfrags;
2870
2871         /* Manage tx mitigation */
2872         priv->tx_count_frames += nfrags + 1;
2873         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2874                 mod_timer(&priv->txtimer,
2875                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2876         } else {
2877                 priv->tx_count_frames = 0;
2878                 priv->hw->desc->set_tx_ic(desc);
2879                 priv->xstats.tx_set_ic_bit++;
2880         }
2881
2882         if (!priv->hwts_tx_en)
2883                 skb_tx_timestamp(skb);
2884
2885         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2886                      priv->hwts_tx_en)) {
2887                 /* declare that device is doing timestamping */
2888                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2889                 priv->hw->desc->enable_tx_timestamp(first);
2890         }
2891
2892         /* Complete the first descriptor before granting the DMA */
2893         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2894                         proto_hdr_len,
2895                         pay_len,
2896                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2897                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2898
2899         /* If context desc is used to change MSS */
2900         if (mss_desc)
2901                 priv->hw->desc->set_tx_owner(mss_desc);
2902
2903         /* The own bit must be the latest setting done when prepare the
2904          * descriptor and then barrier is needed to make sure that
2905          * all is coherent before granting the DMA engine.
2906          */
2907         dma_wmb();
2908
2909         if (netif_msg_pktdata(priv)) {
2910                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2911                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2912                         tx_q->cur_tx, first, nfrags);
2913
2914                 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
2915                                              0);
2916
2917                 pr_info(">>> frame to be transmitted: ");
2918                 print_pkt(skb->data, skb_headlen(skb));
2919         }
2920
2921         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2922
2923         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
2924                                        queue);
2925
2926         return NETDEV_TX_OK;
2927
2928 dma_map_err:
2929         dev_err(priv->device, "Tx dma map failed\n");
2930         dev_kfree_skb(skb);
2931         priv->dev->stats.tx_dropped++;
2932         return NETDEV_TX_OK;
2933 }
2934
2935 /**
2936  *  stmmac_xmit - Tx entry point of the driver
2937  *  @skb : the socket buffer
2938  *  @dev : device pointer
2939  *  Description : this is the tx entry point of the driver.
2940  *  It programs the chain or the ring and supports oversized frames
2941  *  and SG feature.
2942  */
2943 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2944 {
2945         struct stmmac_priv *priv = netdev_priv(dev);
2946         unsigned int nopaged_len = skb_headlen(skb);
2947         int i, csum_insertion = 0, is_jumbo = 0;
2948         u32 queue = skb_get_queue_mapping(skb);
2949         int nfrags = skb_shinfo(skb)->nr_frags;
2950         int entry;
2951         unsigned int first_entry;
2952         struct dma_desc *desc, *first;
2953         struct stmmac_tx_queue *tx_q;
2954         unsigned int enh_desc;
2955         unsigned int des;
2956
2957         tx_q = &priv->tx_queue[queue];
2958
2959         /* Manage oversized TCP frames for GMAC4 device */
2960         if (skb_is_gso(skb) && priv->tso) {
2961                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2962                         return stmmac_tso_xmit(skb, dev);
2963         }
2964
2965         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
2966                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2967                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2968                                                                 queue));
2969                         /* This is a hard error, log it. */
2970                         netdev_err(priv->dev,
2971                                    "%s: Tx Ring full when queue awake\n",
2972                                    __func__);
2973                 }
2974                 return NETDEV_TX_BUSY;
2975         }
2976
2977         if (priv->tx_path_in_lpi_mode)
2978                 stmmac_disable_eee_mode(priv);
2979
2980         entry = tx_q->cur_tx;
2981         first_entry = entry;
2982
2983         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
2984
2985         if (likely(priv->extend_desc))
2986                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2987         else
2988                 desc = tx_q->dma_tx + entry;
2989
2990         first = desc;
2991
2992         tx_q->tx_skbuff[first_entry] = skb;
2993
2994         enh_desc = priv->plat->enh_desc;
2995         /* To program the descriptors according to the size of the frame */
2996         if (enh_desc)
2997                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
2998
2999         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
3000                                          DWMAC_CORE_4_00)) {
3001                 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
3002                 if (unlikely(entry < 0))
3003                         goto dma_map_err;
3004         }
3005
3006         for (i = 0; i < nfrags; i++) {
3007                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3008                 int len = skb_frag_size(frag);
3009                 bool last_segment = (i == (nfrags - 1));
3010
3011                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3012
3013                 if (likely(priv->extend_desc))
3014                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3015                 else
3016                         desc = tx_q->dma_tx + entry;
3017
3018                 des = skb_frag_dma_map(priv->device, frag, 0, len,
3019                                        DMA_TO_DEVICE);
3020                 if (dma_mapping_error(priv->device, des))
3021                         goto dma_map_err; /* should reuse desc w/o issues */
3022
3023                 tx_q->tx_skbuff[entry] = NULL;
3024
3025                 tx_q->tx_skbuff_dma[entry].buf = des;
3026                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3027                         desc->des0 = cpu_to_le32(des);
3028                 else
3029                         desc->des2 = cpu_to_le32(des);
3030
3031                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3032                 tx_q->tx_skbuff_dma[entry].len = len;
3033                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3034
3035                 /* Prepare the descriptor and set the own bit too */
3036                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
3037                                                 priv->mode, 1, last_segment,
3038                                                 skb->len);
3039         }
3040
3041         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3042
3043         tx_q->cur_tx = entry;
3044
3045         if (netif_msg_pktdata(priv)) {
3046                 void *tx_head;
3047
3048                 netdev_dbg(priv->dev,
3049                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3050                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3051                            entry, first, nfrags);
3052
3053                 if (priv->extend_desc)
3054                         tx_head = (void *)tx_q->dma_etx;
3055                 else
3056                         tx_head = (void *)tx_q->dma_tx;
3057
3058                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
3059
3060                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3061                 print_pkt(skb->data, skb->len);
3062         }
3063
3064         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3065                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3066                           __func__);
3067                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3068         }
3069
3070         dev->stats.tx_bytes += skb->len;
3071
3072         /* According to the coalesce parameter the IC bit for the latest
3073          * segment is reset and the timer re-started to clean the tx status.
3074          * This approach takes care about the fragments: desc is the first
3075          * element in case of no SG.
3076          */
3077         priv->tx_count_frames += nfrags + 1;
3078         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
3079                 mod_timer(&priv->txtimer,
3080                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
3081         } else {
3082                 priv->tx_count_frames = 0;
3083                 priv->hw->desc->set_tx_ic(desc);
3084                 priv->xstats.tx_set_ic_bit++;
3085         }
3086
3087         if (!priv->hwts_tx_en)
3088                 skb_tx_timestamp(skb);
3089
3090         /* Ready to fill the first descriptor and set the OWN bit w/o any
3091          * problems because all the descriptors are actually ready to be
3092          * passed to the DMA engine.
3093          */
3094         if (likely(!is_jumbo)) {
3095                 bool last_segment = (nfrags == 0);
3096
3097                 des = dma_map_single(priv->device, skb->data,
3098                                      nopaged_len, DMA_TO_DEVICE);
3099                 if (dma_mapping_error(priv->device, des))
3100                         goto dma_map_err;
3101
3102                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3103                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3104                         first->des0 = cpu_to_le32(des);
3105                 else
3106                         first->des2 = cpu_to_le32(des);
3107
3108                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3109                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3110
3111                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3112                              priv->hwts_tx_en)) {
3113                         /* declare that device is doing timestamping */
3114                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3115                         priv->hw->desc->enable_tx_timestamp(first);
3116                 }
3117
3118                 /* Prepare the first descriptor setting the OWN bit too */
3119                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
3120                                                 csum_insertion, priv->mode, 1,
3121                                                 last_segment, skb->len);
3122
3123                 /* The own bit must be the latest setting done when prepare the
3124                  * descriptor and then barrier is needed to make sure that
3125                  * all is coherent before granting the DMA engine.
3126                  */
3127                 dma_wmb();
3128         }
3129
3130         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3131
3132         if (priv->synopsys_id < DWMAC_CORE_4_00)
3133                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
3134         else
3135                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
3136                                                queue);
3137
3138         return NETDEV_TX_OK;
3139
3140 dma_map_err:
3141         netdev_err(priv->dev, "Tx DMA map failed\n");
3142         dev_kfree_skb(skb);
3143         priv->dev->stats.tx_dropped++;
3144         return NETDEV_TX_OK;
3145 }
3146
3147 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3148 {
3149         struct ethhdr *ehdr;
3150         u16 vlanid;
3151
3152         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
3153             NETIF_F_HW_VLAN_CTAG_RX &&
3154             !__vlan_get_tag(skb, &vlanid)) {
3155                 /* pop the vlan tag */
3156                 ehdr = (struct ethhdr *)skb->data;
3157                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
3158                 skb_pull(skb, VLAN_HLEN);
3159                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
3160         }
3161 }
3162
3163
3164 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3165 {
3166         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3167                 return 0;
3168
3169         return 1;
3170 }
3171
3172 /**
3173  * stmmac_rx_refill - refill used skb preallocated buffers
3174  * @priv: driver private structure
3175  * @queue: RX queue index
3176  * Description : this is to reallocate the skb for the reception process
3177  * that is based on zero-copy.
3178  */
3179 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3180 {
3181         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3182         int dirty = stmmac_rx_dirty(priv, queue);
3183         unsigned int entry = rx_q->dirty_rx;
3184
3185         int bfsize = priv->dma_buf_sz;
3186
3187         while (dirty-- > 0) {
3188                 struct dma_desc *p;
3189
3190                 if (priv->extend_desc)
3191                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3192                 else
3193                         p = rx_q->dma_rx + entry;
3194
3195                 if (likely(!rx_q->rx_skbuff[entry])) {
3196                         struct sk_buff *skb;
3197
3198                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3199                         if (unlikely(!skb)) {
3200                                 /* so for a while no zero-copy! */
3201                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3202                                 if (unlikely(net_ratelimit()))
3203                                         dev_err(priv->device,
3204                                                 "fail to alloc skb entry %d\n",
3205                                                 entry);
3206                                 break;
3207                         }
3208
3209                         rx_q->rx_skbuff[entry] = skb;
3210                         rx_q->rx_skbuff_dma[entry] =
3211                             dma_map_single(priv->device, skb->data, bfsize,
3212                                            DMA_FROM_DEVICE);
3213                         if (dma_mapping_error(priv->device,
3214                                               rx_q->rx_skbuff_dma[entry])) {
3215                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3216                                 dev_kfree_skb(skb);
3217                                 break;
3218                         }
3219
3220                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3221                                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3222                                 p->des1 = 0;
3223                         } else {
3224                                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3225                         }
3226                         if (priv->hw->mode->refill_desc3)
3227                                 priv->hw->mode->refill_desc3(rx_q, p);
3228
3229                         if (rx_q->rx_zeroc_thresh > 0)
3230                                 rx_q->rx_zeroc_thresh--;
3231
3232                         netif_dbg(priv, rx_status, priv->dev,
3233                                   "refill entry #%d\n", entry);
3234                 }
3235                 dma_wmb();
3236
3237                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3238                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
3239                 else
3240                         priv->hw->desc->set_rx_owner(p);
3241
3242                 dma_wmb();
3243
3244                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3245         }
3246         rx_q->dirty_rx = entry;
3247 }
3248
3249 /**
3250  * stmmac_rx - manage the receive process
3251  * @priv: driver private structure
3252  * @limit: napi bugget
3253  * @queue: RX queue index.
3254  * Description :  this the function called by the napi poll method.
3255  * It gets all the frames inside the ring.
3256  */
3257 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3258 {
3259         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3260         unsigned int entry = rx_q->cur_rx;
3261         int coe = priv->hw->rx_csum;
3262         unsigned int next_entry;
3263         unsigned int count = 0;
3264
3265         if (netif_msg_rx_status(priv)) {
3266                 void *rx_head;
3267
3268                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3269                 if (priv->extend_desc)
3270                         rx_head = (void *)rx_q->dma_erx;
3271                 else
3272                         rx_head = (void *)rx_q->dma_rx;
3273
3274                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
3275         }
3276         while (count < limit) {
3277                 int status;
3278                 struct dma_desc *p;
3279                 struct dma_desc *np;
3280
3281                 if (priv->extend_desc)
3282                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3283                 else
3284                         p = rx_q->dma_rx + entry;
3285
3286                 /* read the status of the incoming frame */
3287                 status = priv->hw->desc->rx_status(&priv->dev->stats,
3288                                                    &priv->xstats, p);
3289                 /* check if managed by the DMA otherwise go ahead */
3290                 if (unlikely(status & dma_own))
3291                         break;
3292
3293                 count++;
3294
3295                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3296                 next_entry = rx_q->cur_rx;
3297
3298                 if (priv->extend_desc)
3299                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3300                 else
3301                         np = rx_q->dma_rx + next_entry;
3302
3303                 prefetch(np);
3304
3305                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
3306                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
3307                                                            &priv->xstats,
3308                                                            rx_q->dma_erx +
3309                                                            entry);
3310                 if (unlikely(status == discard_frame)) {
3311                         priv->dev->stats.rx_errors++;
3312                         if (priv->hwts_rx_en && !priv->extend_desc) {
3313                                 /* DESC2 & DESC3 will be overwritten by device
3314                                  * with timestamp value, hence reinitialize
3315                                  * them in stmmac_rx_refill() function so that
3316                                  * device can reuse it.
3317                                  */
3318                                 rx_q->rx_skbuff[entry] = NULL;
3319                                 dma_unmap_single(priv->device,
3320                                                  rx_q->rx_skbuff_dma[entry],
3321                                                  priv->dma_buf_sz,
3322                                                  DMA_FROM_DEVICE);
3323                         }
3324                 } else {
3325                         struct sk_buff *skb;
3326                         int frame_len;
3327                         unsigned int des;
3328
3329                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3330                                 des = le32_to_cpu(p->des0);
3331                         else
3332                                 des = le32_to_cpu(p->des2);
3333
3334                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
3335
3336                         /*  If frame length is greater than skb buffer size
3337                          *  (preallocated during init) then the packet is
3338                          *  ignored
3339                          */
3340                         if (frame_len > priv->dma_buf_sz) {
3341                                 netdev_err(priv->dev,
3342                                            "len %d larger than size (%d)\n",
3343                                            frame_len, priv->dma_buf_sz);
3344                                 priv->dev->stats.rx_length_errors++;
3345                                 break;
3346                         }
3347
3348                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3349                          * Type frames (LLC/LLC-SNAP)
3350                          */
3351                         if (unlikely(status != llc_snap))
3352                                 frame_len -= ETH_FCS_LEN;
3353
3354                         if (netif_msg_rx_status(priv)) {
3355                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3356                                            p, entry, des);
3357                                 if (frame_len > ETH_FRAME_LEN)
3358                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3359                                                    frame_len, status);
3360                         }
3361
3362                         /* The zero-copy is always used for all the sizes
3363                          * in case of GMAC4 because it needs
3364                          * to refill the used descriptors, always.
3365                          */
3366                         if (unlikely(!priv->plat->has_gmac4 &&
3367                                      ((frame_len < priv->rx_copybreak) ||
3368                                      stmmac_rx_threshold_count(rx_q)))) {
3369                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3370                                                                 frame_len);
3371                                 if (unlikely(!skb)) {
3372                                         if (net_ratelimit())
3373                                                 dev_warn(priv->device,
3374                                                          "packet dropped\n");
3375                                         priv->dev->stats.rx_dropped++;
3376                                         break;
3377                                 }
3378
3379                                 dma_sync_single_for_cpu(priv->device,
3380                                                         rx_q->rx_skbuff_dma
3381                                                         [entry], frame_len,
3382                                                         DMA_FROM_DEVICE);
3383                                 skb_copy_to_linear_data(skb,
3384                                                         rx_q->
3385                                                         rx_skbuff[entry]->data,
3386                                                         frame_len);
3387
3388                                 skb_put(skb, frame_len);
3389                                 dma_sync_single_for_device(priv->device,
3390                                                            rx_q->rx_skbuff_dma
3391                                                            [entry], frame_len,
3392                                                            DMA_FROM_DEVICE);
3393                         } else {
3394                                 skb = rx_q->rx_skbuff[entry];
3395                                 if (unlikely(!skb)) {
3396                                         netdev_err(priv->dev,
3397                                                    "%s: Inconsistent Rx chain\n",
3398                                                    priv->dev->name);
3399                                         priv->dev->stats.rx_dropped++;
3400                                         break;
3401                                 }
3402                                 prefetch(skb->data - NET_IP_ALIGN);
3403                                 rx_q->rx_skbuff[entry] = NULL;
3404                                 rx_q->rx_zeroc_thresh++;
3405
3406                                 skb_put(skb, frame_len);
3407                                 dma_unmap_single(priv->device,
3408                                                  rx_q->rx_skbuff_dma[entry],
3409                                                  priv->dma_buf_sz,
3410                                                  DMA_FROM_DEVICE);
3411                         }
3412
3413                         if (netif_msg_pktdata(priv)) {
3414                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3415                                            frame_len);
3416                                 print_pkt(skb->data, frame_len);
3417                         }
3418
3419                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3420
3421                         stmmac_rx_vlan(priv->dev, skb);
3422
3423                         skb->protocol = eth_type_trans(skb, priv->dev);
3424
3425                         if (unlikely(!coe))
3426                                 skb_checksum_none_assert(skb);
3427                         else
3428                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3429
3430                         napi_gro_receive(&rx_q->napi, skb);
3431
3432                         priv->dev->stats.rx_packets++;
3433                         priv->dev->stats.rx_bytes += frame_len;
3434                 }
3435                 entry = next_entry;
3436         }
3437
3438         stmmac_rx_refill(priv, queue);
3439
3440         priv->xstats.rx_pkt_n += count;
3441
3442         return count;
3443 }
3444
3445 /**
3446  *  stmmac_poll - stmmac poll method (NAPI)
3447  *  @napi : pointer to the napi structure.
3448  *  @budget : maximum number of packets that the current CPU can receive from
3449  *            all interfaces.
3450  *  Description :
3451  *  To look at the incoming frames and clear the tx resources.
3452  */
3453 static int stmmac_poll(struct napi_struct *napi, int budget)
3454 {
3455         struct stmmac_rx_queue *rx_q =
3456                 container_of(napi, struct stmmac_rx_queue, napi);
3457         struct stmmac_priv *priv = rx_q->priv_data;
3458         u32 tx_count = priv->plat->tx_queues_to_use;
3459         u32 chan = rx_q->queue_index;
3460         int work_done = 0;
3461         u32 queue;
3462
3463         priv->xstats.napi_poll++;
3464
3465         /* check all the queues */
3466         for (queue = 0; queue < tx_count; queue++)
3467                 stmmac_tx_clean(priv, queue);
3468
3469         work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3470         if (work_done < budget) {
3471                 napi_complete_done(napi, work_done);
3472                 stmmac_enable_dma_irq(priv, chan);
3473         }
3474         return work_done;
3475 }
3476
3477 /**
3478  *  stmmac_tx_timeout
3479  *  @dev : Pointer to net device structure
3480  *  Description: this function is called when a packet transmission fails to
3481  *   complete within a reasonable time. The driver will mark the error in the
3482  *   netdev structure and arrange for the device to be reset to a sane state
3483  *   in order to transmit a new packet.
3484  */
3485 static void stmmac_tx_timeout(struct net_device *dev)
3486 {
3487         struct stmmac_priv *priv = netdev_priv(dev);
3488         u32 tx_count = priv->plat->tx_queues_to_use;
3489         u32 chan;
3490
3491         /* Clear Tx resources and restart transmitting again */
3492         for (chan = 0; chan < tx_count; chan++)
3493                 stmmac_tx_err(priv, chan);
3494 }
3495
3496 /**
3497  *  stmmac_set_rx_mode - entry point for multicast addressing
3498  *  @dev : pointer to the device structure
3499  *  Description:
3500  *  This function is a driver entry point which gets called by the kernel
3501  *  whenever multicast addresses must be enabled/disabled.
3502  *  Return value:
3503  *  void.
3504  */
3505 static void stmmac_set_rx_mode(struct net_device *dev)
3506 {
3507         struct stmmac_priv *priv = netdev_priv(dev);
3508
3509         priv->hw->mac->set_filter(priv->hw, dev);
3510 }
3511
3512 /**
3513  *  stmmac_change_mtu - entry point to change MTU size for the device.
3514  *  @dev : device pointer.
3515  *  @new_mtu : the new MTU size for the device.
3516  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3517  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3518  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3519  *  Return value:
3520  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3521  *  file on failure.
3522  */
3523 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3524 {
3525         struct stmmac_priv *priv = netdev_priv(dev);
3526
3527         if (netif_running(dev)) {
3528                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3529                 return -EBUSY;
3530         }
3531
3532         dev->mtu = new_mtu;
3533
3534         netdev_update_features(dev);
3535
3536         return 0;
3537 }
3538
3539 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3540                                              netdev_features_t features)
3541 {
3542         struct stmmac_priv *priv = netdev_priv(dev);
3543
3544         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3545                 features &= ~NETIF_F_RXCSUM;
3546
3547         if (!priv->plat->tx_coe)
3548                 features &= ~NETIF_F_CSUM_MASK;
3549
3550         /* Some GMAC devices have a bugged Jumbo frame support that
3551          * needs to have the Tx COE disabled for oversized frames
3552          * (due to limited buffer sizes). In this case we disable
3553          * the TX csum insertion in the TDES and not use SF.
3554          */
3555         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3556                 features &= ~NETIF_F_CSUM_MASK;
3557
3558         /* Disable tso if asked by ethtool */
3559         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3560                 if (features & NETIF_F_TSO)
3561                         priv->tso = true;
3562                 else
3563                         priv->tso = false;
3564         }
3565
3566         return features;
3567 }
3568
3569 static int stmmac_set_features(struct net_device *netdev,
3570                                netdev_features_t features)
3571 {
3572         struct stmmac_priv *priv = netdev_priv(netdev);
3573
3574         /* Keep the COE Type in case of csum is supporting */
3575         if (features & NETIF_F_RXCSUM)
3576                 priv->hw->rx_csum = priv->plat->rx_coe;
3577         else
3578                 priv->hw->rx_csum = 0;
3579         /* No check needed because rx_coe has been set before and it will be
3580          * fixed in case of issue.
3581          */
3582         priv->hw->mac->rx_ipc(priv->hw);
3583
3584         return 0;
3585 }
3586
3587 /**
3588  *  stmmac_interrupt - main ISR
3589  *  @irq: interrupt number.
3590  *  @dev_id: to pass the net device pointer.
3591  *  Description: this is the main driver interrupt service routine.
3592  *  It can call:
3593  *  o DMA service routine (to manage incoming frame reception and transmission
3594  *    status)
3595  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3596  *    interrupts.
3597  */
3598 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3599 {
3600         struct net_device *dev = (struct net_device *)dev_id;
3601         struct stmmac_priv *priv = netdev_priv(dev);
3602         u32 rx_cnt = priv->plat->rx_queues_to_use;
3603         u32 tx_cnt = priv->plat->tx_queues_to_use;
3604         u32 queues_count;
3605         u32 queue;
3606
3607         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3608
3609         if (priv->irq_wake)
3610                 pm_wakeup_event(priv->device, 0);
3611
3612         if (unlikely(!dev)) {
3613                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3614                 return IRQ_NONE;
3615         }
3616
3617         /* To handle GMAC own interrupts */
3618         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3619                 int status = priv->hw->mac->host_irq_status(priv->hw,
3620                                                             &priv->xstats);
3621
3622                 if (unlikely(status)) {
3623                         /* For LPI we need to save the tx status */
3624                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3625                                 priv->tx_path_in_lpi_mode = true;
3626                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3627                                 priv->tx_path_in_lpi_mode = false;
3628                 }
3629
3630                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3631                         for (queue = 0; queue < queues_count; queue++) {
3632                                 struct stmmac_rx_queue *rx_q =
3633                                 &priv->rx_queue[queue];
3634
3635                                 status |=
3636                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3637                                                                    queue);
3638
3639                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3640                                     priv->hw->dma->set_rx_tail_ptr)
3641                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3642                                                                 rx_q->rx_tail_addr,
3643                                                                 queue);
3644                         }
3645                 }
3646
3647                 /* PCS link status */
3648                 if (priv->hw->pcs) {
3649                         if (priv->xstats.pcs_link)
3650                                 netif_carrier_on(dev);
3651                         else
3652                                 netif_carrier_off(dev);
3653                 }
3654         }
3655
3656         /* To handle DMA interrupts */
3657         stmmac_dma_interrupt(priv);
3658
3659         return IRQ_HANDLED;
3660 }
3661
3662 #ifdef CONFIG_NET_POLL_CONTROLLER
3663 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3664  * to allow network I/O with interrupts disabled.
3665  */
3666 static void stmmac_poll_controller(struct net_device *dev)
3667 {
3668         disable_irq(dev->irq);
3669         stmmac_interrupt(dev->irq, dev);
3670         enable_irq(dev->irq);
3671 }
3672 #endif
3673
3674 /**
3675  *  stmmac_ioctl - Entry point for the Ioctl
3676  *  @dev: Device pointer.
3677  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3678  *  a proprietary structure used to pass information to the driver.
3679  *  @cmd: IOCTL command
3680  *  Description:
3681  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3682  */
3683 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3684 {
3685         int ret = -EOPNOTSUPP;
3686
3687         if (!netif_running(dev))
3688                 return -EINVAL;
3689
3690         switch (cmd) {
3691         case SIOCGMIIPHY:
3692         case SIOCGMIIREG:
3693         case SIOCSMIIREG:
3694                 if (!dev->phydev)
3695                         return -EINVAL;
3696                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3697                 break;
3698         case SIOCSHWTSTAMP:
3699                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3700                 break;
3701         default:
3702                 break;
3703         }
3704
3705         return ret;
3706 }
3707
3708 #ifdef CONFIG_DEBUG_FS
3709 static struct dentry *stmmac_fs_dir;
3710
3711 static void sysfs_display_ring(void *head, int size, int extend_desc,
3712                                struct seq_file *seq)
3713 {
3714         int i;
3715         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3716         struct dma_desc *p = (struct dma_desc *)head;
3717
3718         for (i = 0; i < size; i++) {
3719                 if (extend_desc) {
3720                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3721                                    i, (unsigned int)virt_to_phys(ep),
3722                                    le32_to_cpu(ep->basic.des0),
3723                                    le32_to_cpu(ep->basic.des1),
3724                                    le32_to_cpu(ep->basic.des2),
3725                                    le32_to_cpu(ep->basic.des3));
3726                         ep++;
3727                 } else {
3728                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3729                                    i, (unsigned int)virt_to_phys(p),
3730                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3731                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3732                         p++;
3733                 }
3734                 seq_printf(seq, "\n");
3735         }
3736 }
3737
3738 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3739 {
3740         struct net_device *dev = seq->private;
3741         struct stmmac_priv *priv = netdev_priv(dev);
3742         u32 rx_count = priv->plat->rx_queues_to_use;
3743         u32 tx_count = priv->plat->tx_queues_to_use;
3744         u32 queue;
3745
3746         for (queue = 0; queue < rx_count; queue++) {
3747                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3748
3749                 seq_printf(seq, "RX Queue %d:\n", queue);
3750
3751                 if (priv->extend_desc) {
3752                         seq_printf(seq, "Extended descriptor ring:\n");
3753                         sysfs_display_ring((void *)rx_q->dma_erx,
3754                                            DMA_RX_SIZE, 1, seq);
3755                 } else {
3756                         seq_printf(seq, "Descriptor ring:\n");
3757                         sysfs_display_ring((void *)rx_q->dma_rx,
3758                                            DMA_RX_SIZE, 0, seq);
3759                 }
3760         }
3761
3762         for (queue = 0; queue < tx_count; queue++) {
3763                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3764
3765                 seq_printf(seq, "TX Queue %d:\n", queue);
3766
3767                 if (priv->extend_desc) {
3768                         seq_printf(seq, "Extended descriptor ring:\n");
3769                         sysfs_display_ring((void *)tx_q->dma_etx,
3770                                            DMA_TX_SIZE, 1, seq);
3771                 } else {
3772                         seq_printf(seq, "Descriptor ring:\n");
3773                         sysfs_display_ring((void *)tx_q->dma_tx,
3774                                            DMA_TX_SIZE, 0, seq);
3775                 }
3776         }
3777
3778         return 0;
3779 }
3780
3781 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3782 {
3783         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3784 }
3785
3786 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3787
3788 static const struct file_operations stmmac_rings_status_fops = {
3789         .owner = THIS_MODULE,
3790         .open = stmmac_sysfs_ring_open,
3791         .read = seq_read,
3792         .llseek = seq_lseek,
3793         .release = single_release,
3794 };
3795
3796 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3797 {
3798         struct net_device *dev = seq->private;
3799         struct stmmac_priv *priv = netdev_priv(dev);
3800
3801         if (!priv->hw_cap_support) {
3802                 seq_printf(seq, "DMA HW features not supported\n");
3803                 return 0;
3804         }
3805
3806         seq_printf(seq, "==============================\n");
3807         seq_printf(seq, "\tDMA HW features\n");
3808         seq_printf(seq, "==============================\n");
3809
3810         seq_printf(seq, "\t10/100 Mbps: %s\n",
3811                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3812         seq_printf(seq, "\t1000 Mbps: %s\n",
3813                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3814         seq_printf(seq, "\tHalf duplex: %s\n",
3815                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3816         seq_printf(seq, "\tHash Filter: %s\n",
3817                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3818         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3819                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3820         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3821                    (priv->dma_cap.pcs) ? "Y" : "N");
3822         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3823                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3824         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3825                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3826         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3827                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3828         seq_printf(seq, "\tRMON module: %s\n",
3829                    (priv->dma_cap.rmon) ? "Y" : "N");
3830         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3831                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3832         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3833                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3834         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3835                    (priv->dma_cap.eee) ? "Y" : "N");
3836         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3837         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3838                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3839         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3840                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3841                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3842         } else {
3843                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3844                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3845                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3846                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3847         }
3848         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3849                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3850         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3851                    priv->dma_cap.number_rx_channel);
3852         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3853                    priv->dma_cap.number_tx_channel);
3854         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3855                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3856
3857         return 0;
3858 }
3859
3860 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3861 {
3862         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3863 }
3864
3865 static const struct file_operations stmmac_dma_cap_fops = {
3866         .owner = THIS_MODULE,
3867         .open = stmmac_sysfs_dma_cap_open,
3868         .read = seq_read,
3869         .llseek = seq_lseek,
3870         .release = single_release,
3871 };
3872
3873 static int stmmac_init_fs(struct net_device *dev)
3874 {
3875         struct stmmac_priv *priv = netdev_priv(dev);
3876
3877         /* Create per netdev entries */
3878         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3879
3880         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3881                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3882
3883                 return -ENOMEM;
3884         }
3885
3886         /* Entry to report DMA RX/TX rings */
3887         priv->dbgfs_rings_status =
3888                 debugfs_create_file("descriptors_status", S_IRUGO,
3889                                     priv->dbgfs_dir, dev,
3890                                     &stmmac_rings_status_fops);
3891
3892         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3893                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3894                 debugfs_remove_recursive(priv->dbgfs_dir);
3895
3896                 return -ENOMEM;
3897         }
3898
3899         /* Entry to report the DMA HW features */
3900         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3901                                             priv->dbgfs_dir,
3902                                             dev, &stmmac_dma_cap_fops);
3903
3904         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3905                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3906                 debugfs_remove_recursive(priv->dbgfs_dir);
3907
3908                 return -ENOMEM;
3909         }
3910
3911         return 0;
3912 }
3913
3914 static void stmmac_exit_fs(struct net_device *dev)
3915 {
3916         struct stmmac_priv *priv = netdev_priv(dev);
3917
3918         debugfs_remove_recursive(priv->dbgfs_dir);
3919 }
3920 #endif /* CONFIG_DEBUG_FS */
3921
3922 static const struct net_device_ops stmmac_netdev_ops = {
3923         .ndo_open = stmmac_open,
3924         .ndo_start_xmit = stmmac_xmit,
3925         .ndo_stop = stmmac_release,
3926         .ndo_change_mtu = stmmac_change_mtu,
3927         .ndo_fix_features = stmmac_fix_features,
3928         .ndo_set_features = stmmac_set_features,
3929         .ndo_set_rx_mode = stmmac_set_rx_mode,
3930         .ndo_tx_timeout = stmmac_tx_timeout,
3931         .ndo_do_ioctl = stmmac_ioctl,
3932 #ifdef CONFIG_NET_POLL_CONTROLLER
3933         .ndo_poll_controller = stmmac_poll_controller,
3934 #endif
3935         .ndo_set_mac_address = eth_mac_addr,
3936 };
3937
3938 /**
3939  *  stmmac_hw_init - Init the MAC device
3940  *  @priv: driver private structure
3941  *  Description: this function is to configure the MAC device according to
3942  *  some platform parameters or the HW capability register. It prepares the
3943  *  driver to use either ring or chain modes and to setup either enhanced or
3944  *  normal descriptors.
3945  */
3946 static int stmmac_hw_init(struct stmmac_priv *priv)
3947 {
3948         struct mac_device_info *mac;
3949
3950         /* Identify the MAC HW device */
3951         if (priv->plat->has_gmac) {
3952                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3953                 mac = dwmac1000_setup(priv->ioaddr,
3954                                       priv->plat->multicast_filter_bins,
3955                                       priv->plat->unicast_filter_entries,
3956                                       &priv->synopsys_id);
3957         } else if (priv->plat->has_gmac4) {
3958                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3959                 mac = dwmac4_setup(priv->ioaddr,
3960                                    priv->plat->multicast_filter_bins,
3961                                    priv->plat->unicast_filter_entries,
3962                                    &priv->synopsys_id);
3963         } else {
3964                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
3965         }
3966         if (!mac)
3967                 return -ENOMEM;
3968
3969         priv->hw = mac;
3970
3971         /* To use the chained or ring mode */
3972         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3973                 priv->hw->mode = &dwmac4_ring_mode_ops;
3974         } else {
3975                 if (chain_mode) {
3976                         priv->hw->mode = &chain_mode_ops;
3977                         dev_info(priv->device, "Chain mode enabled\n");
3978                         priv->mode = STMMAC_CHAIN_MODE;
3979                 } else {
3980                         priv->hw->mode = &ring_mode_ops;
3981                         dev_info(priv->device, "Ring mode enabled\n");
3982                         priv->mode = STMMAC_RING_MODE;
3983                 }
3984         }
3985
3986         /* Get the HW capability (new GMAC newer than 3.50a) */
3987         priv->hw_cap_support = stmmac_get_hw_features(priv);
3988         if (priv->hw_cap_support) {
3989                 dev_info(priv->device, "DMA HW capability register supported\n");
3990
3991                 /* We can override some gmac/dma configuration fields: e.g.
3992                  * enh_desc, tx_coe (e.g. that are passed through the
3993                  * platform) with the values from the HW capability
3994                  * register (if supported).
3995                  */
3996                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
3997                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
3998                 priv->hw->pmt = priv->plat->pmt;
3999
4000                 /* TXCOE doesn't work in thresh DMA mode */
4001                 if (priv->plat->force_thresh_dma_mode)
4002                         priv->plat->tx_coe = 0;
4003                 else
4004                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
4005
4006                 /* In case of GMAC4 rx_coe is from HW cap register. */
4007                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4008
4009                 if (priv->dma_cap.rx_coe_type2)
4010                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4011                 else if (priv->dma_cap.rx_coe_type1)
4012                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4013
4014         } else {
4015                 dev_info(priv->device, "No HW DMA feature register supported\n");
4016         }
4017
4018         /* To use alternate (extended), normal or GMAC4 descriptor structures */
4019         if (priv->synopsys_id >= DWMAC_CORE_4_00)
4020                 priv->hw->desc = &dwmac4_desc_ops;
4021         else
4022                 stmmac_selec_desc_mode(priv);
4023
4024         if (priv->plat->rx_coe) {
4025                 priv->hw->rx_csum = priv->plat->rx_coe;
4026                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4027                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4028                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4029         }
4030         if (priv->plat->tx_coe)
4031                 dev_info(priv->device, "TX Checksum insertion supported\n");
4032
4033         if (priv->plat->pmt) {
4034                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4035                 device_set_wakeup_capable(priv->device, 1);
4036         }
4037
4038         if (priv->dma_cap.tsoen)
4039                 dev_info(priv->device, "TSO supported\n");
4040
4041         return 0;
4042 }
4043
4044 /**
4045  * stmmac_dvr_probe
4046  * @device: device pointer
4047  * @plat_dat: platform data pointer
4048  * @res: stmmac resource pointer
4049  * Description: this is the main probe function used to
4050  * call the alloc_etherdev, allocate the priv structure.
4051  * Return:
4052  * returns 0 on success, otherwise errno.
4053  */
4054 int stmmac_dvr_probe(struct device *device,
4055                      struct plat_stmmacenet_data *plat_dat,
4056                      struct stmmac_resources *res)
4057 {
4058         struct net_device *ndev = NULL;
4059         struct stmmac_priv *priv;
4060         int ret = 0;
4061         u32 queue;
4062
4063         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
4064                                   MTL_MAX_TX_QUEUES,
4065                                   MTL_MAX_RX_QUEUES);
4066         if (!ndev)
4067                 return -ENOMEM;
4068
4069         SET_NETDEV_DEV(ndev, device);
4070
4071         priv = netdev_priv(ndev);
4072         priv->device = device;
4073         priv->dev = ndev;
4074
4075         stmmac_set_ethtool_ops(ndev);
4076         priv->pause = pause;
4077         priv->plat = plat_dat;
4078         priv->ioaddr = res->addr;
4079         priv->dev->base_addr = (unsigned long)res->addr;
4080
4081         priv->dev->irq = res->irq;
4082         priv->wol_irq = res->wol_irq;
4083         priv->lpi_irq = res->lpi_irq;
4084
4085         if (res->mac)
4086                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4087
4088         dev_set_drvdata(device, priv->dev);
4089
4090         /* Verify driver arguments */
4091         stmmac_verify_args();
4092
4093         /* Override with kernel parameters if supplied XXX CRS XXX
4094          * this needs to have multiple instances
4095          */
4096         if ((phyaddr >= 0) && (phyaddr <= 31))
4097                 priv->plat->phy_addr = phyaddr;
4098
4099         if (priv->plat->stmmac_rst)
4100                 reset_control_deassert(priv->plat->stmmac_rst);
4101
4102         /* Init MAC and get the capabilities */
4103         ret = stmmac_hw_init(priv);
4104         if (ret)
4105                 goto error_hw_init;
4106
4107         /* Configure real RX and TX queues */
4108         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4109         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4110
4111         ndev->netdev_ops = &stmmac_netdev_ops;
4112
4113         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4114                             NETIF_F_RXCSUM;
4115
4116         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4117                 ndev->hw_features |= NETIF_F_TSO;
4118                 priv->tso = true;
4119                 dev_info(priv->device, "TSO feature enabled\n");
4120         }
4121         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4122         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4123 #ifdef STMMAC_VLAN_TAG_USED
4124         /* Both mac100 and gmac support receive VLAN tag detection */
4125         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4126 #endif
4127         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4128
4129         /* MTU range: 46 - hw-specific max */
4130         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4131         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4132                 ndev->max_mtu = JUMBO_LEN;
4133         else
4134                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4135         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4136          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4137          */
4138         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4139             (priv->plat->maxmtu >= ndev->min_mtu))
4140                 ndev->max_mtu = priv->plat->maxmtu;
4141         else if (priv->plat->maxmtu < ndev->min_mtu)
4142                 dev_warn(priv->device,
4143                          "%s: warning: maxmtu having invalid value (%d)\n",
4144                          __func__, priv->plat->maxmtu);
4145
4146         if (flow_ctrl)
4147                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4148
4149         /* Rx Watchdog is available in the COREs newer than the 3.40.
4150          * In some case, for example on bugged HW this feature
4151          * has to be disable and this can be done by passing the
4152          * riwt_off field from the platform.
4153          */
4154         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
4155                 priv->use_riwt = 1;
4156                 dev_info(priv->device,
4157                          "Enable RX Mitigation via HW Watchdog Timer\n");
4158         }
4159
4160         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4161                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4162
4163                 netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4164                                (8 * priv->plat->rx_queues_to_use));
4165         }
4166
4167         spin_lock_init(&priv->lock);
4168
4169         /* If a specific clk_csr value is passed from the platform
4170          * this means that the CSR Clock Range selection cannot be
4171          * changed at run-time and it is fixed. Viceversa the driver'll try to
4172          * set the MDC clock dynamically according to the csr actual
4173          * clock input.
4174          */
4175         if (!priv->plat->clk_csr)
4176                 stmmac_clk_csr_set(priv);
4177         else
4178                 priv->clk_csr = priv->plat->clk_csr;
4179
4180         stmmac_check_pcs_mode(priv);
4181
4182         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4183             priv->hw->pcs != STMMAC_PCS_TBI &&
4184             priv->hw->pcs != STMMAC_PCS_RTBI) {
4185                 /* MDIO bus Registration */
4186                 ret = stmmac_mdio_register(ndev);
4187                 if (ret < 0) {
4188                         dev_err(priv->device,
4189                                 "%s: MDIO bus (id: %d) registration failed",
4190                                 __func__, priv->plat->bus_id);
4191                         goto error_mdio_register;
4192                 }
4193         }
4194
4195         ret = register_netdev(ndev);
4196         if (ret) {
4197                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4198                         __func__, ret);
4199                 goto error_netdev_register;
4200         }
4201
4202         return ret;
4203
4204 error_netdev_register:
4205         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4206             priv->hw->pcs != STMMAC_PCS_TBI &&
4207             priv->hw->pcs != STMMAC_PCS_RTBI)
4208                 stmmac_mdio_unregister(ndev);
4209 error_mdio_register:
4210         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4211                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4212
4213                 netif_napi_del(&rx_q->napi);
4214         }
4215 error_hw_init:
4216         free_netdev(ndev);
4217
4218         return ret;
4219 }
4220 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4221
4222 /**
4223  * stmmac_dvr_remove
4224  * @dev: device pointer
4225  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4226  * changes the link status, releases the DMA descriptor rings.
4227  */
4228 int stmmac_dvr_remove(struct device *dev)
4229 {
4230         struct net_device *ndev = dev_get_drvdata(dev);
4231         struct stmmac_priv *priv = netdev_priv(ndev);
4232
4233         netdev_info(priv->dev, "%s: removing driver", __func__);
4234
4235         stmmac_stop_all_dma(priv);
4236
4237         priv->hw->mac->set_mac(priv->ioaddr, false);
4238         netif_carrier_off(ndev);
4239         unregister_netdev(ndev);
4240         if (priv->plat->stmmac_rst)
4241                 reset_control_assert(priv->plat->stmmac_rst);
4242         clk_disable_unprepare(priv->plat->pclk);
4243         clk_disable_unprepare(priv->plat->stmmac_clk);
4244         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4245             priv->hw->pcs != STMMAC_PCS_TBI &&
4246             priv->hw->pcs != STMMAC_PCS_RTBI)
4247                 stmmac_mdio_unregister(ndev);
4248         free_netdev(ndev);
4249
4250         return 0;
4251 }
4252 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4253
4254 /**
4255  * stmmac_suspend - suspend callback
4256  * @dev: device pointer
4257  * Description: this is the function to suspend the device and it is called
4258  * by the platform driver to stop the network queue, release the resources,
4259  * program the PMT register (for WoL), clean and release driver resources.
4260  */
4261 int stmmac_suspend(struct device *dev)
4262 {
4263         struct net_device *ndev = dev_get_drvdata(dev);
4264         struct stmmac_priv *priv = netdev_priv(ndev);
4265         unsigned long flags;
4266
4267         if (!ndev || !netif_running(ndev))
4268                 return 0;
4269
4270         if (ndev->phydev)
4271                 phy_stop(ndev->phydev);
4272
4273         spin_lock_irqsave(&priv->lock, flags);
4274
4275         netif_device_detach(ndev);
4276         stmmac_stop_all_queues(priv);
4277
4278         stmmac_disable_all_queues(priv);
4279
4280         /* Stop TX/RX DMA */
4281         stmmac_stop_all_dma(priv);
4282
4283         /* Enable Power down mode by programming the PMT regs */
4284         if (device_may_wakeup(priv->device)) {
4285                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
4286                 priv->irq_wake = 1;
4287         } else {
4288                 priv->hw->mac->set_mac(priv->ioaddr, false);
4289                 pinctrl_pm_select_sleep_state(priv->device);
4290                 /* Disable clock in case of PWM is off */
4291                 clk_disable(priv->plat->pclk);
4292                 clk_disable(priv->plat->stmmac_clk);
4293         }
4294         spin_unlock_irqrestore(&priv->lock, flags);
4295
4296         priv->oldlink = 0;
4297         priv->speed = SPEED_UNKNOWN;
4298         priv->oldduplex = DUPLEX_UNKNOWN;
4299         return 0;
4300 }
4301 EXPORT_SYMBOL_GPL(stmmac_suspend);
4302
4303 /**
4304  * stmmac_reset_queues_param - reset queue parameters
4305  * @dev: device pointer
4306  */
4307 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4308 {
4309         u32 rx_cnt = priv->plat->rx_queues_to_use;
4310         u32 tx_cnt = priv->plat->tx_queues_to_use;
4311         u32 queue;
4312
4313         for (queue = 0; queue < rx_cnt; queue++) {
4314                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4315
4316                 rx_q->cur_rx = 0;
4317                 rx_q->dirty_rx = 0;
4318         }
4319
4320         for (queue = 0; queue < tx_cnt; queue++) {
4321                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4322
4323                 tx_q->cur_tx = 0;
4324                 tx_q->dirty_tx = 0;
4325         }
4326 }
4327
4328 /**
4329  * stmmac_resume - resume callback
4330  * @dev: device pointer
4331  * Description: when resume this function is invoked to setup the DMA and CORE
4332  * in a usable state.
4333  */
4334 int stmmac_resume(struct device *dev)
4335 {
4336         struct net_device *ndev = dev_get_drvdata(dev);
4337         struct stmmac_priv *priv = netdev_priv(ndev);
4338         unsigned long flags;
4339
4340         if (!netif_running(ndev))
4341                 return 0;
4342
4343         /* Power Down bit, into the PM register, is cleared
4344          * automatically as soon as a magic packet or a Wake-up frame
4345          * is received. Anyway, it's better to manually clear
4346          * this bit because it can generate problems while resuming
4347          * from another devices (e.g. serial console).
4348          */
4349         if (device_may_wakeup(priv->device)) {
4350                 spin_lock_irqsave(&priv->lock, flags);
4351                 priv->hw->mac->pmt(priv->hw, 0);
4352                 spin_unlock_irqrestore(&priv->lock, flags);
4353                 priv->irq_wake = 0;
4354         } else {
4355                 pinctrl_pm_select_default_state(priv->device);
4356                 /* enable the clk previously disabled */
4357                 clk_enable(priv->plat->stmmac_clk);
4358                 clk_enable(priv->plat->pclk);
4359                 /* reset the phy so that it's ready */
4360                 if (priv->mii)
4361                         stmmac_mdio_reset(priv->mii);
4362         }
4363
4364         netif_device_attach(ndev);
4365
4366         spin_lock_irqsave(&priv->lock, flags);
4367
4368         stmmac_reset_queues_param(priv);
4369
4370         /* reset private mss value to force mss context settings at
4371          * next tso xmit (only used for gmac4).
4372          */
4373         priv->mss = 0;
4374
4375         stmmac_clear_descriptors(priv);
4376
4377         stmmac_hw_setup(ndev, false);
4378         stmmac_init_tx_coalesce(priv);
4379         stmmac_set_rx_mode(ndev);
4380
4381         stmmac_enable_all_queues(priv);
4382
4383         stmmac_start_all_queues(priv);
4384
4385         spin_unlock_irqrestore(&priv->lock, flags);
4386
4387         if (ndev->phydev)
4388                 phy_start(ndev->phydev);
4389
4390         return 0;
4391 }
4392 EXPORT_SYMBOL_GPL(stmmac_resume);
4393
4394 #ifndef MODULE
4395 static int __init stmmac_cmdline_opt(char *str)
4396 {
4397         char *opt;
4398
4399         if (!str || !*str)
4400                 return -EINVAL;
4401         while ((opt = strsep(&str, ",")) != NULL) {
4402                 if (!strncmp(opt, "debug:", 6)) {
4403                         if (kstrtoint(opt + 6, 0, &debug))
4404                                 goto err;
4405                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4406                         if (kstrtoint(opt + 8, 0, &phyaddr))
4407                                 goto err;
4408                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4409                         if (kstrtoint(opt + 7, 0, &buf_sz))
4410                                 goto err;
4411                 } else if (!strncmp(opt, "tc:", 3)) {
4412                         if (kstrtoint(opt + 3, 0, &tc))
4413                                 goto err;
4414                 } else if (!strncmp(opt, "watchdog:", 9)) {
4415                         if (kstrtoint(opt + 9, 0, &watchdog))
4416                                 goto err;
4417                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4418                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4419                                 goto err;
4420                 } else if (!strncmp(opt, "pause:", 6)) {
4421                         if (kstrtoint(opt + 6, 0, &pause))
4422                                 goto err;
4423                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4424                         if (kstrtoint(opt + 10, 0, &eee_timer))
4425                                 goto err;
4426                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4427                         if (kstrtoint(opt + 11, 0, &chain_mode))
4428                                 goto err;
4429                 }
4430         }
4431         return 0;
4432
4433 err:
4434         pr_err("%s: ERROR broken module parameter conversion", __func__);
4435         return -EINVAL;
4436 }
4437
4438 __setup("stmmaceth=", stmmac_cmdline_opt);
4439 #endif /* MODULE */
4440
4441 static int __init stmmac_init(void)
4442 {
4443 #ifdef CONFIG_DEBUG_FS
4444         /* Create debugfs main directory if it doesn't exist yet */
4445         if (!stmmac_fs_dir) {
4446                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4447
4448                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4449                         pr_err("ERROR %s, debugfs create directory failed\n",
4450                                STMMAC_RESOURCE_NAME);
4451
4452                         return -ENOMEM;
4453                 }
4454         }
4455 #endif
4456
4457         return 0;
4458 }
4459
4460 static void __exit stmmac_exit(void)
4461 {
4462 #ifdef CONFIG_DEBUG_FS
4463         debugfs_remove_recursive(stmmac_fs_dir);
4464 #endif
4465 }
4466
4467 module_init(stmmac_init)
4468 module_exit(stmmac_exit)
4469
4470 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4471 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4472 MODULE_LICENSE("GPL");