drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_clk_csr_set - dynamically set the MDC clock
 143  * @priv: driver private structure
 144  * Description: this is to dynamically set the MDC clock according to the csr
 145  * clock input.
 146  * Note:
 147  *      If a specific clk_csr value is passed from the platform
 148  *      this means that the CSR Clock Range selection cannot be
 149  *      changed at run-time and it is fixed (as reported in the driver
 150  *      documentation). Viceversa the driver will try to set the MDC
 151  *      clock dynamically according to the actual clock input.
 152  */
 153 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 154 {
 155         u32 clk_rate;
 156
 157         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 158
 159         /* Platform provided default clk_csr would be assumed valid
 160          * for all other cases except for the below mentioned ones.
 161          * For values higher than the IEEE 802.3 specified frequency
 162          * we can not estimate the proper divider as it is not known
 163          * the frequency of clk_csr_i. So we do not change the default
 164          * divider.
 165          */
 166         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 167                 if (clk_rate < CSR_F_35M)
 168                         priv->clk_csr = STMMAC_CSR_20_35M;
 169                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 170                         priv->clk_csr = STMMAC_CSR_35_60M;
 171                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 172                         priv->clk_csr = STMMAC_CSR_60_100M;
 173                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 174                         priv->clk_csr = STMMAC_CSR_100_150M;
 175                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 176                         priv->clk_csr = STMMAC_CSR_150_250M;
 177                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 178                         priv->clk_csr = STMMAC_CSR_250_300M;
 179         }
 180 }
 181
 182 static void print_pkt(unsigned char *buf, int len)
 183 {
 184         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 185         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 186 }
 187
 188 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv)
 189 {
 190         u32 avail;
 191
 192         if (priv->dirty_tx > priv->cur_tx)
 193                 avail = priv->dirty_tx - priv->cur_tx - 1;
 194         else
 195                 avail = DMA_TX_SIZE - priv->cur_tx + priv->dirty_tx - 1;
 196
 197         return avail;
 198 }
 199
 200 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv)
 201 {
 202         u32 dirty;
 203
 204         if (priv->dirty_rx <= priv->cur_rx)
 205                 dirty = priv->cur_rx - priv->dirty_rx;
 206         else
 207                 dirty = DMA_RX_SIZE - priv->dirty_rx + priv->cur_rx;
 208
 209         return dirty;
 210 }
 211
 212 /**
 213  * stmmac_hw_fix_mac_speed - callback for speed selection
 214  * @priv: driver private structure
 215  * Description: on some platforms (e.g. ST), some HW system configuration
 216  * registers have to be set according to the link speed negotiated.
 217  */
 218 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 219 {
 220         struct net_device *ndev = priv->dev;
 221         struct phy_device *phydev = ndev->phydev;
 222
 223         if (likely(priv->plat->fix_mac_speed))
 224                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 225 }
 226
 227 /**
 228  * stmmac_enable_eee_mode - check and enter in LPI mode
 229  * @priv: driver private structure
 230  * Description: this function is to verify and enter in LPI mode in case of
 231  * EEE.
 232  */
 233 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 234 {
 235         /* Check and enter in LPI mode */
 236         if ((priv->dirty_tx == priv->cur_tx) &&
 237             (priv->tx_path_in_lpi_mode == false))
 238                 priv->hw->mac->set_eee_mode(priv->hw,
 239                                             priv->plat->en_tx_lpi_clockgating);
 240 }
 241
 242 /**
 243  * stmmac_disable_eee_mode - disable and exit from LPI mode
 244  * @priv: driver private structure
 245  * Description: this function is to exit and disable EEE in case of
 246  * LPI state is true. This is called by the xmit.
 247  */
 248 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 249 {
 250         priv->hw->mac->reset_eee_mode(priv->hw);
 251         del_timer_sync(&priv->eee_ctrl_timer);
 252         priv->tx_path_in_lpi_mode = false;
 253 }
 254
 255 /**
 256  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 257  * @arg : data hook
 258  * Description:
 259  *  if there is no data transfer and if we are not in LPI state,
 260  *  then MAC Transmitter can be moved to LPI state.
 261  */
 262 static void stmmac_eee_ctrl_timer(unsigned long arg)
 263 {
 264         struct stmmac_priv *priv = (struct stmmac_priv *)arg;
 265
 266         stmmac_enable_eee_mode(priv);
 267         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 268 }
 269
 270 /**
 271  * stmmac_eee_init - init EEE
 272  * @priv: driver private structure
 273  * Description:
 274  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 275  *  can also manage EEE, this function enable the LPI state and start related
 276  *  timer.
 277  */
 278 bool stmmac_eee_init(struct stmmac_priv *priv)
 279 {
 280         struct net_device *ndev = priv->dev;
 281         unsigned long flags;
 282         bool ret = false;
 283
 284         /* Using PCS we cannot dial with the phy registers at this stage
 285          * so we do not support extra feature like EEE.
 286          */
 287         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 288             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 289             (priv->hw->pcs == STMMAC_PCS_RTBI))
 290                 goto out;
 291
 292         /* MAC core supports the EEE feature. */
 293         if (priv->dma_cap.eee) {
 294                 int tx_lpi_timer = priv->tx_lpi_timer;
 295
 296                 /* Check if the PHY supports EEE */
 297                 if (phy_init_eee(ndev->phydev, 1)) {
 298                         /* To manage at run-time if the EEE cannot be supported
 299                          * anymore (for example because the lp caps have been
 300                          * changed).
 301                          * In that case the driver disable own timers.
 302                          */
 303                         spin_lock_irqsave(&priv->lock, flags);
 304                         if (priv->eee_active) {
 305                                 netdev_dbg(priv->dev, "disable EEE\n");
 306                                 del_timer_sync(&priv->eee_ctrl_timer);
 307                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 308                                                              tx_lpi_timer);
 309                         }
 310                         priv->eee_active = 0;
 311                         spin_unlock_irqrestore(&priv->lock, flags);
 312                         goto out;
 313                 }
 314                 /* Activate the EEE and start timers */
 315                 spin_lock_irqsave(&priv->lock, flags);
 316                 if (!priv->eee_active) {
 317                         priv->eee_active = 1;
 318                         setup_timer(&priv->eee_ctrl_timer,
 319                                     stmmac_eee_ctrl_timer,
 320                                     (unsigned long)priv);
 321                         mod_timer(&priv->eee_ctrl_timer,
 322                                   STMMAC_LPI_T(eee_timer));
 323
 324                         priv->hw->mac->set_eee_timer(priv->hw,
 325                                                      STMMAC_DEFAULT_LIT_LS,
 326                                                      tx_lpi_timer);
 327                 }
 328                 /* Set HW EEE according to the speed */
 329                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 330
 331                 ret = true;
 332                 spin_unlock_irqrestore(&priv->lock, flags);
 333
 334                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 335         }
 336 out:
 337         return ret;
 338 }
 339
 340 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 341  * @priv: driver private structure
 342  * @p : descriptor pointer
 343  * @skb : the socket buffer
 344  * Description :
 345  * This function will read timestamp from the descriptor & pass it to stack.
 346  * and also perform some sanity checks.
 347  */
 348 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 349                                    struct dma_desc *p, struct sk_buff *skb)
 350 {
 351         struct skb_shared_hwtstamps shhwtstamp;
 352         u64 ns;
 353
 354         if (!priv->hwts_tx_en)
 355                 return;
 356
 357         /* exit if skb doesn't support hw tstamp */
 358         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 359                 return;
 360
 361         /* check tx tstamp status */
 362         if (!priv->hw->desc->get_tx_timestamp_status(p)) {
 363                 /* get the valid tstamp */
 364                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 365
 366                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 367                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 368
 369                 netdev_info(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 370                 /* pass tstamp to stack */
 371                 skb_tstamp_tx(skb, &shhwtstamp);
 372         }
 373
 374         return;
 375 }
 376
 377 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 378  * @priv: driver private structure
 379  * @p : descriptor pointer
 380  * @np : next descriptor pointer
 381  * @skb : the socket buffer
 382  * Description :
 383  * This function will read received packet's timestamp from the descriptor
 384  * and pass it to stack. It also perform some sanity checks.
 385  */
 386 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 387                                    struct dma_desc *np, struct sk_buff *skb)
 388 {
 389         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 390         u64 ns;
 391
 392         if (!priv->hwts_rx_en)
 393                 return;
 394
 395         /* Check if timestamp is available */
 396         if (!priv->hw->desc->get_rx_timestamp_status(p, priv->adv_ts)) {
 397                 /* For GMAC4, the valid timestamp is from CTX next desc. */
 398                 if (priv->plat->has_gmac4)
 399                         ns = priv->hw->desc->get_timestamp(np, priv->adv_ts);
 400                 else
 401                         ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 402
 403                 netdev_info(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 404                 shhwtstamp = skb_hwtstamps(skb);
 405                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 406                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 407         } else  {
 408                 netdev_err(priv->dev, "cannot get RX hw timestamp\n");
 409         }
 410 }
 411
 412 /**
 413  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 414  *  @dev: device pointer.
 415  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 416  *  a proprietary structure used to pass information to the driver.
 417  *  Description:
 418  *  This function configures the MAC to enable/disable both outgoing(TX)
 419  *  and incoming(RX) packets time stamping based on user input.
 420  *  Return Value:
 421  *  0 on success and an appropriate -ve integer on failure.
 422  */
 423 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 424 {
 425         struct stmmac_priv *priv = netdev_priv(dev);
 426         struct hwtstamp_config config;
 427         struct timespec64 now;
 428         u64 temp = 0;
 429         u32 ptp_v2 = 0;
 430         u32 tstamp_all = 0;
 431         u32 ptp_over_ipv4_udp = 0;
 432         u32 ptp_over_ipv6_udp = 0;
 433         u32 ptp_over_ethernet = 0;
 434         u32 snap_type_sel = 0;
 435         u32 ts_master_en = 0;
 436         u32 ts_event_en = 0;
 437         u32 value = 0;
 438         u32 sec_inc;
 439
 440         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 441                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 442                 priv->hwts_tx_en = 0;
 443                 priv->hwts_rx_en = 0;
 444
 445                 return -EOPNOTSUPP;
 446         }
 447
 448         if (copy_from_user(&config, ifr->ifr_data,
 449                            sizeof(struct hwtstamp_config)))
 450                 return -EFAULT;
 451
 452         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 453                    __func__, config.flags, config.tx_type, config.rx_filter);
 454
 455         /* reserved for future extensions */
 456         if (config.flags)
 457                 return -EINVAL;
 458
 459         if (config.tx_type != HWTSTAMP_TX_OFF &&
 460             config.tx_type != HWTSTAMP_TX_ON)
 461                 return -ERANGE;
 462
 463         if (priv->adv_ts) {
 464                 switch (config.rx_filter) {
 465                 case HWTSTAMP_FILTER_NONE:
 466                         /* time stamp no incoming packet at all */
 467                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 468                         break;
 469
 470                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 471                         /* PTP v1, UDP, any kind of event packet */
 472                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 473                         /* take time stamp for all event messages */
 474                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 475
 476                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 477                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 478                         break;
 479
 480                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 481                         /* PTP v1, UDP, Sync packet */
 482                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 483                         /* take time stamp for SYNC messages only */
 484                         ts_event_en = PTP_TCR_TSEVNTENA;
 485
 486                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 487                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 488                         break;
 489
 490                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 491                         /* PTP v1, UDP, Delay_req packet */
 492                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 493                         /* take time stamp for Delay_Req messages only */
 494                         ts_master_en = PTP_TCR_TSMSTRENA;
 495                         ts_event_en = PTP_TCR_TSEVNTENA;
 496
 497                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 498                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 499                         break;
 500
 501                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 502                         /* PTP v2, UDP, any kind of event packet */
 503                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 504                         ptp_v2 = PTP_TCR_TSVER2ENA;
 505                         /* take time stamp for all event messages */
 506                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 507
 508                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 509                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 510                         break;
 511
 512                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 513                         /* PTP v2, UDP, Sync packet */
 514                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 515                         ptp_v2 = PTP_TCR_TSVER2ENA;
 516                         /* take time stamp for SYNC messages only */
 517                         ts_event_en = PTP_TCR_TSEVNTENA;
 518
 519                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 520                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 521                         break;
 522
 523                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 524                         /* PTP v2, UDP, Delay_req packet */
 525                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 526                         ptp_v2 = PTP_TCR_TSVER2ENA;
 527                         /* take time stamp for Delay_Req messages only */
 528                         ts_master_en = PTP_TCR_TSMSTRENA;
 529                         ts_event_en = PTP_TCR_TSEVNTENA;
 530
 531                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 532                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 533                         break;
 534
 535                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 536                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 537                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 538                         ptp_v2 = PTP_TCR_TSVER2ENA;
 539                         /* take time stamp for all event messages */
 540                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 541
 542                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 543                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 544                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 545                         break;
 546
 547                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 548                         /* PTP v2/802.AS1, any layer, Sync packet */
 549                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 550                         ptp_v2 = PTP_TCR_TSVER2ENA;
 551                         /* take time stamp for SYNC messages only */
 552                         ts_event_en = PTP_TCR_TSEVNTENA;
 553
 554                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 555                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 556                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 557                         break;
 558
 559                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 560                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 561                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 562                         ptp_v2 = PTP_TCR_TSVER2ENA;
 563                         /* take time stamp for Delay_Req messages only */
 564                         ts_master_en = PTP_TCR_TSMSTRENA;
 565                         ts_event_en = PTP_TCR_TSEVNTENA;
 566
 567                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 568                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 569                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 570                         break;
 571
 572                 case HWTSTAMP_FILTER_ALL:
 573                         /* time stamp any incoming packet */
 574                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 575                         tstamp_all = PTP_TCR_TSENALL;
 576                         break;
 577
 578                 default:
 579                         return -ERANGE;
 580                 }
 581         } else {
 582                 switch (config.rx_filter) {
 583                 case HWTSTAMP_FILTER_NONE:
 584                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 585                         break;
 586                 default:
 587                         /* PTP v1, UDP, any kind of event packet */
 588                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 589                         break;
 590                 }
 591         }
 592         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 593         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 594
 595         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 596                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 597         else {
 598                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 599                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 600                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 601                          ts_master_en | snap_type_sel);
 602                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 603
 604                 /* program Sub Second Increment reg */
 605                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 606                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 607                         priv->plat->has_gmac4);
 608                 temp = div_u64(1000000000ULL, sec_inc);
 609
 610                 /* calculate default added value:
 611                  * formula is :
 612                  * addend = (2^32)/freq_div_ratio;
 613                  * where, freq_div_ratio = 1e9ns/sec_inc
 614                  */
 615                 temp = (u64)(temp << 32);
 616                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 617                 priv->hw->ptp->config_addend(priv->ptpaddr,
 618                                              priv->default_addend);
 619
 620                 /* initialize system time */
 621                 ktime_get_real_ts64(&now);
 622
 623                 /* lower 32 bits of tv_sec are safe until y2106 */
 624                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 625                                             now.tv_nsec);
 626         }
 627
 628         return copy_to_user(ifr->ifr_data, &config,
 629                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 630 }
 631
 632 /**
 633  * stmmac_init_ptp - init PTP
 634  * @priv: driver private structure
 635  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 636  * This is done by looking at the HW cap. register.
 637  * This function also registers the ptp driver.
 638  */
 639 static int stmmac_init_ptp(struct stmmac_priv *priv)
 640 {
 641         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 642                 return -EOPNOTSUPP;
 643
 644         priv->adv_ts = 0;
 645         /* Check if adv_ts can be enabled for dwmac 4.x core */
 646         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 647                 priv->adv_ts = 1;
 648         /* Dwmac 3.x core with extend_desc can support adv_ts */
 649         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 650                 priv->adv_ts = 1;
 651
 652         if (priv->dma_cap.time_stamp)
 653                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 654
 655         if (priv->adv_ts)
 656                 netdev_info(priv->dev,
 657                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 658
 659         priv->hw->ptp = &stmmac_ptp;
 660         priv->hwts_tx_en = 0;
 661         priv->hwts_rx_en = 0;
 662
 663         stmmac_ptp_register(priv);
 664
 665         return 0;
 666 }
 667
 668 static void stmmac_release_ptp(struct stmmac_priv *priv)
 669 {
 670         if (priv->plat->clk_ptp_ref)
 671                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 672         stmmac_ptp_unregister(priv);
 673 }
 674
 675 /**
 676  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 677  *  @priv: driver private structure
 678  *  Description: It is used for configuring the flow control in all queues
 679  */
 680 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 681 {
 682         u32 tx_cnt = priv->plat->tx_queues_to_use;
 683
 684         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 685                                  priv->pause, tx_cnt);
 686 }
 687
 688 /**
 689  * stmmac_adjust_link - adjusts the link parameters
 690  * @dev: net device structure
 691  * Description: this is the helper called by the physical abstraction layer
 692  * drivers to communicate the phy link status. According the speed and duplex
 693  * this driver can invoke registered glue-logic as well.
 694  * It also invoke the eee initialization because it could happen when switch
 695  * on different networks (that are eee capable).
 696  */
 697 static void stmmac_adjust_link(struct net_device *dev)
 698 {
 699         struct stmmac_priv *priv = netdev_priv(dev);
 700         struct phy_device *phydev = dev->phydev;
 701         unsigned long flags;
 702         int new_state = 0;
 703
 704         if (!phydev)
 705                 return;
 706
 707         spin_lock_irqsave(&priv->lock, flags);
 708
 709         if (phydev->link) {
 710                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 711
 712                 /* Now we make sure that we can be in full duplex mode.
 713                  * If not, we operate in half-duplex mode. */
 714                 if (phydev->duplex != priv->oldduplex) {
 715                         new_state = 1;
 716                         if (!(phydev->duplex))
 717                                 ctrl &= ~priv->hw->link.duplex;
 718                         else
 719                                 ctrl |= priv->hw->link.duplex;
 720                         priv->oldduplex = phydev->duplex;
 721                 }
 722                 /* Flow Control operation */
 723                 if (phydev->pause)
 724                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 725
 726                 if (phydev->speed != priv->speed) {
 727                         new_state = 1;
 728                         switch (phydev->speed) {
 729                         case 1000:
 730                                 if (priv->plat->has_gmac ||
 731                                     priv->plat->has_gmac4)
 732                                         ctrl &= ~priv->hw->link.port;
 733                                 break;
 734                         case 100:
 735                                 if (priv->plat->has_gmac ||
 736                                     priv->plat->has_gmac4) {
 737                                         ctrl |= priv->hw->link.port;
 738                                         ctrl |= priv->hw->link.speed;
 739                                 } else {
 740                                         ctrl &= ~priv->hw->link.port;
 741                                 }
 742                                 break;
 743                         case 10:
 744                                 if (priv->plat->has_gmac ||
 745                                     priv->plat->has_gmac4) {
 746                                         ctrl |= priv->hw->link.port;
 747                                         ctrl &= ~(priv->hw->link.speed);
 748                                 } else {
 749                                         ctrl &= ~priv->hw->link.port;
 750                                 }
 751                                 break;
 752                         default:
 753                                 netif_warn(priv, link, priv->dev,
 754                                            "broken speed: %d\n", phydev->speed);
 755                                 phydev->speed = SPEED_UNKNOWN;
 756                                 break;
 757                         }
 758                         if (phydev->speed != SPEED_UNKNOWN)
 759                                 stmmac_hw_fix_mac_speed(priv);
 760                         priv->speed = phydev->speed;
 761                 }
 762
 763                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 764
 765                 if (!priv->oldlink) {
 766                         new_state = 1;
 767                         priv->oldlink = 1;
 768                 }
 769         } else if (priv->oldlink) {
 770                 new_state = 1;
 771                 priv->oldlink = 0;
 772                 priv->speed = SPEED_UNKNOWN;
 773                 priv->oldduplex = DUPLEX_UNKNOWN;
 774         }
 775
 776         if (new_state && netif_msg_link(priv))
 777                 phy_print_status(phydev);
 778
 779         spin_unlock_irqrestore(&priv->lock, flags);
 780
 781         if (phydev->is_pseudo_fixed_link)
 782                 /* Stop PHY layer to call the hook to adjust the link in case
 783                  * of a switch is attached to the stmmac driver.
 784                  */
 785                 phydev->irq = PHY_IGNORE_INTERRUPT;
 786         else
 787                 /* At this stage, init the EEE if supported.
 788                  * Never called in case of fixed_link.
 789                  */
 790                 priv->eee_enabled = stmmac_eee_init(priv);
 791 }
 792
 793 /**
 794  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 795  * @priv: driver private structure
 796  * Description: this is to verify if the HW supports the PCS.
 797  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 798  * configured for the TBI, RTBI, or SGMII PHY interface.
 799  */
 800 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 801 {
 802         int interface = priv->plat->interface;
 803
 804         if (priv->dma_cap.pcs) {
 805                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 806                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 807                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 808                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 809                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 810                         priv->hw->pcs = STMMAC_PCS_RGMII;
 811                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 812                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 813                         priv->hw->pcs = STMMAC_PCS_SGMII;
 814                 }
 815         }
 816 }
 817
 818 /**
 819  * stmmac_init_phy - PHY initialization
 820  * @dev: net device structure
 821  * Description: it initializes the driver's PHY state, and attaches the PHY
 822  * to the mac driver.
 823  *  Return value:
 824  *  0 on success
 825  */
 826 static int stmmac_init_phy(struct net_device *dev)
 827 {
 828         struct stmmac_priv *priv = netdev_priv(dev);
 829         struct phy_device *phydev;
 830         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 831         char bus_id[MII_BUS_ID_SIZE];
 832         int interface = priv->plat->interface;
 833         int max_speed = priv->plat->max_speed;
 834         priv->oldlink = 0;
 835         priv->speed = SPEED_UNKNOWN;
 836         priv->oldduplex = DUPLEX_UNKNOWN;
 837
 838         if (priv->plat->phy_node) {
 839                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 840                                         &stmmac_adjust_link, 0, interface);
 841         } else {
 842                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 843                          priv->plat->bus_id);
 844
 845                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 846                          priv->plat->phy_addr);
 847                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 848                            phy_id_fmt);
 849
 850                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 851                                      interface);
 852         }
 853
 854         if (IS_ERR_OR_NULL(phydev)) {
 855                 netdev_err(priv->dev, "Could not attach to PHY\n");
 856                 if (!phydev)
 857                         return -ENODEV;
 858
 859                 return PTR_ERR(phydev);
 860         }
 861
 862         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 863         if ((interface == PHY_INTERFACE_MODE_MII) ||
 864             (interface == PHY_INTERFACE_MODE_RMII) ||
 865                 (max_speed < 1000 && max_speed > 0))
 866                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 867                                          SUPPORTED_1000baseT_Full);
 868
 869         /*
 870          * Broken HW is sometimes missing the pull-up resistor on the
 871          * MDIO line, which results in reads to non-existent devices returning
 872          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 873          * device as well.
 874          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 875          */
 876         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 877                 phy_disconnect(phydev);
 878                 return -ENODEV;
 879         }
 880
 881         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 882          * subsequent PHY polling, make sure we force a link transition if
 883          * we have a UP/DOWN/UP transition
 884          */
 885         if (phydev->is_pseudo_fixed_link)
 886                 phydev->irq = PHY_POLL;
 887
 888         phy_attached_info(phydev);
 889         return 0;
 890 }
 891
 892 static void stmmac_display_rings(struct stmmac_priv *priv)
 893 {
 894         void *head_rx, *head_tx;
 895
 896         if (priv->extend_desc) {
 897                 head_rx = (void *)priv->dma_erx;
 898                 head_tx = (void *)priv->dma_etx;
 899         } else {
 900                 head_rx = (void *)priv->dma_rx;
 901                 head_tx = (void *)priv->dma_tx;
 902         }
 903
 904         /* Display Rx ring */
 905         priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 906         /* Display Tx ring */
 907         priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
 908 }
 909
 910 static int stmmac_set_bfsize(int mtu, int bufsize)
 911 {
 912         int ret = bufsize;
 913
 914         if (mtu >= BUF_SIZE_4KiB)
 915                 ret = BUF_SIZE_8KiB;
 916         else if (mtu >= BUF_SIZE_2KiB)
 917                 ret = BUF_SIZE_4KiB;
 918         else if (mtu > DEFAULT_BUFSIZE)
 919                 ret = BUF_SIZE_2KiB;
 920         else
 921                 ret = DEFAULT_BUFSIZE;
 922
 923         return ret;
 924 }
 925
 926 /**
 927  * stmmac_clear_descriptors - clear descriptors
 928  * @priv: driver private structure
 929  * Description: this function is called to clear the tx and rx descriptors
 930  * in case of both basic and extended descriptors are used.
 931  */
 932 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
 933 {
 934         int i;
 935
 936         /* Clear the Rx/Tx descriptors */
 937         for (i = 0; i < DMA_RX_SIZE; i++)
 938                 if (priv->extend_desc)
 939                         priv->hw->desc->init_rx_desc(&priv->dma_erx[i].basic,
 940                                                      priv->use_riwt, priv->mode,
 941                                                      (i == DMA_RX_SIZE - 1));
 942                 else
 943                         priv->hw->desc->init_rx_desc(&priv->dma_rx[i],
 944                                                      priv->use_riwt, priv->mode,
 945                                                      (i == DMA_RX_SIZE - 1));
 946         for (i = 0; i < DMA_TX_SIZE; i++)
 947                 if (priv->extend_desc)
 948                         priv->hw->desc->init_tx_desc(&priv->dma_etx[i].basic,
 949                                                      priv->mode,
 950                                                      (i == DMA_TX_SIZE - 1));
 951                 else
 952                         priv->hw->desc->init_tx_desc(&priv->dma_tx[i],
 953                                                      priv->mode,
 954                                                      (i == DMA_TX_SIZE - 1));
 955 }
 956
 957 /**
 958  * stmmac_init_rx_buffers - init the RX descriptor buffer.
 959  * @priv: driver private structure
 960  * @p: descriptor pointer
 961  * @i: descriptor index
 962  * @flags: gfp flag.
 963  * Description: this function is called to allocate a receive buffer, perform
 964  * the DMA mapping and init the descriptor.
 965  */
 966 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
 967                                   int i, gfp_t flags)
 968 {
 969         struct sk_buff *skb;
 970
 971         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
 972         if (!skb) {
 973                 netdev_err(priv->dev,
 974                            "%s: Rx init fails; skb is NULL\n", __func__);
 975                 return -ENOMEM;
 976         }
 977         priv->rx_skbuff[i] = skb;
 978         priv->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
 979                                                 priv->dma_buf_sz,
 980                                                 DMA_FROM_DEVICE);
 981         if (dma_mapping_error(priv->device, priv->rx_skbuff_dma[i])) {
 982                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
 983                 dev_kfree_skb_any(skb);
 984                 return -EINVAL;
 985         }
 986
 987         if (priv->synopsys_id >= DWMAC_CORE_4_00)
 988                 p->des0 = cpu_to_le32(priv->rx_skbuff_dma[i]);
 989         else
 990                 p->des2 = cpu_to_le32(priv->rx_skbuff_dma[i]);
 991
 992         if ((priv->hw->mode->init_desc3) &&
 993             (priv->dma_buf_sz == BUF_SIZE_16KiB))
 994                 priv->hw->mode->init_desc3(p);
 995
 996         return 0;
 997 }
 998
 999 static void stmmac_free_rx_buffers(struct stmmac_priv *priv, int i)
1000 {
1001         if (priv->rx_skbuff[i]) {
1002                 dma_unmap_single(priv->device, priv->rx_skbuff_dma[i],
1003                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1004                 dev_kfree_skb_any(priv->rx_skbuff[i]);
1005         }
1006         priv->rx_skbuff[i] = NULL;
1007 }
1008
1009 /**
1010  * init_dma_desc_rings - init the RX/TX descriptor rings
1011  * @dev: net device structure
1012  * @flags: gfp flag.
1013  * Description: this function initializes the DMA RX/TX descriptors
1014  * and allocates the socket buffers. It supports the chained and ring
1015  * modes.
1016  */
1017 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1018 {
1019         int i;
1020         struct stmmac_priv *priv = netdev_priv(dev);
1021         unsigned int bfsize = 0;
1022         int ret = -ENOMEM;
1023
1024         if (priv->hw->mode->set_16kib_bfsize)
1025                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1026
1027         if (bfsize < BUF_SIZE_16KiB)
1028                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1029
1030         priv->dma_buf_sz = bfsize;
1031
1032         netif_dbg(priv, probe, priv->dev,
1033                   "(%s) dma_rx_phy=0x%08x dma_tx_phy=0x%08x\n",
1034                   __func__, (u32)priv->dma_rx_phy, (u32)priv->dma_tx_phy);
1035
1036         /* RX INITIALIZATION */
1037         netif_dbg(priv, probe, priv->dev,
1038                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1039
1040         for (i = 0; i < DMA_RX_SIZE; i++) {
1041                 struct dma_desc *p;
1042                 if (priv->extend_desc)
1043                         p = &((priv->dma_erx + i)->basic);
1044                 else
1045                         p = priv->dma_rx + i;
1046
1047                 ret = stmmac_init_rx_buffers(priv, p, i, flags);
1048                 if (ret)
1049                         goto err_init_rx_buffers;
1050
1051                 netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1052                           priv->rx_skbuff[i], priv->rx_skbuff[i]->data,
1053                           (unsigned int)priv->rx_skbuff_dma[i]);
1054         }
1055         priv->cur_rx = 0;
1056         priv->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1057         buf_sz = bfsize;
1058
1059         /* Setup the chained descriptor addresses */
1060         if (priv->mode == STMMAC_CHAIN_MODE) {
1061                 if (priv->extend_desc) {
1062                         priv->hw->mode->init(priv->dma_erx, priv->dma_rx_phy,
1063                                              DMA_RX_SIZE, 1);
1064                         priv->hw->mode->init(priv->dma_etx, priv->dma_tx_phy,
1065                                              DMA_TX_SIZE, 1);
1066                 } else {
1067                         priv->hw->mode->init(priv->dma_rx, priv->dma_rx_phy,
1068                                              DMA_RX_SIZE, 0);
1069                         priv->hw->mode->init(priv->dma_tx, priv->dma_tx_phy,
1070                                              DMA_TX_SIZE, 0);
1071                 }
1072         }
1073
1074         /* TX INITIALIZATION */
1075         for (i = 0; i < DMA_TX_SIZE; i++) {
1076                 struct dma_desc *p;
1077                 if (priv->extend_desc)
1078                         p = &((priv->dma_etx + i)->basic);
1079                 else
1080                         p = priv->dma_tx + i;
1081
1082                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1083                         p->des0 = 0;
1084                         p->des1 = 0;
1085                         p->des2 = 0;
1086                         p->des3 = 0;
1087                 } else {
1088                         p->des2 = 0;
1089                 }
1090
1091                 priv->tx_skbuff_dma[i].buf = 0;
1092                 priv->tx_skbuff_dma[i].map_as_page = false;
1093                 priv->tx_skbuff_dma[i].len = 0;
1094                 priv->tx_skbuff_dma[i].last_segment = false;
1095                 priv->tx_skbuff[i] = NULL;
1096         }
1097
1098         priv->dirty_tx = 0;
1099         priv->cur_tx = 0;
1100         netdev_reset_queue(priv->dev);
1101
1102         stmmac_clear_descriptors(priv);
1103
1104         if (netif_msg_hw(priv))
1105                 stmmac_display_rings(priv);
1106
1107         return 0;
1108 err_init_rx_buffers:
1109         while (--i >= 0)
1110                 stmmac_free_rx_buffers(priv, i);
1111         return ret;
1112 }
1113
1114 static void dma_free_rx_skbufs(struct stmmac_priv *priv)
1115 {
1116         int i;
1117
1118         for (i = 0; i < DMA_RX_SIZE; i++)
1119                 stmmac_free_rx_buffers(priv, i);
1120 }
1121
1122 static void dma_free_tx_skbufs(struct stmmac_priv *priv)
1123 {
1124         int i;
1125
1126         for (i = 0; i < DMA_TX_SIZE; i++) {
1127                 if (priv->tx_skbuff_dma[i].buf) {
1128                         if (priv->tx_skbuff_dma[i].map_as_page)
1129                                 dma_unmap_page(priv->device,
1130                                                priv->tx_skbuff_dma[i].buf,
1131                                                priv->tx_skbuff_dma[i].len,
1132                                                DMA_TO_DEVICE);
1133                         else
1134                                 dma_unmap_single(priv->device,
1135                                                  priv->tx_skbuff_dma[i].buf,
1136                                                  priv->tx_skbuff_dma[i].len,
1137                                                  DMA_TO_DEVICE);
1138                 }
1139
1140                 if (priv->tx_skbuff[i]) {
1141                         dev_kfree_skb_any(priv->tx_skbuff[i]);
1142                         priv->tx_skbuff[i] = NULL;
1143                         priv->tx_skbuff_dma[i].buf = 0;
1144                         priv->tx_skbuff_dma[i].map_as_page = false;
1145                 }
1146         }
1147 }
1148
1149 /**
1150  * alloc_dma_desc_resources - alloc TX/RX resources.
1151  * @priv: private structure
1152  * Description: according to which descriptor can be used (extend or basic)
1153  * this function allocates the resources for TX and RX paths. In case of
1154  * reception, for example, it pre-allocated the RX socket buffer in order to
1155  * allow zero-copy mechanism.
1156  */
1157 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1158 {
1159         int ret = -ENOMEM;
1160
1161         priv->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE, sizeof(dma_addr_t),
1162                                             GFP_KERNEL);
1163         if (!priv->rx_skbuff_dma)
1164                 return -ENOMEM;
1165
1166         priv->rx_skbuff = kmalloc_array(DMA_RX_SIZE, sizeof(struct sk_buff *),
1167                                         GFP_KERNEL);
1168         if (!priv->rx_skbuff)
1169                 goto err_rx_skbuff;
1170
1171         priv->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1172                                             sizeof(*priv->tx_skbuff_dma),
1173                                             GFP_KERNEL);
1174         if (!priv->tx_skbuff_dma)
1175                 goto err_tx_skbuff_dma;
1176
1177         priv->tx_skbuff = kmalloc_array(DMA_TX_SIZE, sizeof(struct sk_buff *),
1178                                         GFP_KERNEL);
1179         if (!priv->tx_skbuff)
1180                 goto err_tx_skbuff;
1181
1182         if (priv->extend_desc) {
1183                 priv->dma_erx = dma_zalloc_coherent(priv->device, DMA_RX_SIZE *
1184                                                     sizeof(struct
1185                                                            dma_extended_desc),
1186                                                     &priv->dma_rx_phy,
1187                                                     GFP_KERNEL);
1188                 if (!priv->dma_erx)
1189                         goto err_dma;
1190
1191                 priv->dma_etx = dma_zalloc_coherent(priv->device, DMA_TX_SIZE *
1192                                                     sizeof(struct
1193                                                            dma_extended_desc),
1194                                                     &priv->dma_tx_phy,
1195                                                     GFP_KERNEL);
1196                 if (!priv->dma_etx) {
1197                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1198                                           sizeof(struct dma_extended_desc),
1199                                           priv->dma_erx, priv->dma_rx_phy);
1200                         goto err_dma;
1201                 }
1202         } else {
1203                 priv->dma_rx = dma_zalloc_coherent(priv->device, DMA_RX_SIZE *
1204                                                    sizeof(struct dma_desc),
1205                                                    &priv->dma_rx_phy,
1206                                                    GFP_KERNEL);
1207                 if (!priv->dma_rx)
1208                         goto err_dma;
1209
1210                 priv->dma_tx = dma_zalloc_coherent(priv->device, DMA_TX_SIZE *
1211                                                    sizeof(struct dma_desc),
1212                                                    &priv->dma_tx_phy,
1213                                                    GFP_KERNEL);
1214                 if (!priv->dma_tx) {
1215                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1216                                           sizeof(struct dma_desc),
1217                                           priv->dma_rx, priv->dma_rx_phy);
1218                         goto err_dma;
1219                 }
1220         }
1221
1222         return 0;
1223
1224 err_dma:
1225         kfree(priv->tx_skbuff);
1226 err_tx_skbuff:
1227         kfree(priv->tx_skbuff_dma);
1228 err_tx_skbuff_dma:
1229         kfree(priv->rx_skbuff);
1230 err_rx_skbuff:
1231         kfree(priv->rx_skbuff_dma);
1232         return ret;
1233 }
1234
1235 static void free_dma_desc_resources(struct stmmac_priv *priv)
1236 {
1237         /* Release the DMA TX/RX socket buffers */
1238         dma_free_rx_skbufs(priv);
1239         dma_free_tx_skbufs(priv);
1240
1241         /* Free DMA regions of consistent memory previously allocated */
1242         if (!priv->extend_desc) {
1243                 dma_free_coherent(priv->device,
1244                                   DMA_TX_SIZE * sizeof(struct dma_desc),
1245                                   priv->dma_tx, priv->dma_tx_phy);
1246                 dma_free_coherent(priv->device,
1247                                   DMA_RX_SIZE * sizeof(struct dma_desc),
1248                                   priv->dma_rx, priv->dma_rx_phy);
1249         } else {
1250                 dma_free_coherent(priv->device, DMA_TX_SIZE *
1251                                   sizeof(struct dma_extended_desc),
1252                                   priv->dma_etx, priv->dma_tx_phy);
1253                 dma_free_coherent(priv->device, DMA_RX_SIZE *
1254                                   sizeof(struct dma_extended_desc),
1255                                   priv->dma_erx, priv->dma_rx_phy);
1256         }
1257         kfree(priv->rx_skbuff_dma);
1258         kfree(priv->rx_skbuff);
1259         kfree(priv->tx_skbuff_dma);
1260         kfree(priv->tx_skbuff);
1261 }
1262
1263 /**
1264  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1265  *  @priv: driver private structure
1266  *  Description: It is used for enabling the rx queues in the MAC
1267  */
1268 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1269 {
1270         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1271         int queue;
1272         u8 mode;
1273
1274         for (queue = 0; queue < rx_queues_count; queue++) {
1275                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1276                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1277         }
1278 }
1279
1280 /**
1281  * stmmac_start_rx_dma - start RX DMA channel
1282  * @priv: driver private structure
1283  * @chan: RX channel index
1284  * Description:
1285  * This starts a RX DMA channel
1286  */
1287 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1288 {
1289         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1290         priv->hw->dma->start_rx(priv->ioaddr, chan);
1291 }
1292
1293 /**
1294  * stmmac_start_tx_dma - start TX DMA channel
1295  * @priv: driver private structure
1296  * @chan: TX channel index
1297  * Description:
1298  * This starts a TX DMA channel
1299  */
1300 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1301 {
1302         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1303         priv->hw->dma->start_tx(priv->ioaddr, chan);
1304 }
1305
1306 /**
1307  * stmmac_stop_rx_dma - stop RX DMA channel
1308  * @priv: driver private structure
1309  * @chan: RX channel index
1310  * Description:
1311  * This stops a RX DMA channel
1312  */
1313 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1314 {
1315         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1316         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1317 }
1318
1319 /**
1320  * stmmac_stop_tx_dma - stop TX DMA channel
1321  * @priv: driver private structure
1322  * @chan: TX channel index
1323  * Description:
1324  * This stops a TX DMA channel
1325  */
1326 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1327 {
1328         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1329         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1330 }
1331
1332 /**
1333  * stmmac_start_all_dma - start all RX and TX DMA channels
1334  * @priv: driver private structure
1335  * Description:
1336  * This starts all the RX and TX DMA channels
1337  */
1338 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1339 {
1340         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1341         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1342         u32 chan = 0;
1343
1344         for (chan = 0; chan < rx_channels_count; chan++)
1345                 stmmac_start_rx_dma(priv, chan);
1346
1347         for (chan = 0; chan < tx_channels_count; chan++)
1348                 stmmac_start_tx_dma(priv, chan);
1349 }
1350
1351 /**
1352  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1353  * @priv: driver private structure
1354  * Description:
1355  * This stops the RX and TX DMA channels
1356  */
1357 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1358 {
1359         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1360         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1361         u32 chan = 0;
1362
1363         for (chan = 0; chan < rx_channels_count; chan++)
1364                 stmmac_stop_rx_dma(priv, chan);
1365
1366         for (chan = 0; chan < tx_channels_count; chan++)
1367                 stmmac_stop_tx_dma(priv, chan);
1368 }
1369
1370 /**
1371  *  stmmac_dma_operation_mode - HW DMA operation mode
1372  *  @priv: driver private structure
1373  *  Description: it is used for configuring the DMA operation mode register in
1374  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1375  */
1376 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1377 {
1378         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1379         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1380         int rxfifosz = priv->plat->rx_fifo_size;
1381         u32 txmode = 0;
1382         u32 rxmode = 0;
1383         u32 chan = 0;
1384
1385         if (rxfifosz == 0)
1386                 rxfifosz = priv->dma_cap.rx_fifo_size;
1387
1388         if (priv->plat->force_thresh_dma_mode) {
1389                 txmode = tc;
1390                 rxmode = tc;
1391         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1392                 /*
1393                  * In case of GMAC, SF mode can be enabled
1394                  * to perform the TX COE in HW. This depends on:
1395                  * 1) TX COE if actually supported
1396                  * 2) There is no bugged Jumbo frame support
1397                  *    that needs to not insert csum in the TDES.
1398                  */
1399                 txmode = SF_DMA_MODE;
1400                 rxmode = SF_DMA_MODE;
1401                 priv->xstats.threshold = SF_DMA_MODE;
1402         } else {
1403                 txmode = tc;
1404                 rxmode = SF_DMA_MODE;
1405         }
1406
1407         /* configure all channels */
1408         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1409                 for (chan = 0; chan < rx_channels_count; chan++)
1410                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1411                                                    rxfifosz);
1412
1413                 for (chan = 0; chan < tx_channels_count; chan++)
1414                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1415         } else {
1416                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1417                                         rxfifosz);
1418         }
1419 }
1420
1421 /**
1422  * stmmac_tx_clean - to manage the transmission completion
1423  * @priv: driver private structure
1424  * Description: it reclaims the transmit resources after transmission completes.
1425  */
1426 static void stmmac_tx_clean(struct stmmac_priv *priv)
1427 {
1428         unsigned int bytes_compl = 0, pkts_compl = 0;
1429         unsigned int entry = priv->dirty_tx;
1430
1431         netif_tx_lock(priv->dev);
1432
1433         priv->xstats.tx_clean++;
1434
1435         while (entry != priv->cur_tx) {
1436                 struct sk_buff *skb = priv->tx_skbuff[entry];
1437                 struct dma_desc *p;
1438                 int status;
1439
1440                 if (priv->extend_desc)
1441                         p = (struct dma_desc *)(priv->dma_etx + entry);
1442                 else
1443                         p = priv->dma_tx + entry;
1444
1445                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1446                                                       &priv->xstats, p,
1447                                                       priv->ioaddr);
1448                 /* Check if the descriptor is owned by the DMA */
1449                 if (unlikely(status & tx_dma_own))
1450                         break;
1451
1452                 /* Just consider the last segment and ...*/
1453                 if (likely(!(status & tx_not_ls))) {
1454                         /* ... verify the status error condition */
1455                         if (unlikely(status & tx_err)) {
1456                                 priv->dev->stats.tx_errors++;
1457                         } else {
1458                                 priv->dev->stats.tx_packets++;
1459                                 priv->xstats.tx_pkt_n++;
1460                         }
1461                         stmmac_get_tx_hwtstamp(priv, p, skb);
1462                 }
1463
1464                 if (likely(priv->tx_skbuff_dma[entry].buf)) {
1465                         if (priv->tx_skbuff_dma[entry].map_as_page)
1466                                 dma_unmap_page(priv->device,
1467                                                priv->tx_skbuff_dma[entry].buf,
1468                                                priv->tx_skbuff_dma[entry].len,
1469                                                DMA_TO_DEVICE);
1470                         else
1471                                 dma_unmap_single(priv->device,
1472                                                  priv->tx_skbuff_dma[entry].buf,
1473                                                  priv->tx_skbuff_dma[entry].len,
1474                                                  DMA_TO_DEVICE);
1475                         priv->tx_skbuff_dma[entry].buf = 0;
1476                         priv->tx_skbuff_dma[entry].len = 0;
1477                         priv->tx_skbuff_dma[entry].map_as_page = false;
1478                 }
1479
1480                 if (priv->hw->mode->clean_desc3)
1481                         priv->hw->mode->clean_desc3(priv, p);
1482
1483                 priv->tx_skbuff_dma[entry].last_segment = false;
1484                 priv->tx_skbuff_dma[entry].is_jumbo = false;
1485
1486                 if (likely(skb != NULL)) {
1487                         pkts_compl++;
1488                         bytes_compl += skb->len;
1489                         dev_consume_skb_any(skb);
1490                         priv->tx_skbuff[entry] = NULL;
1491                 }
1492
1493                 priv->hw->desc->release_tx_desc(p, priv->mode);
1494
1495                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1496         }
1497         priv->dirty_tx = entry;
1498
1499         netdev_completed_queue(priv->dev, pkts_compl, bytes_compl);
1500
1501         if (unlikely(netif_queue_stopped(priv->dev) &&
1502             stmmac_tx_avail(priv) > STMMAC_TX_THRESH)) {
1503                 netif_dbg(priv, tx_done, priv->dev,
1504                           "%s: restart transmit\n", __func__);
1505                 netif_wake_queue(priv->dev);
1506         }
1507
1508         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1509                 stmmac_enable_eee_mode(priv);
1510                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1511         }
1512         netif_tx_unlock(priv->dev);
1513 }
1514
1515 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1516 {
1517         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1518 }
1519
1520 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1521 {
1522         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1523 }
1524
1525 /**
1526  * stmmac_tx_err - to manage the tx error
1527  * @priv: driver private structure
1528  * @chan: channel index
1529  * Description: it cleans the descriptors and restarts the transmission
1530  * in case of transmission errors.
1531  */
1532 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1533 {
1534         int i;
1535         netif_stop_queue(priv->dev);
1536
1537         stmmac_stop_tx_dma(priv, chan);
1538         dma_free_tx_skbufs(priv);
1539         for (i = 0; i < DMA_TX_SIZE; i++)
1540                 if (priv->extend_desc)
1541                         priv->hw->desc->init_tx_desc(&priv->dma_etx[i].basic,
1542                                                      priv->mode,
1543                                                      (i == DMA_TX_SIZE - 1));
1544                 else
1545                         priv->hw->desc->init_tx_desc(&priv->dma_tx[i],
1546                                                      priv->mode,
1547                                                      (i == DMA_TX_SIZE - 1));
1548         priv->dirty_tx = 0;
1549         priv->cur_tx = 0;
1550         netdev_reset_queue(priv->dev);
1551         stmmac_start_tx_dma(priv, chan);
1552
1553         priv->dev->stats.tx_errors++;
1554         netif_wake_queue(priv->dev);
1555 }
1556
1557 /**
1558  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1559  *  @priv: driver private structure
1560  *  @txmode: TX operating mode
1561  *  @rxmode: RX operating mode
1562  *  @chan: channel index
1563  *  Description: it is used for configuring of the DMA operation mode in
1564  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1565  *  mode.
1566  */
1567 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1568                                           u32 rxmode, u32 chan)
1569 {
1570         int rxfifosz = priv->plat->rx_fifo_size;
1571
1572         if (rxfifosz == 0)
1573                 rxfifosz = priv->dma_cap.rx_fifo_size;
1574
1575         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1576                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1577                                            rxfifosz);
1578                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan);
1579         } else {
1580                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1581                                         rxfifosz);
1582         }
1583 }
1584
1585 /**
1586  * stmmac_dma_interrupt - DMA ISR
1587  * @priv: driver private structure
1588  * Description: this is the DMA ISR. It is called by the main ISR.
1589  * It calls the dwmac dma routine and schedule poll method in case of some
1590  * work can be done.
1591  */
1592 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
1593 {
1594         u32 tx_channel_count = priv->plat->tx_queues_to_use;
1595         int status;
1596         u32 chan;
1597
1598         for (chan = 0; chan < tx_channel_count; chan++) {
1599                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
1600                                                       &priv->xstats, chan);
1601                 if (likely((status & handle_rx)) || (status & handle_tx)) {
1602                         if (likely(napi_schedule_prep(&priv->napi))) {
1603                                 stmmac_disable_dma_irq(priv, chan);
1604                                 __napi_schedule(&priv->napi);
1605                         }
1606                 }
1607
1608                 if (unlikely(status & tx_hard_error_bump_tc)) {
1609                         /* Try to bump up the dma threshold on this failure */
1610                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
1611                             (tc <= 256)) {
1612                                 tc += 64;
1613                                 if (priv->plat->force_thresh_dma_mode)
1614                                         stmmac_set_dma_operation_mode(priv,
1615                                                                       tc,
1616                                                                       tc,
1617                                                                       chan);
1618                                 else
1619                                         stmmac_set_dma_operation_mode(priv,
1620                                                                     tc,
1621                                                                     SF_DMA_MODE,
1622                                                                     chan);
1623                                 priv->xstats.threshold = tc;
1624                         }
1625                 } else if (unlikely(status == tx_hard_error)) {
1626                         stmmac_tx_err(priv, chan);
1627                 }
1628         }
1629 }
1630
1631 /**
1632  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
1633  * @priv: driver private structure
1634  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
1635  */
1636 static void stmmac_mmc_setup(struct stmmac_priv *priv)
1637 {
1638         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
1639                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
1640
1641         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1642                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
1643                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
1644         } else {
1645                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
1646                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
1647         }
1648
1649         dwmac_mmc_intr_all_mask(priv->mmcaddr);
1650
1651         if (priv->dma_cap.rmon) {
1652                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
1653                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
1654         } else
1655                 netdev_info(priv->dev, "No MAC Management Counters available\n");
1656 }
1657
1658 /**
1659  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
1660  * @priv: driver private structure
1661  * Description: select the Enhanced/Alternate or Normal descriptors.
1662  * In case of Enhanced/Alternate, it checks if the extended descriptors are
1663  * supported by the HW capability register.
1664  */
1665 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
1666 {
1667         if (priv->plat->enh_desc) {
1668                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
1669
1670                 /* GMAC older than 3.50 has no extended descriptors */
1671                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
1672                         dev_info(priv->device, "Enabled extended descriptors\n");
1673                         priv->extend_desc = 1;
1674                 } else
1675                         dev_warn(priv->device, "Extended descriptors not supported\n");
1676
1677                 priv->hw->desc = &enh_desc_ops;
1678         } else {
1679                 dev_info(priv->device, "Normal descriptors\n");
1680                 priv->hw->desc = &ndesc_ops;
1681         }
1682 }
1683
1684 /**
1685  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
1686  * @priv: driver private structure
1687  * Description:
1688  *  new GMAC chip generations have a new register to indicate the
1689  *  presence of the optional feature/functions.
1690  *  This can be also used to override the value passed through the
1691  *  platform and necessary for old MAC10/100 and GMAC chips.
1692  */
1693 static int stmmac_get_hw_features(struct stmmac_priv *priv)
1694 {
1695         u32 ret = 0;
1696
1697         if (priv->hw->dma->get_hw_feature) {
1698                 priv->hw->dma->get_hw_feature(priv->ioaddr,
1699                                               &priv->dma_cap);
1700                 ret = 1;
1701         }
1702
1703         return ret;
1704 }
1705
1706 /**
1707  * stmmac_check_ether_addr - check if the MAC addr is valid
1708  * @priv: driver private structure
1709  * Description:
1710  * it is to verify if the MAC address is valid, in case of failures it
1711  * generates a random MAC address
1712  */
1713 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
1714 {
1715         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
1716                 priv->hw->mac->get_umac_addr(priv->hw,
1717                                              priv->dev->dev_addr, 0);
1718                 if (!is_valid_ether_addr(priv->dev->dev_addr))
1719                         eth_hw_addr_random(priv->dev);
1720                 netdev_info(priv->dev, "device MAC address %pM\n",
1721                             priv->dev->dev_addr);
1722         }
1723 }
1724
1725 /**
1726  * stmmac_init_dma_engine - DMA init.
1727  * @priv: driver private structure
1728  * Description:
1729  * It inits the DMA invoking the specific MAC/GMAC callback.
1730  * Some DMA parameters can be passed from the platform;
1731  * in case of these are not passed a default is kept for the MAC or GMAC.
1732  */
1733 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
1734 {
1735         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1736         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1737         u32 dummy_dma_rx_phy = 0;
1738         u32 dummy_dma_tx_phy = 0;
1739         u32 chan = 0;
1740         int atds = 0;
1741         int ret = 0;
1742
1743         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
1744                 dev_err(priv->device, "Invalid DMA configuration\n");
1745                 return -EINVAL;
1746         }
1747
1748         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
1749                 atds = 1;
1750
1751         ret = priv->hw->dma->reset(priv->ioaddr);
1752         if (ret) {
1753                 dev_err(priv->device, "Failed to reset the dma\n");
1754                 return ret;
1755         }
1756
1757         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1758                 /* DMA Configuration */
1759                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
1760                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
1761
1762                 /* DMA RX Channel Configuration */
1763                 for (chan = 0; chan < rx_channels_count; chan++) {
1764                         priv->hw->dma->init_rx_chan(priv->ioaddr,
1765                                                     priv->plat->dma_cfg,
1766                                                     priv->dma_rx_phy, chan);
1767
1768                         priv->rx_tail_addr = priv->dma_rx_phy +
1769                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
1770                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
1771                                                        priv->rx_tail_addr,
1772                                                        chan);
1773                 }
1774
1775                 /* DMA TX Channel Configuration */
1776                 for (chan = 0; chan < tx_channels_count; chan++) {
1777                         priv->hw->dma->init_chan(priv->ioaddr,
1778                                                         priv->plat->dma_cfg,
1779                                                         chan);
1780
1781                         priv->hw->dma->init_tx_chan(priv->ioaddr,
1782                                                     priv->plat->dma_cfg,
1783                                                     priv->dma_tx_phy, chan);
1784
1785                         priv->tx_tail_addr = priv->dma_tx_phy +
1786                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
1787                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
1788                                                        priv->tx_tail_addr,
1789                                                        chan);
1790                 }
1791         } else {
1792                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
1793                                     priv->dma_tx_phy, priv->dma_rx_phy, atds);
1794         }
1795
1796         if (priv->plat->axi && priv->hw->dma->axi)
1797                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
1798
1799         return ret;
1800 }
1801
1802 /**
1803  * stmmac_tx_timer - mitigation sw timer for tx.
1804  * @data: data pointer
1805  * Description:
1806  * This is the timer handler to directly invoke the stmmac_tx_clean.
1807  */
1808 static void stmmac_tx_timer(unsigned long data)
1809 {
1810         struct stmmac_priv *priv = (struct stmmac_priv *)data;
1811
1812         stmmac_tx_clean(priv);
1813 }
1814
1815 /**
1816  * stmmac_init_tx_coalesce - init tx mitigation options.
1817  * @priv: driver private structure
1818  * Description:
1819  * This inits the transmit coalesce parameters: i.e. timer rate,
1820  * timer handler and default threshold used for enabling the
1821  * interrupt on completion bit.
1822  */
1823 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
1824 {
1825         priv->tx_coal_frames = STMMAC_TX_FRAMES;
1826         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
1827         init_timer(&priv->txtimer);
1828         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
1829         priv->txtimer.data = (unsigned long)priv;
1830         priv->txtimer.function = stmmac_tx_timer;
1831         add_timer(&priv->txtimer);
1832 }
1833
1834 static void stmmac_set_rings_length(struct stmmac_priv *priv)
1835 {
1836         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1837         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1838         u32 chan;
1839
1840         /* set TX ring length */
1841         if (priv->hw->dma->set_tx_ring_len) {
1842                 for (chan = 0; chan < tx_channels_count; chan++)
1843                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
1844                                                        (DMA_TX_SIZE - 1), chan);
1845         }
1846
1847         /* set RX ring length */
1848         if (priv->hw->dma->set_rx_ring_len) {
1849                 for (chan = 0; chan < rx_channels_count; chan++)
1850                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
1851                                                        (DMA_RX_SIZE - 1), chan);
1852         }
1853 }
1854
1855 /**
1856  *  stmmac_set_tx_queue_weight - Set TX queue weight
1857  *  @priv: driver private structure
1858  *  Description: It is used for setting TX queues weight
1859  */
1860 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
1861 {
1862         u32 tx_queues_count = priv->plat->tx_queues_to_use;
1863         u32 weight;
1864         u32 queue;
1865
1866         for (queue = 0; queue < tx_queues_count; queue++) {
1867                 weight = priv->plat->tx_queues_cfg[queue].weight;
1868                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
1869         }
1870 }
1871
1872 /**
1873  *  stmmac_configure_cbs - Configure CBS in TX queue
1874  *  @priv: driver private structure
1875  *  Description: It is used for configuring CBS in AVB TX queues
1876  */
1877 static void stmmac_configure_cbs(struct stmmac_priv *priv)
1878 {
1879         u32 tx_queues_count = priv->plat->tx_queues_to_use;
1880         u32 mode_to_use;
1881         u32 queue;
1882
1883         for (queue = 0; queue < tx_queues_count; queue++) {
1884                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
1885                 if (mode_to_use == MTL_QUEUE_DCB)
1886                         continue;
1887
1888                 priv->hw->mac->config_cbs(priv->hw,
1889                                 priv->plat->tx_queues_cfg[queue].send_slope,
1890                                 priv->plat->tx_queues_cfg[queue].idle_slope,
1891                                 priv->plat->tx_queues_cfg[queue].high_credit,
1892                                 priv->plat->tx_queues_cfg[queue].low_credit,
1893                                 queue);
1894         }
1895 }
1896
1897 /**
1898  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
1899  *  @priv: driver private structure
1900  *  Description: It is used for mapping RX queues to RX dma channels
1901  */
1902 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
1903 {
1904         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1905         u32 queue;
1906         u32 chan;
1907
1908         for (queue = 0; queue < rx_queues_count; queue++) {
1909                 chan = priv->plat->rx_queues_cfg[queue].chan;
1910                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
1911         }
1912 }
1913
1914 /**
1915  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
1916  *  @priv: driver private structure
1917  *  Description: It is used for configuring the RX Queue Priority
1918  */
1919 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
1920 {
1921         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1922         u32 queue;
1923         u32 prio;
1924
1925         for (queue = 0; queue < rx_queues_count; queue++) {
1926                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
1927                         continue;
1928
1929                 prio = priv->plat->rx_queues_cfg[queue].prio;
1930                 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
1931         }
1932 }
1933
1934 /**
1935  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
1936  *  @priv: driver private structure
1937  *  Description: It is used for configuring the TX Queue Priority
1938  */
1939 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
1940 {
1941         u32 tx_queues_count = priv->plat->tx_queues_to_use;
1942         u32 queue;
1943         u32 prio;
1944
1945         for (queue = 0; queue < tx_queues_count; queue++) {
1946                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
1947                         continue;
1948
1949                 prio = priv->plat->tx_queues_cfg[queue].prio;
1950                 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
1951         }
1952 }
1953
1954 /**
1955  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
1956  *  @priv: driver private structure
1957  *  Description: It is used for configuring the RX queue routing
1958  */
1959 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
1960 {
1961         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1962         u32 queue;
1963         u8 packet;
1964
1965         for (queue = 0; queue < rx_queues_count; queue++) {
1966                 /* no specific packet type routing specified for the queue */
1967                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
1968                         continue;
1969
1970                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
1971                 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue);
1972         }
1973 }
1974
1975 /**
1976  *  stmmac_mtl_configuration - Configure MTL
1977  *  @priv: driver private structure
1978  *  Description: It is used for configurring MTL
1979  */
1980 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
1981 {
1982         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1983         u32 tx_queues_count = priv->plat->tx_queues_to_use;
1984
1985         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
1986                 stmmac_set_tx_queue_weight(priv);
1987
1988         /* Configure MTL RX algorithms */
1989         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
1990                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
1991                                                 priv->plat->rx_sched_algorithm);
1992
1993         /* Configure MTL TX algorithms */
1994         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
1995                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
1996                                                 priv->plat->tx_sched_algorithm);
1997
1998         /* Configure CBS in AVB TX queues */
1999         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2000                 stmmac_configure_cbs(priv);
2001
2002         /* Map RX MTL to DMA channels */
2003         if (rx_queues_count > 1 && priv->hw->mac->map_mtl_to_dma)
2004                 stmmac_rx_queue_dma_chan_map(priv);
2005
2006         /* Enable MAC RX Queues */
2007         if (priv->hw->mac->rx_queue_enable)
2008                 stmmac_mac_enable_rx_queues(priv);
2009
2010         /* Set RX priorities */
2011         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
2012                 stmmac_mac_config_rx_queues_prio(priv);
2013
2014         /* Set TX priorities */
2015         if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
2016                 stmmac_mac_config_tx_queues_prio(priv);
2017
2018         /* Set RX routing */
2019         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
2020                 stmmac_mac_config_rx_queues_routing(priv);
2021 }
2022
2023 /**
2024  * stmmac_hw_setup - setup mac in a usable state.
2025  *  @dev : pointer to the device structure.
2026  *  Description:
2027  *  this is the main function to setup the HW in a usable state because the
2028  *  dma engine is reset, the core registers are configured (e.g. AXI,
2029  *  Checksum features, timers). The DMA is ready to start receiving and
2030  *  transmitting.
2031  *  Return value:
2032  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2033  *  file on failure.
2034  */
2035 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2036 {
2037         struct stmmac_priv *priv = netdev_priv(dev);
2038         u32 rx_cnt = priv->plat->rx_queues_to_use;
2039         u32 tx_cnt = priv->plat->tx_queues_to_use;
2040         u32 chan;
2041         int ret;
2042
2043         /* DMA initialization and SW reset */
2044         ret = stmmac_init_dma_engine(priv);
2045         if (ret < 0) {
2046                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2047                            __func__);
2048                 return ret;
2049         }
2050
2051         /* Copy the MAC addr into the HW  */
2052         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2053
2054         /* PS and related bits will be programmed according to the speed */
2055         if (priv->hw->pcs) {
2056                 int speed = priv->plat->mac_port_sel_speed;
2057
2058                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2059                     (speed == SPEED_1000)) {
2060                         priv->hw->ps = speed;
2061                 } else {
2062                         dev_warn(priv->device, "invalid port speed\n");
2063                         priv->hw->ps = 0;
2064                 }
2065         }
2066
2067         /* Initialize the MAC Core */
2068         priv->hw->mac->core_init(priv->hw, dev->mtu);
2069
2070         /* Initialize MTL*/
2071         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2072                 stmmac_mtl_configuration(priv);
2073
2074         ret = priv->hw->mac->rx_ipc(priv->hw);
2075         if (!ret) {
2076                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2077                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2078                 priv->hw->rx_csum = 0;
2079         }
2080
2081         /* Enable the MAC Rx/Tx */
2082         priv->hw->mac->set_mac(priv->ioaddr, true);
2083
2084         /* Set the HW DMA mode and the COE */
2085         stmmac_dma_operation_mode(priv);
2086
2087         stmmac_mmc_setup(priv);
2088
2089         if (init_ptp) {
2090                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2091                 if (ret < 0)
2092                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2093
2094                 ret = stmmac_init_ptp(priv);
2095                 if (ret == -EOPNOTSUPP)
2096                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2097                 else if (ret)
2098                         netdev_warn(priv->dev, "PTP init failed\n");
2099         }
2100
2101 #ifdef CONFIG_DEBUG_FS
2102         ret = stmmac_init_fs(dev);
2103         if (ret < 0)
2104                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2105                             __func__);
2106 #endif
2107         /* Start the ball rolling... */
2108         stmmac_start_all_dma(priv);
2109
2110         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2111
2112         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2113                 priv->rx_riwt = MAX_DMA_RIWT;
2114                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2115         }
2116
2117         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2118                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2119
2120         /* set TX and RX rings length */
2121         stmmac_set_rings_length(priv);
2122
2123         /* Enable TSO */
2124         if (priv->tso) {
2125                 for (chan = 0; chan < tx_cnt; chan++)
2126                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2127         }
2128
2129         return 0;
2130 }
2131
2132 static void stmmac_hw_teardown(struct net_device *dev)
2133 {
2134         struct stmmac_priv *priv = netdev_priv(dev);
2135
2136         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2137 }
2138
2139 /**
2140  *  stmmac_open - open entry point of the driver
2141  *  @dev : pointer to the device structure.
2142  *  Description:
2143  *  This function is the open entry point of the driver.
2144  *  Return value:
2145  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2146  *  file on failure.
2147  */
2148 static int stmmac_open(struct net_device *dev)
2149 {
2150         struct stmmac_priv *priv = netdev_priv(dev);
2151         int ret;
2152
2153         stmmac_check_ether_addr(priv);
2154
2155         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2156             priv->hw->pcs != STMMAC_PCS_TBI &&
2157             priv->hw->pcs != STMMAC_PCS_RTBI) {
2158                 ret = stmmac_init_phy(dev);
2159                 if (ret) {
2160                         netdev_err(priv->dev,
2161                                    "%s: Cannot attach to PHY (error: %d)\n",
2162                                    __func__, ret);
2163                         return ret;
2164                 }
2165         }
2166
2167         /* Extra statistics */
2168         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2169         priv->xstats.threshold = tc;
2170
2171         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2172         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2173
2174         ret = alloc_dma_desc_resources(priv);
2175         if (ret < 0) {
2176                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2177                            __func__);
2178                 goto dma_desc_error;
2179         }
2180
2181         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2182         if (ret < 0) {
2183                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2184                            __func__);
2185                 goto init_error;
2186         }
2187
2188         ret = stmmac_hw_setup(dev, true);
2189         if (ret < 0) {
2190                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2191                 goto init_error;
2192         }
2193
2194         stmmac_init_tx_coalesce(priv);
2195
2196         if (dev->phydev)
2197                 phy_start(dev->phydev);
2198
2199         /* Request the IRQ lines */
2200         ret = request_irq(dev->irq, stmmac_interrupt,
2201                           IRQF_SHARED, dev->name, dev);
2202         if (unlikely(ret < 0)) {
2203                 netdev_err(priv->dev,
2204                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2205                            __func__, dev->irq, ret);
2206                 goto irq_error;
2207         }
2208
2209         /* Request the Wake IRQ in case of another line is used for WoL */
2210         if (priv->wol_irq != dev->irq) {
2211                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2212                                   IRQF_SHARED, dev->name, dev);
2213                 if (unlikely(ret < 0)) {
2214                         netdev_err(priv->dev,
2215                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2216                                    __func__, priv->wol_irq, ret);
2217                         goto wolirq_error;
2218                 }
2219         }
2220
2221         /* Request the IRQ lines */
2222         if (priv->lpi_irq > 0) {
2223                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2224                                   dev->name, dev);
2225                 if (unlikely(ret < 0)) {
2226                         netdev_err(priv->dev,
2227                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2228                                    __func__, priv->lpi_irq, ret);
2229                         goto lpiirq_error;
2230                 }
2231         }
2232
2233         napi_enable(&priv->napi);
2234         netif_start_queue(dev);
2235
2236         return 0;
2237
2238 lpiirq_error:
2239         if (priv->wol_irq != dev->irq)
2240                 free_irq(priv->wol_irq, dev);
2241 wolirq_error:
2242         free_irq(dev->irq, dev);
2243 irq_error:
2244         if (dev->phydev)
2245                 phy_stop(dev->phydev);
2246
2247         del_timer_sync(&priv->txtimer);
2248         stmmac_hw_teardown(dev);
2249 init_error:
2250         free_dma_desc_resources(priv);
2251 dma_desc_error:
2252         if (dev->phydev)
2253                 phy_disconnect(dev->phydev);
2254
2255         return ret;
2256 }
2257
2258 /**
2259  *  stmmac_release - close entry point of the driver
2260  *  @dev : device pointer.
2261  *  Description:
2262  *  This is the stop entry point of the driver.
2263  */
2264 static int stmmac_release(struct net_device *dev)
2265 {
2266         struct stmmac_priv *priv = netdev_priv(dev);
2267
2268         if (priv->eee_enabled)
2269                 del_timer_sync(&priv->eee_ctrl_timer);
2270
2271         /* Stop and disconnect the PHY */
2272         if (dev->phydev) {
2273                 phy_stop(dev->phydev);
2274                 phy_disconnect(dev->phydev);
2275         }
2276
2277         netif_stop_queue(dev);
2278
2279         napi_disable(&priv->napi);
2280
2281         del_timer_sync(&priv->txtimer);
2282
2283         /* Free the IRQ lines */
2284         free_irq(dev->irq, dev);
2285         if (priv->wol_irq != dev->irq)
2286                 free_irq(priv->wol_irq, dev);
2287         if (priv->lpi_irq > 0)
2288                 free_irq(priv->lpi_irq, dev);
2289
2290         /* Stop TX/RX DMA and clear the descriptors */
2291         stmmac_stop_all_dma(priv);
2292
2293         /* Release and free the Rx/Tx resources */
2294         free_dma_desc_resources(priv);
2295
2296         /* Disable the MAC Rx/Tx */
2297         priv->hw->mac->set_mac(priv->ioaddr, false);
2298
2299         netif_carrier_off(dev);
2300
2301 #ifdef CONFIG_DEBUG_FS
2302         stmmac_exit_fs(dev);
2303 #endif
2304
2305         stmmac_release_ptp(priv);
2306
2307         return 0;
2308 }
2309
2310 /**
2311  *  stmmac_tso_allocator - close entry point of the driver
2312  *  @priv: driver private structure
2313  *  @des: buffer start address
2314  *  @total_len: total length to fill in descriptors
2315  *  @last_segmant: condition for the last descriptor
2316  *  Description:
2317  *  This function fills descriptor and request new descriptors according to
2318  *  buffer length to fill
2319  */
2320 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2321                                  int total_len, bool last_segment)
2322 {
2323         struct dma_desc *desc;
2324         int tmp_len;
2325         u32 buff_size;
2326
2327         tmp_len = total_len;
2328
2329         while (tmp_len > 0) {
2330                 priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE);
2331                 desc = priv->dma_tx + priv->cur_tx;
2332
2333                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2334                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2335                             TSO_MAX_BUFF_SIZE : tmp_len;
2336
2337                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2338                         0, 1,
2339                         (last_segment) && (buff_size < TSO_MAX_BUFF_SIZE),
2340                         0, 0);
2341
2342                 tmp_len -= TSO_MAX_BUFF_SIZE;
2343         }
2344 }
2345
2346 /**
2347  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2348  *  @skb : the socket buffer
2349  *  @dev : device pointer
2350  *  Description: this is the transmit function that is called on TSO frames
2351  *  (support available on GMAC4 and newer chips).
2352  *  Diagram below show the ring programming in case of TSO frames:
2353  *
2354  *  First Descriptor
2355  *   --------
2356  *   | DES0 |---> buffer1 = L2/L3/L4 header
2357  *   | DES1 |---> TCP Payload (can continue on next descr...)
2358  *   | DES2 |---> buffer 1 and 2 len
2359  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2360  *   --------
2361  *      |
2362  *     ...
2363  *      |
2364  *   --------
2365  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2366  *   | DES1 | --|
2367  *   | DES2 | --> buffer 1 and 2 len
2368  *   | DES3 |
2369  *   --------
2370  *
2371  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2372  */
2373 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2374 {
2375         u32 pay_len, mss;
2376         int tmp_pay_len = 0;
2377         struct stmmac_priv *priv = netdev_priv(dev);
2378         int nfrags = skb_shinfo(skb)->nr_frags;
2379         unsigned int first_entry, des;
2380         struct dma_desc *desc, *first, *mss_desc = NULL;
2381         u8 proto_hdr_len;
2382         int i;
2383
2384         /* Compute header lengths */
2385         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2386
2387         /* Desc availability based on threshold should be enough safe */
2388         if (unlikely(stmmac_tx_avail(priv) <
2389                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2390                 if (!netif_queue_stopped(dev)) {
2391                         netif_stop_queue(dev);
2392                         /* This is a hard error, log it. */
2393                         netdev_err(priv->dev,
2394                                    "%s: Tx Ring full when queue awake\n",
2395                                    __func__);
2396                 }
2397                 return NETDEV_TX_BUSY;
2398         }
2399
2400         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2401
2402         mss = skb_shinfo(skb)->gso_size;
2403
2404         /* set new MSS value if needed */
2405         if (mss != priv->mss) {
2406                 mss_desc = priv->dma_tx + priv->cur_tx;
2407                 priv->hw->desc->set_mss(mss_desc, mss);
2408                 priv->mss = mss;
2409                 priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE);
2410         }
2411
2412         if (netif_msg_tx_queued(priv)) {
2413                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2414                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2415                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2416                         skb->data_len);
2417         }
2418
2419         first_entry = priv->cur_tx;
2420
2421         desc = priv->dma_tx + first_entry;
2422         first = desc;
2423
2424         /* first descriptor: fill Headers on Buf1 */
2425         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2426                              DMA_TO_DEVICE);
2427         if (dma_mapping_error(priv->device, des))
2428                 goto dma_map_err;
2429
2430         priv->tx_skbuff_dma[first_entry].buf = des;
2431         priv->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2432         priv->tx_skbuff[first_entry] = skb;
2433
2434         first->des0 = cpu_to_le32(des);
2435
2436         /* Fill start of payload in buff2 of first descriptor */
2437         if (pay_len)
2438                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2439
2440         /* If needed take extra descriptors to fill the remaining payload */
2441         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2442
2443         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0));
2444
2445         /* Prepare fragments */
2446         for (i = 0; i < nfrags; i++) {
2447                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2448
2449                 des = skb_frag_dma_map(priv->device, frag, 0,
2450                                        skb_frag_size(frag),
2451                                        DMA_TO_DEVICE);
2452                 if (dma_mapping_error(priv->device, des))
2453                         goto dma_map_err;
2454
2455                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2456                                      (i == nfrags - 1));
2457
2458                 priv->tx_skbuff_dma[priv->cur_tx].buf = des;
2459                 priv->tx_skbuff_dma[priv->cur_tx].len = skb_frag_size(frag);
2460                 priv->tx_skbuff[priv->cur_tx] = NULL;
2461                 priv->tx_skbuff_dma[priv->cur_tx].map_as_page = true;
2462         }
2463
2464         priv->tx_skbuff_dma[priv->cur_tx].last_segment = true;
2465
2466         priv->cur_tx = STMMAC_GET_ENTRY(priv->cur_tx, DMA_TX_SIZE);
2467
2468         if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) {
2469                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2470                           __func__);
2471                 netif_stop_queue(dev);
2472         }
2473
2474         dev->stats.tx_bytes += skb->len;
2475         priv->xstats.tx_tso_frames++;
2476         priv->xstats.tx_tso_nfrags += nfrags;
2477
2478         /* Manage tx mitigation */
2479         priv->tx_count_frames += nfrags + 1;
2480         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2481                 mod_timer(&priv->txtimer,
2482                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2483         } else {
2484                 priv->tx_count_frames = 0;
2485                 priv->hw->desc->set_tx_ic(desc);
2486                 priv->xstats.tx_set_ic_bit++;
2487         }
2488
2489         if (!priv->hwts_tx_en)
2490                 skb_tx_timestamp(skb);
2491
2492         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2493                      priv->hwts_tx_en)) {
2494                 /* declare that device is doing timestamping */
2495                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2496                 priv->hw->desc->enable_tx_timestamp(first);
2497         }
2498
2499         /* Complete the first descriptor before granting the DMA */
2500         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2501                         proto_hdr_len,
2502                         pay_len,
2503                         1, priv->tx_skbuff_dma[first_entry].last_segment,
2504                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2505
2506         /* If context desc is used to change MSS */
2507         if (mss_desc)
2508                 priv->hw->desc->set_tx_owner(mss_desc);
2509
2510         /* The own bit must be the latest setting done when prepare the
2511          * descriptor and then barrier is needed to make sure that
2512          * all is coherent before granting the DMA engine.
2513          */
2514         dma_wmb();
2515
2516         if (netif_msg_pktdata(priv)) {
2517                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2518                         __func__, priv->cur_tx, priv->dirty_tx, first_entry,
2519                         priv->cur_tx, first, nfrags);
2520
2521                 priv->hw->desc->display_ring((void *)priv->dma_tx, DMA_TX_SIZE,
2522                                              0);
2523
2524                 pr_info(">>> frame to be transmitted: ");
2525                 print_pkt(skb->data, skb_headlen(skb));
2526         }
2527
2528         netdev_sent_queue(dev, skb->len);
2529
2530         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr,
2531                                        STMMAC_CHAN0);
2532
2533         return NETDEV_TX_OK;
2534
2535 dma_map_err:
2536         dev_err(priv->device, "Tx dma map failed\n");
2537         dev_kfree_skb(skb);
2538         priv->dev->stats.tx_dropped++;
2539         return NETDEV_TX_OK;
2540 }
2541
2542 /**
2543  *  stmmac_xmit - Tx entry point of the driver
2544  *  @skb : the socket buffer
2545  *  @dev : device pointer
2546  *  Description : this is the tx entry point of the driver.
2547  *  It programs the chain or the ring and supports oversized frames
2548  *  and SG feature.
2549  */
2550 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2551 {
2552         struct stmmac_priv *priv = netdev_priv(dev);
2553         unsigned int nopaged_len = skb_headlen(skb);
2554         int i, csum_insertion = 0, is_jumbo = 0;
2555         int nfrags = skb_shinfo(skb)->nr_frags;
2556         unsigned int entry, first_entry;
2557         struct dma_desc *desc, *first;
2558         unsigned int enh_desc;
2559         unsigned int des;
2560
2561         /* Manage oversized TCP frames for GMAC4 device */
2562         if (skb_is_gso(skb) && priv->tso) {
2563                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2564                         return stmmac_tso_xmit(skb, dev);
2565         }
2566
2567         if (unlikely(stmmac_tx_avail(priv) < nfrags + 1)) {
2568                 if (!netif_queue_stopped(dev)) {
2569                         netif_stop_queue(dev);
2570                         /* This is a hard error, log it. */
2571                         netdev_err(priv->dev,
2572                                    "%s: Tx Ring full when queue awake\n",
2573                                    __func__);
2574                 }
2575                 return NETDEV_TX_BUSY;
2576         }
2577
2578         if (priv->tx_path_in_lpi_mode)
2579                 stmmac_disable_eee_mode(priv);
2580
2581         entry = priv->cur_tx;
2582         first_entry = entry;
2583
2584         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
2585
2586         if (likely(priv->extend_desc))
2587                 desc = (struct dma_desc *)(priv->dma_etx + entry);
2588         else
2589                 desc = priv->dma_tx + entry;
2590
2591         first = desc;
2592
2593         priv->tx_skbuff[first_entry] = skb;
2594
2595         enh_desc = priv->plat->enh_desc;
2596         /* To program the descriptors according to the size of the frame */
2597         if (enh_desc)
2598                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
2599
2600         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
2601                                          DWMAC_CORE_4_00)) {
2602                 entry = priv->hw->mode->jumbo_frm(priv, skb, csum_insertion);
2603                 if (unlikely(entry < 0))
2604                         goto dma_map_err;
2605         }
2606
2607         for (i = 0; i < nfrags; i++) {
2608                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2609                 int len = skb_frag_size(frag);
2610                 bool last_segment = (i == (nfrags - 1));
2611
2612                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
2613
2614                 if (likely(priv->extend_desc))
2615                         desc = (struct dma_desc *)(priv->dma_etx + entry);
2616                 else
2617                         desc = priv->dma_tx + entry;
2618
2619                 des = skb_frag_dma_map(priv->device, frag, 0, len,
2620                                        DMA_TO_DEVICE);
2621                 if (dma_mapping_error(priv->device, des))
2622                         goto dma_map_err; /* should reuse desc w/o issues */
2623
2624                 priv->tx_skbuff[entry] = NULL;
2625
2626                 priv->tx_skbuff_dma[entry].buf = des;
2627                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
2628                         desc->des0 = cpu_to_le32(des);
2629                 else
2630                         desc->des2 = cpu_to_le32(des);
2631
2632                 priv->tx_skbuff_dma[entry].map_as_page = true;
2633                 priv->tx_skbuff_dma[entry].len = len;
2634                 priv->tx_skbuff_dma[entry].last_segment = last_segment;
2635
2636                 /* Prepare the descriptor and set the own bit too */
2637                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
2638                                                 priv->mode, 1, last_segment);
2639         }
2640
2641         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
2642
2643         priv->cur_tx = entry;
2644
2645         if (netif_msg_pktdata(priv)) {
2646                 void *tx_head;
2647
2648                 netdev_dbg(priv->dev,
2649                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
2650                            __func__, priv->cur_tx, priv->dirty_tx, first_entry,
2651                            entry, first, nfrags);
2652
2653                 if (priv->extend_desc)
2654                         tx_head = (void *)priv->dma_etx;
2655                 else
2656                         tx_head = (void *)priv->dma_tx;
2657
2658                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
2659
2660                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
2661                 print_pkt(skb->data, skb->len);
2662         }
2663
2664         if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) {
2665                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2666                           __func__);
2667                 netif_stop_queue(dev);
2668         }
2669
2670         dev->stats.tx_bytes += skb->len;
2671
2672         /* According to the coalesce parameter the IC bit for the latest
2673          * segment is reset and the timer re-started to clean the tx status.
2674          * This approach takes care about the fragments: desc is the first
2675          * element in case of no SG.
2676          */
2677         priv->tx_count_frames += nfrags + 1;
2678         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2679                 mod_timer(&priv->txtimer,
2680                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2681         } else {
2682                 priv->tx_count_frames = 0;
2683                 priv->hw->desc->set_tx_ic(desc);
2684                 priv->xstats.tx_set_ic_bit++;
2685         }
2686
2687         if (!priv->hwts_tx_en)
2688                 skb_tx_timestamp(skb);
2689
2690         /* Ready to fill the first descriptor and set the OWN bit w/o any
2691          * problems because all the descriptors are actually ready to be
2692          * passed to the DMA engine.
2693          */
2694         if (likely(!is_jumbo)) {
2695                 bool last_segment = (nfrags == 0);
2696
2697                 des = dma_map_single(priv->device, skb->data,
2698                                      nopaged_len, DMA_TO_DEVICE);
2699                 if (dma_mapping_error(priv->device, des))
2700                         goto dma_map_err;
2701
2702                 priv->tx_skbuff_dma[first_entry].buf = des;
2703                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
2704                         first->des0 = cpu_to_le32(des);
2705                 else
2706                         first->des2 = cpu_to_le32(des);
2707
2708                 priv->tx_skbuff_dma[first_entry].len = nopaged_len;
2709                 priv->tx_skbuff_dma[first_entry].last_segment = last_segment;
2710
2711                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2712                              priv->hwts_tx_en)) {
2713                         /* declare that device is doing timestamping */
2714                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2715                         priv->hw->desc->enable_tx_timestamp(first);
2716                 }
2717
2718                 /* Prepare the first descriptor setting the OWN bit too */
2719                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
2720                                                 csum_insertion, priv->mode, 1,
2721                                                 last_segment);
2722
2723                 /* The own bit must be the latest setting done when prepare the
2724                  * descriptor and then barrier is needed to make sure that
2725                  * all is coherent before granting the DMA engine.
2726                  */
2727                 dma_wmb();
2728         }
2729
2730         netdev_sent_queue(dev, skb->len);
2731
2732         if (priv->synopsys_id < DWMAC_CORE_4_00)
2733                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
2734         else
2735                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, priv->tx_tail_addr,
2736                                                STMMAC_CHAN0);
2737
2738         return NETDEV_TX_OK;
2739
2740 dma_map_err:
2741         netdev_err(priv->dev, "Tx DMA map failed\n");
2742         dev_kfree_skb(skb);
2743         priv->dev->stats.tx_dropped++;
2744         return NETDEV_TX_OK;
2745 }
2746
2747 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
2748 {
2749         struct ethhdr *ehdr;
2750         u16 vlanid;
2751
2752         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
2753             NETIF_F_HW_VLAN_CTAG_RX &&
2754             !__vlan_get_tag(skb, &vlanid)) {
2755                 /* pop the vlan tag */
2756                 ehdr = (struct ethhdr *)skb->data;
2757                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
2758                 skb_pull(skb, VLAN_HLEN);
2759                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
2760         }
2761 }
2762
2763
2764 static inline int stmmac_rx_threshold_count(struct stmmac_priv *priv)
2765 {
2766         if (priv->rx_zeroc_thresh < STMMAC_RX_THRESH)
2767                 return 0;
2768
2769         return 1;
2770 }
2771
2772 /**
2773  * stmmac_rx_refill - refill used skb preallocated buffers
2774  * @priv: driver private structure
2775  * Description : this is to reallocate the skb for the reception process
2776  * that is based on zero-copy.
2777  */
2778 static inline void stmmac_rx_refill(struct stmmac_priv *priv)
2779 {
2780         int bfsize = priv->dma_buf_sz;
2781         unsigned int entry = priv->dirty_rx;
2782         int dirty = stmmac_rx_dirty(priv);
2783
2784         while (dirty-- > 0) {
2785                 struct dma_desc *p;
2786
2787                 if (priv->extend_desc)
2788                         p = (struct dma_desc *)(priv->dma_erx + entry);
2789                 else
2790                         p = priv->dma_rx + entry;
2791
2792                 if (likely(priv->rx_skbuff[entry] == NULL)) {
2793                         struct sk_buff *skb;
2794
2795                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
2796                         if (unlikely(!skb)) {
2797                                 /* so for a while no zero-copy! */
2798                                 priv->rx_zeroc_thresh = STMMAC_RX_THRESH;
2799                                 if (unlikely(net_ratelimit()))
2800                                         dev_err(priv->device,
2801                                                 "fail to alloc skb entry %d\n",
2802                                                 entry);
2803                                 break;
2804                         }
2805
2806                         priv->rx_skbuff[entry] = skb;
2807                         priv->rx_skbuff_dma[entry] =
2808                             dma_map_single(priv->device, skb->data, bfsize,
2809                                            DMA_FROM_DEVICE);
2810                         if (dma_mapping_error(priv->device,
2811                                               priv->rx_skbuff_dma[entry])) {
2812                                 netdev_err(priv->dev, "Rx DMA map failed\n");
2813                                 dev_kfree_skb(skb);
2814                                 break;
2815                         }
2816
2817                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
2818                                 p->des0 = cpu_to_le32(priv->rx_skbuff_dma[entry]);
2819                                 p->des1 = 0;
2820                         } else {
2821                                 p->des2 = cpu_to_le32(priv->rx_skbuff_dma[entry]);
2822                         }
2823                         if (priv->hw->mode->refill_desc3)
2824                                 priv->hw->mode->refill_desc3(priv, p);
2825
2826                         if (priv->rx_zeroc_thresh > 0)
2827                                 priv->rx_zeroc_thresh--;
2828
2829                         netif_dbg(priv, rx_status, priv->dev,
2830                                   "refill entry #%d\n", entry);
2831                 }
2832                 dma_wmb();
2833
2834                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
2835                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
2836                 else
2837                         priv->hw->desc->set_rx_owner(p);
2838
2839                 dma_wmb();
2840
2841                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
2842         }
2843         priv->dirty_rx = entry;
2844 }
2845
2846 /**
2847  * stmmac_rx - manage the receive process
2848  * @priv: driver private structure
2849  * @limit: napi bugget.
2850  * Description :  this the function called by the napi poll method.
2851  * It gets all the frames inside the ring.
2852  */
2853 static int stmmac_rx(struct stmmac_priv *priv, int limit)
2854 {
2855         unsigned int entry = priv->cur_rx;
2856         unsigned int next_entry;
2857         unsigned int count = 0;
2858         int coe = priv->hw->rx_csum;
2859
2860         if (netif_msg_rx_status(priv)) {
2861                 void *rx_head;
2862
2863                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
2864                 if (priv->extend_desc)
2865                         rx_head = (void *)priv->dma_erx;
2866                 else
2867                         rx_head = (void *)priv->dma_rx;
2868
2869                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
2870         }
2871         while (count < limit) {
2872                 int status;
2873                 struct dma_desc *p;
2874                 struct dma_desc *np;
2875
2876                 if (priv->extend_desc)
2877                         p = (struct dma_desc *)(priv->dma_erx + entry);
2878                 else
2879                         p = priv->dma_rx + entry;
2880
2881                 /* read the status of the incoming frame */
2882                 status = priv->hw->desc->rx_status(&priv->dev->stats,
2883                                                    &priv->xstats, p);
2884                 /* check if managed by the DMA otherwise go ahead */
2885                 if (unlikely(status & dma_own))
2886                         break;
2887
2888                 count++;
2889
2890                 priv->cur_rx = STMMAC_GET_ENTRY(priv->cur_rx, DMA_RX_SIZE);
2891                 next_entry = priv->cur_rx;
2892
2893                 if (priv->extend_desc)
2894                         np = (struct dma_desc *)(priv->dma_erx + next_entry);
2895                 else
2896                         np = priv->dma_rx + next_entry;
2897
2898                 prefetch(np);
2899
2900                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
2901                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
2902                                                            &priv->xstats,
2903                                                            priv->dma_erx +
2904                                                            entry);
2905                 if (unlikely(status == discard_frame)) {
2906                         priv->dev->stats.rx_errors++;
2907                         if (priv->hwts_rx_en && !priv->extend_desc) {
2908                                 /* DESC2 & DESC3 will be overwritten by device
2909                                  * with timestamp value, hence reinitialize
2910                                  * them in stmmac_rx_refill() function so that
2911                                  * device can reuse it.
2912                                  */
2913                                 priv->rx_skbuff[entry] = NULL;
2914                                 dma_unmap_single(priv->device,
2915                                                  priv->rx_skbuff_dma[entry],
2916                                                  priv->dma_buf_sz,
2917                                                  DMA_FROM_DEVICE);
2918                         }
2919                 } else {
2920                         struct sk_buff *skb;
2921                         int frame_len;
2922                         unsigned int des;
2923
2924                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
2925                                 des = le32_to_cpu(p->des0);
2926                         else
2927                                 des = le32_to_cpu(p->des2);
2928
2929                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
2930
2931                         /*  If frame length is greater than skb buffer size
2932                          *  (preallocated during init) then the packet is
2933                          *  ignored
2934                          */
2935                         if (frame_len > priv->dma_buf_sz) {
2936                                 netdev_err(priv->dev,
2937                                            "len %d larger than size (%d)\n",
2938                                            frame_len, priv->dma_buf_sz);
2939                                 priv->dev->stats.rx_length_errors++;
2940                                 break;
2941                         }
2942
2943                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
2944                          * Type frames (LLC/LLC-SNAP)
2945                          */
2946                         if (unlikely(status != llc_snap))
2947                                 frame_len -= ETH_FCS_LEN;
2948
2949                         if (netif_msg_rx_status(priv)) {
2950                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
2951                                            p, entry, des);
2952                                 if (frame_len > ETH_FRAME_LEN)
2953                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
2954                                                    frame_len, status);
2955                         }
2956
2957                         /* The zero-copy is always used for all the sizes
2958                          * in case of GMAC4 because it needs
2959                          * to refill the used descriptors, always.
2960                          */
2961                         if (unlikely(!priv->plat->has_gmac4 &&
2962                                      ((frame_len < priv->rx_copybreak) ||
2963                                      stmmac_rx_threshold_count(priv)))) {
2964                                 skb = netdev_alloc_skb_ip_align(priv->dev,
2965                                                                 frame_len);
2966                                 if (unlikely(!skb)) {
2967                                         if (net_ratelimit())
2968                                                 dev_warn(priv->device,
2969                                                          "packet dropped\n");
2970                                         priv->dev->stats.rx_dropped++;
2971                                         break;
2972                                 }
2973
2974                                 dma_sync_single_for_cpu(priv->device,
2975                                                         priv->rx_skbuff_dma
2976                                                         [entry], frame_len,
2977                                                         DMA_FROM_DEVICE);
2978                                 skb_copy_to_linear_data(skb,
2979                                                         priv->
2980                                                         rx_skbuff[entry]->data,
2981                                                         frame_len);
2982
2983                                 skb_put(skb, frame_len);
2984                                 dma_sync_single_for_device(priv->device,
2985                                                            priv->rx_skbuff_dma
2986                                                            [entry], frame_len,
2987                                                            DMA_FROM_DEVICE);
2988                         } else {
2989                                 skb = priv->rx_skbuff[entry];
2990                                 if (unlikely(!skb)) {
2991                                         netdev_err(priv->dev,
2992                                                    "%s: Inconsistent Rx chain\n",
2993                                                    priv->dev->name);
2994                                         priv->dev->stats.rx_dropped++;
2995                                         break;
2996                                 }
2997                                 prefetch(skb->data - NET_IP_ALIGN);
2998                                 priv->rx_skbuff[entry] = NULL;
2999                                 priv->rx_zeroc_thresh++;
3000
3001                                 skb_put(skb, frame_len);
3002                                 dma_unmap_single(priv->device,
3003                                                  priv->rx_skbuff_dma[entry],
3004                                                  priv->dma_buf_sz,
3005                                                  DMA_FROM_DEVICE);
3006                         }
3007
3008                         if (netif_msg_pktdata(priv)) {
3009                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3010                                            frame_len);
3011                                 print_pkt(skb->data, frame_len);
3012                         }
3013
3014                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3015
3016                         stmmac_rx_vlan(priv->dev, skb);
3017
3018                         skb->protocol = eth_type_trans(skb, priv->dev);
3019
3020                         if (unlikely(!coe))
3021                                 skb_checksum_none_assert(skb);
3022                         else
3023                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3024
3025                         napi_gro_receive(&priv->napi, skb);
3026
3027                         priv->dev->stats.rx_packets++;
3028                         priv->dev->stats.rx_bytes += frame_len;
3029                 }
3030                 entry = next_entry;
3031         }
3032
3033         stmmac_rx_refill(priv);
3034
3035         priv->xstats.rx_pkt_n += count;
3036
3037         return count;
3038 }
3039
3040 /**
3041  *  stmmac_poll - stmmac poll method (NAPI)
3042  *  @napi : pointer to the napi structure.
3043  *  @budget : maximum number of packets that the current CPU can receive from
3044  *            all interfaces.
3045  *  Description :
3046  *  To look at the incoming frames and clear the tx resources.
3047  */
3048 static int stmmac_poll(struct napi_struct *napi, int budget)
3049 {
3050         struct stmmac_priv *priv = container_of(napi, struct stmmac_priv, napi);
3051         int work_done = 0;
3052         u32 chan = STMMAC_CHAN0;
3053
3054         priv->xstats.napi_poll++;
3055         stmmac_tx_clean(priv);
3056
3057         work_done = stmmac_rx(priv, budget);
3058         if (work_done < budget) {
3059                 napi_complete_done(napi, work_done);
3060                 stmmac_enable_dma_irq(priv, chan);
3061         }
3062         return work_done;
3063 }
3064
3065 /**
3066  *  stmmac_tx_timeout
3067  *  @dev : Pointer to net device structure
3068  *  Description: this function is called when a packet transmission fails to
3069  *   complete within a reasonable time. The driver will mark the error in the
3070  *   netdev structure and arrange for the device to be reset to a sane state
3071  *   in order to transmit a new packet.
3072  */
3073 static void stmmac_tx_timeout(struct net_device *dev)
3074 {
3075         struct stmmac_priv *priv = netdev_priv(dev);
3076         u32 chan = STMMAC_CHAN0;
3077
3078         /* Clear Tx resources and restart transmitting again */
3079         stmmac_tx_err(priv, chan);
3080 }
3081
3082 /**
3083  *  stmmac_set_rx_mode - entry point for multicast addressing
3084  *  @dev : pointer to the device structure
3085  *  Description:
3086  *  This function is a driver entry point which gets called by the kernel
3087  *  whenever multicast addresses must be enabled/disabled.
3088  *  Return value:
3089  *  void.
3090  */
3091 static void stmmac_set_rx_mode(struct net_device *dev)
3092 {
3093         struct stmmac_priv *priv = netdev_priv(dev);
3094
3095         priv->hw->mac->set_filter(priv->hw, dev);
3096 }
3097
3098 /**
3099  *  stmmac_change_mtu - entry point to change MTU size for the device.
3100  *  @dev : device pointer.
3101  *  @new_mtu : the new MTU size for the device.
3102  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3103  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3104  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3105  *  Return value:
3106  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3107  *  file on failure.
3108  */
3109 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3110 {
3111         struct stmmac_priv *priv = netdev_priv(dev);
3112
3113         if (netif_running(dev)) {
3114                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3115                 return -EBUSY;
3116         }
3117
3118         dev->mtu = new_mtu;
3119
3120         netdev_update_features(dev);
3121
3122         return 0;
3123 }
3124
3125 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3126                                              netdev_features_t features)
3127 {
3128         struct stmmac_priv *priv = netdev_priv(dev);
3129
3130         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3131                 features &= ~NETIF_F_RXCSUM;
3132
3133         if (!priv->plat->tx_coe)
3134                 features &= ~NETIF_F_CSUM_MASK;
3135
3136         /* Some GMAC devices have a bugged Jumbo frame support that
3137          * needs to have the Tx COE disabled for oversized frames
3138          * (due to limited buffer sizes). In this case we disable
3139          * the TX csum insertion in the TDES and not use SF.
3140          */
3141         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3142                 features &= ~NETIF_F_CSUM_MASK;
3143
3144         /* Disable tso if asked by ethtool */
3145         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3146                 if (features & NETIF_F_TSO)
3147                         priv->tso = true;
3148                 else
3149                         priv->tso = false;
3150         }
3151
3152         return features;
3153 }
3154
3155 static int stmmac_set_features(struct net_device *netdev,
3156                                netdev_features_t features)
3157 {
3158         struct stmmac_priv *priv = netdev_priv(netdev);
3159
3160         /* Keep the COE Type in case of csum is supporting */
3161         if (features & NETIF_F_RXCSUM)
3162                 priv->hw->rx_csum = priv->plat->rx_coe;
3163         else
3164                 priv->hw->rx_csum = 0;
3165         /* No check needed because rx_coe has been set before and it will be
3166          * fixed in case of issue.
3167          */
3168         priv->hw->mac->rx_ipc(priv->hw);
3169
3170         return 0;
3171 }
3172
3173 /**
3174  *  stmmac_interrupt - main ISR
3175  *  @irq: interrupt number.
3176  *  @dev_id: to pass the net device pointer.
3177  *  Description: this is the main driver interrupt service routine.
3178  *  It can call:
3179  *  o DMA service routine (to manage incoming frame reception and transmission
3180  *    status)
3181  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3182  *    interrupts.
3183  */
3184 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3185 {
3186         struct net_device *dev = (struct net_device *)dev_id;
3187         struct stmmac_priv *priv = netdev_priv(dev);
3188         u32 rx_cnt = priv->plat->rx_queues_to_use;
3189         u32 tx_cnt = priv->plat->tx_queues_to_use;
3190         u32 queues_count;
3191         u32 queue;
3192
3193         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3194
3195         if (priv->irq_wake)
3196                 pm_wakeup_event(priv->device, 0);
3197
3198         if (unlikely(!dev)) {
3199                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3200                 return IRQ_NONE;
3201         }
3202
3203         /* To handle GMAC own interrupts */
3204         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3205                 int status = priv->hw->mac->host_irq_status(priv->hw,
3206                                                             &priv->xstats);
3207
3208                 if (unlikely(status)) {
3209                         /* For LPI we need to save the tx status */
3210                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3211                                 priv->tx_path_in_lpi_mode = true;
3212                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3213                                 priv->tx_path_in_lpi_mode = false;
3214                 }
3215
3216                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3217                         for (queue = 0; queue < queues_count; queue++) {
3218                                 status |=
3219                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3220                                                                    queue);
3221
3222                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3223                                     priv->hw->dma->set_rx_tail_ptr)
3224                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3225                                                                 priv->rx_tail_addr,
3226                                                                 queue);
3227                         }
3228                 }
3229
3230                 /* PCS link status */
3231                 if (priv->hw->pcs) {
3232                         if (priv->xstats.pcs_link)
3233                                 netif_carrier_on(dev);
3234                         else
3235                                 netif_carrier_off(dev);
3236                 }
3237         }
3238
3239         /* To handle DMA interrupts */
3240         stmmac_dma_interrupt(priv);
3241
3242         return IRQ_HANDLED;
3243 }
3244
3245 #ifdef CONFIG_NET_POLL_CONTROLLER
3246 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3247  * to allow network I/O with interrupts disabled.
3248  */
3249 static void stmmac_poll_controller(struct net_device *dev)
3250 {
3251         disable_irq(dev->irq);
3252         stmmac_interrupt(dev->irq, dev);
3253         enable_irq(dev->irq);
3254 }
3255 #endif
3256
3257 /**
3258  *  stmmac_ioctl - Entry point for the Ioctl
3259  *  @dev: Device pointer.
3260  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3261  *  a proprietary structure used to pass information to the driver.
3262  *  @cmd: IOCTL command
3263  *  Description:
3264  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3265  */
3266 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3267 {
3268         int ret = -EOPNOTSUPP;
3269
3270         if (!netif_running(dev))
3271                 return -EINVAL;
3272
3273         switch (cmd) {
3274         case SIOCGMIIPHY:
3275         case SIOCGMIIREG:
3276         case SIOCSMIIREG:
3277                 if (!dev->phydev)
3278                         return -EINVAL;
3279                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3280                 break;
3281         case SIOCSHWTSTAMP:
3282                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3283                 break;
3284         default:
3285                 break;
3286         }
3287
3288         return ret;
3289 }
3290
3291 #ifdef CONFIG_DEBUG_FS
3292 static struct dentry *stmmac_fs_dir;
3293
3294 static void sysfs_display_ring(void *head, int size, int extend_desc,
3295                                struct seq_file *seq)
3296 {
3297         int i;
3298         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3299         struct dma_desc *p = (struct dma_desc *)head;
3300
3301         for (i = 0; i < size; i++) {
3302                 if (extend_desc) {
3303                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3304                                    i, (unsigned int)virt_to_phys(ep),
3305                                    le32_to_cpu(ep->basic.des0),
3306                                    le32_to_cpu(ep->basic.des1),
3307                                    le32_to_cpu(ep->basic.des2),
3308                                    le32_to_cpu(ep->basic.des3));
3309                         ep++;
3310                 } else {
3311                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3312                                    i, (unsigned int)virt_to_phys(ep),
3313                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3314                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3315                         p++;
3316                 }
3317                 seq_printf(seq, "\n");
3318         }
3319 }
3320
3321 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3322 {
3323         struct net_device *dev = seq->private;
3324         struct stmmac_priv *priv = netdev_priv(dev);
3325
3326         if (priv->extend_desc) {
3327                 seq_printf(seq, "Extended RX descriptor ring:\n");
3328                 sysfs_display_ring((void *)priv->dma_erx, DMA_RX_SIZE, 1, seq);
3329                 seq_printf(seq, "Extended TX descriptor ring:\n");
3330                 sysfs_display_ring((void *)priv->dma_etx, DMA_TX_SIZE, 1, seq);
3331         } else {
3332                 seq_printf(seq, "RX descriptor ring:\n");
3333                 sysfs_display_ring((void *)priv->dma_rx, DMA_RX_SIZE, 0, seq);
3334                 seq_printf(seq, "TX descriptor ring:\n");
3335                 sysfs_display_ring((void *)priv->dma_tx, DMA_TX_SIZE, 0, seq);
3336         }
3337
3338         return 0;
3339 }
3340
3341 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3342 {
3343         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3344 }
3345
3346 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3347
3348 static const struct file_operations stmmac_rings_status_fops = {
3349         .owner = THIS_MODULE,
3350         .open = stmmac_sysfs_ring_open,
3351         .read = seq_read,
3352         .llseek = seq_lseek,
3353         .release = single_release,
3354 };
3355
3356 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3357 {
3358         struct net_device *dev = seq->private;
3359         struct stmmac_priv *priv = netdev_priv(dev);
3360
3361         if (!priv->hw_cap_support) {
3362                 seq_printf(seq, "DMA HW features not supported\n");
3363                 return 0;
3364         }
3365
3366         seq_printf(seq, "==============================\n");
3367         seq_printf(seq, "\tDMA HW features\n");
3368         seq_printf(seq, "==============================\n");
3369
3370         seq_printf(seq, "\t10/100 Mbps: %s\n",
3371                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3372         seq_printf(seq, "\t1000 Mbps: %s\n",
3373                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3374         seq_printf(seq, "\tHalf duplex: %s\n",
3375                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3376         seq_printf(seq, "\tHash Filter: %s\n",
3377                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3378         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3379                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3380         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3381                    (priv->dma_cap.pcs) ? "Y" : "N");
3382         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3383                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3384         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3385                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3386         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3387                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3388         seq_printf(seq, "\tRMON module: %s\n",
3389                    (priv->dma_cap.rmon) ? "Y" : "N");
3390         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3391                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3392         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3393                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3394         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3395                    (priv->dma_cap.eee) ? "Y" : "N");
3396         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3397         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3398                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3399         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3400                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3401                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3402         } else {
3403                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3404                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3405                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3406                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3407         }
3408         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3409                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3410         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3411                    priv->dma_cap.number_rx_channel);
3412         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3413                    priv->dma_cap.number_tx_channel);
3414         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3415                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3416
3417         return 0;
3418 }
3419
3420 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3421 {
3422         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3423 }
3424
3425 static const struct file_operations stmmac_dma_cap_fops = {
3426         .owner = THIS_MODULE,
3427         .open = stmmac_sysfs_dma_cap_open,
3428         .read = seq_read,
3429         .llseek = seq_lseek,
3430         .release = single_release,
3431 };
3432
3433 static int stmmac_init_fs(struct net_device *dev)
3434 {
3435         struct stmmac_priv *priv = netdev_priv(dev);
3436
3437         /* Create per netdev entries */
3438         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3439
3440         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3441                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3442
3443                 return -ENOMEM;
3444         }
3445
3446         /* Entry to report DMA RX/TX rings */
3447         priv->dbgfs_rings_status =
3448                 debugfs_create_file("descriptors_status", S_IRUGO,
3449                                     priv->dbgfs_dir, dev,
3450                                     &stmmac_rings_status_fops);
3451
3452         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3453                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3454                 debugfs_remove_recursive(priv->dbgfs_dir);
3455
3456                 return -ENOMEM;
3457         }
3458
3459         /* Entry to report the DMA HW features */
3460         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3461                                             priv->dbgfs_dir,
3462                                             dev, &stmmac_dma_cap_fops);
3463
3464         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3465                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3466                 debugfs_remove_recursive(priv->dbgfs_dir);
3467
3468                 return -ENOMEM;
3469         }
3470
3471         return 0;
3472 }
3473
3474 static void stmmac_exit_fs(struct net_device *dev)
3475 {
3476         struct stmmac_priv *priv = netdev_priv(dev);
3477
3478         debugfs_remove_recursive(priv->dbgfs_dir);
3479 }
3480 #endif /* CONFIG_DEBUG_FS */
3481
3482 static const struct net_device_ops stmmac_netdev_ops = {
3483         .ndo_open = stmmac_open,
3484         .ndo_start_xmit = stmmac_xmit,
3485         .ndo_stop = stmmac_release,
3486         .ndo_change_mtu = stmmac_change_mtu,
3487         .ndo_fix_features = stmmac_fix_features,
3488         .ndo_set_features = stmmac_set_features,
3489         .ndo_set_rx_mode = stmmac_set_rx_mode,
3490         .ndo_tx_timeout = stmmac_tx_timeout,
3491         .ndo_do_ioctl = stmmac_ioctl,
3492 #ifdef CONFIG_NET_POLL_CONTROLLER
3493         .ndo_poll_controller = stmmac_poll_controller,
3494 #endif
3495         .ndo_set_mac_address = eth_mac_addr,
3496 };
3497
3498 /**
3499  *  stmmac_hw_init - Init the MAC device
3500  *  @priv: driver private structure
3501  *  Description: this function is to configure the MAC device according to
3502  *  some platform parameters or the HW capability register. It prepares the
3503  *  driver to use either ring or chain modes and to setup either enhanced or
3504  *  normal descriptors.
3505  */
3506 static int stmmac_hw_init(struct stmmac_priv *priv)
3507 {
3508         struct mac_device_info *mac;
3509
3510         /* Identify the MAC HW device */
3511         if (priv->plat->has_gmac) {
3512                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3513                 mac = dwmac1000_setup(priv->ioaddr,
3514                                       priv->plat->multicast_filter_bins,
3515                                       priv->plat->unicast_filter_entries,
3516                                       &priv->synopsys_id);
3517         } else if (priv->plat->has_gmac4) {
3518                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
3519                 mac = dwmac4_setup(priv->ioaddr,
3520                                    priv->plat->multicast_filter_bins,
3521                                    priv->plat->unicast_filter_entries,
3522                                    &priv->synopsys_id);
3523         } else {
3524                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
3525         }
3526         if (!mac)
3527                 return -ENOMEM;
3528
3529         priv->hw = mac;
3530
3531         /* To use the chained or ring mode */
3532         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3533                 priv->hw->mode = &dwmac4_ring_mode_ops;
3534         } else {
3535                 if (chain_mode) {
3536                         priv->hw->mode = &chain_mode_ops;
3537                         dev_info(priv->device, "Chain mode enabled\n");
3538                         priv->mode = STMMAC_CHAIN_MODE;
3539                 } else {
3540                         priv->hw->mode = &ring_mode_ops;
3541                         dev_info(priv->device, "Ring mode enabled\n");
3542                         priv->mode = STMMAC_RING_MODE;
3543                 }
3544         }
3545
3546         /* Get the HW capability (new GMAC newer than 3.50a) */
3547         priv->hw_cap_support = stmmac_get_hw_features(priv);
3548         if (priv->hw_cap_support) {
3549                 dev_info(priv->device, "DMA HW capability register supported\n");
3550
3551                 /* We can override some gmac/dma configuration fields: e.g.
3552                  * enh_desc, tx_coe (e.g. that are passed through the
3553                  * platform) with the values from the HW capability
3554                  * register (if supported).
3555                  */
3556                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
3557                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
3558                 priv->hw->pmt = priv->plat->pmt;
3559
3560                 /* TXCOE doesn't work in thresh DMA mode */
3561                 if (priv->plat->force_thresh_dma_mode)
3562                         priv->plat->tx_coe = 0;
3563                 else
3564                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
3565
3566                 /* In case of GMAC4 rx_coe is from HW cap register. */
3567                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
3568
3569                 if (priv->dma_cap.rx_coe_type2)
3570                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
3571                 else if (priv->dma_cap.rx_coe_type1)
3572                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
3573
3574         } else {
3575                 dev_info(priv->device, "No HW DMA feature register supported\n");
3576         }
3577
3578         /* To use alternate (extended), normal or GMAC4 descriptor structures */
3579         if (priv->synopsys_id >= DWMAC_CORE_4_00)
3580                 priv->hw->desc = &dwmac4_desc_ops;
3581         else
3582                 stmmac_selec_desc_mode(priv);
3583
3584         if (priv->plat->rx_coe) {
3585                 priv->hw->rx_csum = priv->plat->rx_coe;
3586                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
3587                 if (priv->synopsys_id < DWMAC_CORE_4_00)
3588                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
3589         }
3590         if (priv->plat->tx_coe)
3591                 dev_info(priv->device, "TX Checksum insertion supported\n");
3592
3593         if (priv->plat->pmt) {
3594                 dev_info(priv->device, "Wake-Up On Lan supported\n");
3595                 device_set_wakeup_capable(priv->device, 1);
3596         }
3597
3598         if (priv->dma_cap.tsoen)
3599                 dev_info(priv->device, "TSO supported\n");
3600
3601         return 0;
3602 }
3603
3604 /**
3605  * stmmac_dvr_probe
3606  * @device: device pointer
3607  * @plat_dat: platform data pointer
3608  * @res: stmmac resource pointer
3609  * Description: this is the main probe function used to
3610  * call the alloc_etherdev, allocate the priv structure.
3611  * Return:
3612  * returns 0 on success, otherwise errno.
3613  */
3614 int stmmac_dvr_probe(struct device *device,
3615                      struct plat_stmmacenet_data *plat_dat,
3616                      struct stmmac_resources *res)
3617 {
3618         int ret = 0;
3619         struct net_device *ndev = NULL;
3620         struct stmmac_priv *priv;
3621
3622         ndev = alloc_etherdev(sizeof(struct stmmac_priv));
3623         if (!ndev)
3624                 return -ENOMEM;
3625
3626         SET_NETDEV_DEV(ndev, device);
3627
3628         priv = netdev_priv(ndev);
3629         priv->device = device;
3630         priv->dev = ndev;
3631
3632         stmmac_set_ethtool_ops(ndev);
3633         priv->pause = pause;
3634         priv->plat = plat_dat;
3635         priv->ioaddr = res->addr;
3636         priv->dev->base_addr = (unsigned long)res->addr;
3637
3638         priv->dev->irq = res->irq;
3639         priv->wol_irq = res->wol_irq;
3640         priv->lpi_irq = res->lpi_irq;
3641
3642         if (res->mac)
3643                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
3644
3645         dev_set_drvdata(device, priv->dev);
3646
3647         /* Verify driver arguments */
3648         stmmac_verify_args();
3649
3650         /* Override with kernel parameters if supplied XXX CRS XXX
3651          * this needs to have multiple instances
3652          */
3653         if ((phyaddr >= 0) && (phyaddr <= 31))
3654                 priv->plat->phy_addr = phyaddr;
3655
3656         if (priv->plat->stmmac_rst)
3657                 reset_control_deassert(priv->plat->stmmac_rst);
3658
3659         /* Init MAC and get the capabilities */
3660         ret = stmmac_hw_init(priv);
3661         if (ret)
3662                 goto error_hw_init;
3663
3664         ndev->netdev_ops = &stmmac_netdev_ops;
3665
3666         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3667                             NETIF_F_RXCSUM;
3668
3669         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3670                 ndev->hw_features |= NETIF_F_TSO;
3671                 priv->tso = true;
3672                 dev_info(priv->device, "TSO feature enabled\n");
3673         }
3674         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
3675         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
3676 #ifdef STMMAC_VLAN_TAG_USED
3677         /* Both mac100 and gmac support receive VLAN tag detection */
3678         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
3679 #endif
3680         priv->msg_enable = netif_msg_init(debug, default_msg_level);
3681
3682         /* MTU range: 46 - hw-specific max */
3683         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
3684         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
3685                 ndev->max_mtu = JUMBO_LEN;
3686         else
3687                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
3688         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
3689          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
3690          */
3691         if ((priv->plat->maxmtu < ndev->max_mtu) &&
3692             (priv->plat->maxmtu >= ndev->min_mtu))
3693                 ndev->max_mtu = priv->plat->maxmtu;
3694         else if (priv->plat->maxmtu < ndev->min_mtu)
3695                 dev_warn(priv->device,
3696                          "%s: warning: maxmtu having invalid value (%d)\n",
3697                          __func__, priv->plat->maxmtu);
3698
3699         if (flow_ctrl)
3700                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
3701
3702         /* Rx Watchdog is available in the COREs newer than the 3.40.
3703          * In some case, for example on bugged HW this feature
3704          * has to be disable and this can be done by passing the
3705          * riwt_off field from the platform.
3706          */
3707         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
3708                 priv->use_riwt = 1;
3709                 dev_info(priv->device,
3710                          "Enable RX Mitigation via HW Watchdog Timer\n");
3711         }
3712
3713         netif_napi_add(ndev, &priv->napi, stmmac_poll, 64);
3714
3715         spin_lock_init(&priv->lock);
3716
3717         /* If a specific clk_csr value is passed from the platform
3718          * this means that the CSR Clock Range selection cannot be
3719          * changed at run-time and it is fixed. Viceversa the driver'll try to
3720          * set the MDC clock dynamically according to the csr actual
3721          * clock input.
3722          */
3723         if (!priv->plat->clk_csr)
3724                 stmmac_clk_csr_set(priv);
3725         else
3726                 priv->clk_csr = priv->plat->clk_csr;
3727
3728         stmmac_check_pcs_mode(priv);
3729
3730         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
3731             priv->hw->pcs != STMMAC_PCS_TBI &&
3732             priv->hw->pcs != STMMAC_PCS_RTBI) {
3733                 /* MDIO bus Registration */
3734                 ret = stmmac_mdio_register(ndev);
3735                 if (ret < 0) {
3736                         dev_err(priv->device,
3737                                 "%s: MDIO bus (id: %d) registration failed",
3738                                 __func__, priv->plat->bus_id);
3739                         goto error_mdio_register;
3740                 }
3741         }
3742
3743         ret = register_netdev(ndev);
3744         if (ret) {
3745                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
3746                         __func__, ret);
3747                 goto error_netdev_register;
3748         }
3749
3750         return ret;
3751
3752 error_netdev_register:
3753         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
3754             priv->hw->pcs != STMMAC_PCS_TBI &&
3755             priv->hw->pcs != STMMAC_PCS_RTBI)
3756                 stmmac_mdio_unregister(ndev);
3757 error_mdio_register:
3758         netif_napi_del(&priv->napi);
3759 error_hw_init:
3760         free_netdev(ndev);
3761
3762         return ret;
3763 }
3764 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
3765
3766 /**
3767  * stmmac_dvr_remove
3768  * @dev: device pointer
3769  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
3770  * changes the link status, releases the DMA descriptor rings.
3771  */
3772 int stmmac_dvr_remove(struct device *dev)
3773 {
3774         struct net_device *ndev = dev_get_drvdata(dev);
3775         struct stmmac_priv *priv = netdev_priv(ndev);
3776
3777         netdev_info(priv->dev, "%s: removing driver", __func__);
3778
3779         stmmac_stop_all_dma(priv);
3780
3781         priv->hw->mac->set_mac(priv->ioaddr, false);
3782         netif_carrier_off(ndev);
3783         unregister_netdev(ndev);
3784         if (priv->plat->stmmac_rst)
3785                 reset_control_assert(priv->plat->stmmac_rst);
3786         clk_disable_unprepare(priv->plat->pclk);
3787         clk_disable_unprepare(priv->plat->stmmac_clk);
3788         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
3789             priv->hw->pcs != STMMAC_PCS_TBI &&
3790             priv->hw->pcs != STMMAC_PCS_RTBI)
3791                 stmmac_mdio_unregister(ndev);
3792         free_netdev(ndev);
3793
3794         return 0;
3795 }
3796 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
3797
3798 /**
3799  * stmmac_suspend - suspend callback
3800  * @dev: device pointer
3801  * Description: this is the function to suspend the device and it is called
3802  * by the platform driver to stop the network queue, release the resources,
3803  * program the PMT register (for WoL), clean and release driver resources.
3804  */
3805 int stmmac_suspend(struct device *dev)
3806 {
3807         struct net_device *ndev = dev_get_drvdata(dev);
3808         struct stmmac_priv *priv = netdev_priv(ndev);
3809         unsigned long flags;
3810
3811         if (!ndev || !netif_running(ndev))
3812                 return 0;
3813
3814         if (ndev->phydev)
3815                 phy_stop(ndev->phydev);
3816
3817         spin_lock_irqsave(&priv->lock, flags);
3818
3819         netif_device_detach(ndev);
3820         netif_stop_queue(ndev);
3821
3822         napi_disable(&priv->napi);
3823
3824         /* Stop TX/RX DMA */
3825         stmmac_stop_all_dma(priv);
3826
3827         /* Enable Power down mode by programming the PMT regs */
3828         if (device_may_wakeup(priv->device)) {
3829                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
3830                 priv->irq_wake = 1;
3831         } else {
3832                 priv->hw->mac->set_mac(priv->ioaddr, false);
3833                 pinctrl_pm_select_sleep_state(priv->device);
3834                 /* Disable clock in case of PWM is off */
3835                 clk_disable(priv->plat->pclk);
3836                 clk_disable(priv->plat->stmmac_clk);
3837         }
3838         spin_unlock_irqrestore(&priv->lock, flags);
3839
3840         priv->oldlink = 0;
3841         priv->speed = SPEED_UNKNOWN;
3842         priv->oldduplex = DUPLEX_UNKNOWN;
3843         return 0;
3844 }
3845 EXPORT_SYMBOL_GPL(stmmac_suspend);
3846
3847 /**
3848  * stmmac_resume - resume callback
3849  * @dev: device pointer
3850  * Description: when resume this function is invoked to setup the DMA and CORE
3851  * in a usable state.
3852  */
3853 int stmmac_resume(struct device *dev)
3854 {
3855         struct net_device *ndev = dev_get_drvdata(dev);
3856         struct stmmac_priv *priv = netdev_priv(ndev);
3857         unsigned long flags;
3858
3859         if (!netif_running(ndev))
3860                 return 0;
3861
3862         /* Power Down bit, into the PM register, is cleared
3863          * automatically as soon as a magic packet or a Wake-up frame
3864          * is received. Anyway, it's better to manually clear
3865          * this bit because it can generate problems while resuming
3866          * from another devices (e.g. serial console).
3867          */
3868         if (device_may_wakeup(priv->device)) {
3869                 spin_lock_irqsave(&priv->lock, flags);
3870                 priv->hw->mac->pmt(priv->hw, 0);
3871                 spin_unlock_irqrestore(&priv->lock, flags);
3872                 priv->irq_wake = 0;
3873         } else {
3874                 pinctrl_pm_select_default_state(priv->device);
3875                 /* enable the clk previously disabled */
3876                 clk_enable(priv->plat->stmmac_clk);
3877                 clk_enable(priv->plat->pclk);
3878                 /* reset the phy so that it's ready */
3879                 if (priv->mii)
3880                         stmmac_mdio_reset(priv->mii);
3881         }
3882
3883         netif_device_attach(ndev);
3884
3885         spin_lock_irqsave(&priv->lock, flags);
3886
3887         priv->cur_rx = 0;
3888         priv->dirty_rx = 0;
3889         priv->dirty_tx = 0;
3890         priv->cur_tx = 0;
3891         /* reset private mss value to force mss context settings at
3892          * next tso xmit (only used for gmac4).
3893          */
3894         priv->mss = 0;
3895
3896         stmmac_clear_descriptors(priv);
3897
3898         stmmac_hw_setup(ndev, false);
3899         stmmac_init_tx_coalesce(priv);
3900         stmmac_set_rx_mode(ndev);
3901
3902         napi_enable(&priv->napi);
3903
3904         netif_start_queue(ndev);
3905
3906         spin_unlock_irqrestore(&priv->lock, flags);
3907
3908         if (ndev->phydev)
3909                 phy_start(ndev->phydev);
3910
3911         return 0;
3912 }
3913 EXPORT_SYMBOL_GPL(stmmac_resume);
3914
3915 #ifndef MODULE
3916 static int __init stmmac_cmdline_opt(char *str)
3917 {
3918         char *opt;
3919
3920         if (!str || !*str)
3921                 return -EINVAL;
3922         while ((opt = strsep(&str, ",")) != NULL) {
3923                 if (!strncmp(opt, "debug:", 6)) {
3924                         if (kstrtoint(opt + 6, 0, &debug))
3925                                 goto err;
3926                 } else if (!strncmp(opt, "phyaddr:", 8)) {
3927                         if (kstrtoint(opt + 8, 0, &phyaddr))
3928                                 goto err;
3929                 } else if (!strncmp(opt, "buf_sz:", 7)) {
3930                         if (kstrtoint(opt + 7, 0, &buf_sz))
3931                                 goto err;
3932                 } else if (!strncmp(opt, "tc:", 3)) {
3933                         if (kstrtoint(opt + 3, 0, &tc))
3934                                 goto err;
3935                 } else if (!strncmp(opt, "watchdog:", 9)) {
3936                         if (kstrtoint(opt + 9, 0, &watchdog))
3937                                 goto err;
3938                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
3939                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
3940                                 goto err;
3941                 } else if (!strncmp(opt, "pause:", 6)) {
3942                         if (kstrtoint(opt + 6, 0, &pause))
3943                                 goto err;
3944                 } else if (!strncmp(opt, "eee_timer:", 10)) {
3945                         if (kstrtoint(opt + 10, 0, &eee_timer))
3946                                 goto err;
3947                 } else if (!strncmp(opt, "chain_mode:", 11)) {
3948                         if (kstrtoint(opt + 11, 0, &chain_mode))
3949                                 goto err;
3950                 }
3951         }
3952         return 0;
3953
3954 err:
3955         pr_err("%s: ERROR broken module parameter conversion", __func__);
3956         return -EINVAL;
3957 }
3958
3959 __setup("stmmaceth=", stmmac_cmdline_opt);
3960 #endif /* MODULE */
3961
3962 static int __init stmmac_init(void)
3963 {
3964 #ifdef CONFIG_DEBUG_FS
3965         /* Create debugfs main directory if it doesn't exist yet */
3966         if (!stmmac_fs_dir) {
3967                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
3968
3969                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
3970                         pr_err("ERROR %s, debugfs create directory failed\n",
3971                                STMMAC_RESOURCE_NAME);
3972
3973                         return -ENOMEM;
3974                 }
3975         }
3976 #endif
3977
3978         return 0;
3979 }
3980
3981 static void __exit stmmac_exit(void)
3982 {
3983 #ifdef CONFIG_DEBUG_FS
3984         debugfs_remove_recursive(stmmac_fs_dir);
3985 #endif
3986 }
3987
3988 module_init(stmmac_init)
3989 module_exit(stmmac_exit)
3990
3991 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
3992 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
3993 MODULE_LICENSE("GPL");