drivers/net/ethernet/ti/cpsw.c

   1 /*
   2  * Texas Instruments Ethernet Switch Driver
   3  *
   4  * Copyright (C) 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/io.h>
  18 #include <linux/clk.h>
  19 #include <linux/timer.h>
  20 #include <linux/module.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/irqreturn.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/if_ether.h>
  25 #include <linux/etherdevice.h>
  26 #include <linux/netdevice.h>
  27 #include <linux/net_tstamp.h>
  28 #include <linux/phy.h>
  29 #include <linux/phy/phy.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/delay.h>
  32 #include <linux/pm_runtime.h>
  33 #include <linux/gpio/consumer.h>
  34 #include <linux/of.h>
  35 #include <linux/of_mdio.h>
  36 #include <linux/of_net.h>
  37 #include <linux/of_device.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/kmemleak.h>
  40 #include <linux/sys_soc.h>
  41
  42 #include <linux/pinctrl/consumer.h>
  43 #include <net/pkt_cls.h>
  44
  45 #include "cpsw.h"
  46 #include "cpsw_ale.h"
  47 #include "cpts.h"
  48 #include "davinci_cpdma.h"
  49
  50 #include <net/pkt_sched.h>
  51
  52 #define CPSW_DEBUG      (NETIF_MSG_HW           | NETIF_MSG_WOL         | \
  53                          NETIF_MSG_DRV          | NETIF_MSG_LINK        | \
  54                          NETIF_MSG_IFUP         | NETIF_MSG_INTR        | \
  55                          NETIF_MSG_PROBE        | NETIF_MSG_TIMER       | \
  56                          NETIF_MSG_IFDOWN       | NETIF_MSG_RX_ERR      | \
  57                          NETIF_MSG_TX_ERR       | NETIF_MSG_TX_DONE     | \
  58                          NETIF_MSG_PKTDATA      | NETIF_MSG_TX_QUEUED   | \
  59                          NETIF_MSG_RX_STATUS)
  60
  61 #define cpsw_info(priv, type, format, ...)              \
  62 do {                                                            \
  63         if (netif_msg_##type(priv) && net_ratelimit())          \
  64                 dev_info(priv->dev, format, ## __VA_ARGS__);    \
  65 } while (0)
  66
  67 #define cpsw_err(priv, type, format, ...)               \
  68 do {                                                            \
  69         if (netif_msg_##type(priv) && net_ratelimit())          \
  70                 dev_err(priv->dev, format, ## __VA_ARGS__);     \
  71 } while (0)
  72
  73 #define cpsw_dbg(priv, type, format, ...)               \
  74 do {                                                            \
  75         if (netif_msg_##type(priv) && net_ratelimit())          \
  76                 dev_dbg(priv->dev, format, ## __VA_ARGS__);     \
  77 } while (0)
  78
  79 #define cpsw_notice(priv, type, format, ...)            \
  80 do {                                                            \
  81         if (netif_msg_##type(priv) && net_ratelimit())          \
  82                 dev_notice(priv->dev, format, ## __VA_ARGS__);  \
  83 } while (0)
  84
  85 #define ALE_ALL_PORTS           0x7
  86
  87 #define CPSW_MAJOR_VERSION(reg)         (reg >> 8 & 0x7)
  88 #define CPSW_MINOR_VERSION(reg)         (reg & 0xff)
  89 #define CPSW_RTL_VERSION(reg)           ((reg >> 11) & 0x1f)
  90
  91 #define CPSW_VERSION_1          0x19010a
  92 #define CPSW_VERSION_2          0x19010c
  93 #define CPSW_VERSION_3          0x19010f
  94 #define CPSW_VERSION_4          0x190112
  95
  96 #define HOST_PORT_NUM           0
  97 #define CPSW_ALE_PORTS_NUM      3
  98 #define SLIVER_SIZE             0x40
  99
 100 #define CPSW1_HOST_PORT_OFFSET  0x028
 101 #define CPSW1_SLAVE_OFFSET      0x050
 102 #define CPSW1_SLAVE_SIZE        0x040
 103 #define CPSW1_CPDMA_OFFSET      0x100
 104 #define CPSW1_STATERAM_OFFSET   0x200
 105 #define CPSW1_HW_STATS          0x400
 106 #define CPSW1_CPTS_OFFSET       0x500
 107 #define CPSW1_ALE_OFFSET        0x600
 108 #define CPSW1_SLIVER_OFFSET     0x700
 109
 110 #define CPSW2_HOST_PORT_OFFSET  0x108
 111 #define CPSW2_SLAVE_OFFSET      0x200
 112 #define CPSW2_SLAVE_SIZE        0x100
 113 #define CPSW2_CPDMA_OFFSET      0x800
 114 #define CPSW2_HW_STATS          0x900
 115 #define CPSW2_STATERAM_OFFSET   0xa00
 116 #define CPSW2_CPTS_OFFSET       0xc00
 117 #define CPSW2_ALE_OFFSET        0xd00
 118 #define CPSW2_SLIVER_OFFSET     0xd80
 119 #define CPSW2_BD_OFFSET         0x2000
 120
 121 #define CPDMA_RXTHRESH          0x0c0
 122 #define CPDMA_RXFREE            0x0e0
 123 #define CPDMA_TXHDP             0x00
 124 #define CPDMA_RXHDP             0x20
 125 #define CPDMA_TXCP              0x40
 126 #define CPDMA_RXCP              0x60
 127
 128 #define CPSW_POLL_WEIGHT        64
 129 #define CPSW_RX_VLAN_ENCAP_HDR_SIZE             4
 130 #define CPSW_MIN_PACKET_SIZE    (VLAN_ETH_ZLEN)
 131 #define CPSW_MAX_PACKET_SIZE    (VLAN_ETH_FRAME_LEN +\
 132                                  ETH_FCS_LEN +\
 133                                  CPSW_RX_VLAN_ENCAP_HDR_SIZE)
 134
 135 #define RX_PRIORITY_MAPPING     0x76543210
 136 #define TX_PRIORITY_MAPPING     0x33221100
 137 #define CPDMA_TX_PRIORITY_MAP   0x76543210
 138
 139 #define CPSW_VLAN_AWARE         BIT(1)
 140 #define CPSW_RX_VLAN_ENCAP      BIT(2)
 141 #define CPSW_ALE_VLAN_AWARE     1
 142
 143 #define CPSW_FIFO_NORMAL_MODE           (0 << 16)
 144 #define CPSW_FIFO_DUAL_MAC_MODE         (1 << 16)
 145 #define CPSW_FIFO_RATE_LIMIT_MODE       (2 << 16)
 146
 147 #define CPSW_INTPACEEN          (0x3f << 16)
 148 #define CPSW_INTPRESCALE_MASK   (0x7FF << 0)
 149 #define CPSW_CMINTMAX_CNT       63
 150 #define CPSW_CMINTMIN_CNT       2
 151 #define CPSW_CMINTMAX_INTVL     (1000 / CPSW_CMINTMIN_CNT)
 152 #define CPSW_CMINTMIN_INTVL     ((1000 / CPSW_CMINTMAX_CNT) + 1)
 153
 154 #define cpsw_slave_index(cpsw, priv)                            \
 155                 ((cpsw->data.dual_emac) ? priv->emac_port :     \
 156                 cpsw->data.active_slave)
 157 #define IRQ_NUM                 2
 158 #define CPSW_MAX_QUEUES         8
 159 #define CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT 256
 160 #define CPSW_FIFO_QUEUE_TYPE_SHIFT      16
 161 #define CPSW_FIFO_SHAPE_EN_SHIFT        16
 162 #define CPSW_FIFO_RATE_EN_SHIFT         20
 163 #define CPSW_TC_NUM                     4
 164 #define CPSW_FIFO_SHAPERS_NUM           (CPSW_TC_NUM - 1)
 165 #define CPSW_PCT_MASK                   0x7f
 166
 167 #define CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT       29
 168 #define CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK         GENMASK(2, 0)
 169 #define CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT        16
 170 #define CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT   8
 171 #define CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK     GENMASK(1, 0)
 172 enum {
 173         CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG = 0,
 174         CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV,
 175         CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG,
 176         CPSW_RX_VLAN_ENCAP_HDR_PKT_UNTAG,
 177 };
 178
 179 static int debug_level;
 180 module_param(debug_level, int, 0);
 181 MODULE_PARM_DESC(debug_level, "cpsw debug level (NETIF_MSG bits)");
 182
 183 static int ale_ageout = 10;
 184 module_param(ale_ageout, int, 0);
 185 MODULE_PARM_DESC(ale_ageout, "cpsw ale ageout interval (seconds)");
 186
 187 static int rx_packet_max = CPSW_MAX_PACKET_SIZE;
 188 module_param(rx_packet_max, int, 0);
 189 MODULE_PARM_DESC(rx_packet_max, "maximum receive packet size (bytes)");
 190
 191 static int descs_pool_size = CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT;
 192 module_param(descs_pool_size, int, 0444);
 193 MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
 194
 195 struct cpsw_wr_regs {
 196         u32     id_ver;
 197         u32     soft_reset;
 198         u32     control;
 199         u32     int_control;
 200         u32     rx_thresh_en;
 201         u32     rx_en;
 202         u32     tx_en;
 203         u32     misc_en;
 204         u32     mem_allign1[8];
 205         u32     rx_thresh_stat;
 206         u32     rx_stat;
 207         u32     tx_stat;
 208         u32     misc_stat;
 209         u32     mem_allign2[8];
 210         u32     rx_imax;
 211         u32     tx_imax;
 212
 213 };
 214
 215 struct cpsw_ss_regs {
 216         u32     id_ver;
 217         u32     control;
 218         u32     soft_reset;
 219         u32     stat_port_en;
 220         u32     ptype;
 221         u32     soft_idle;
 222         u32     thru_rate;
 223         u32     gap_thresh;
 224         u32     tx_start_wds;
 225         u32     flow_control;
 226         u32     vlan_ltype;
 227         u32     ts_ltype;
 228         u32     dlr_ltype;
 229 };
 230
 231 /* CPSW_PORT_V1 */
 232 #define CPSW1_MAX_BLKS      0x00 /* Maximum FIFO Blocks */
 233 #define CPSW1_BLK_CNT       0x04 /* FIFO Block Usage Count (Read Only) */
 234 #define CPSW1_TX_IN_CTL     0x08 /* Transmit FIFO Control */
 235 #define CPSW1_PORT_VLAN     0x0c /* VLAN Register */
 236 #define CPSW1_TX_PRI_MAP    0x10 /* Tx Header Priority to Switch Pri Mapping */
 237 #define CPSW1_TS_CTL        0x14 /* Time Sync Control */
 238 #define CPSW1_TS_SEQ_LTYPE  0x18 /* Time Sync Sequence ID Offset and Msg Type */
 239 #define CPSW1_TS_VLAN       0x1c /* Time Sync VLAN1 and VLAN2 */
 240
 241 /* CPSW_PORT_V2 */
 242 #define CPSW2_CONTROL       0x00 /* Control Register */
 243 #define CPSW2_MAX_BLKS      0x08 /* Maximum FIFO Blocks */
 244 #define CPSW2_BLK_CNT       0x0c /* FIFO Block Usage Count (Read Only) */
 245 #define CPSW2_TX_IN_CTL     0x10 /* Transmit FIFO Control */
 246 #define CPSW2_PORT_VLAN     0x14 /* VLAN Register */
 247 #define CPSW2_TX_PRI_MAP    0x18 /* Tx Header Priority to Switch Pri Mapping */
 248 #define CPSW2_TS_SEQ_MTYPE  0x1c /* Time Sync Sequence ID Offset and Msg Type */
 249
 250 /* CPSW_PORT_V1 and V2 */
 251 #define SA_LO               0x20 /* CPGMAC_SL Source Address Low */
 252 #define SA_HI               0x24 /* CPGMAC_SL Source Address High */
 253 #define SEND_PERCENT        0x28 /* Transmit Queue Send Percentages */
 254
 255 /* CPSW_PORT_V2 only */
 256 #define RX_DSCP_PRI_MAP0    0x30 /* Rx DSCP Priority to Rx Packet Mapping */
 257 #define RX_DSCP_PRI_MAP1    0x34 /* Rx DSCP Priority to Rx Packet Mapping */
 258 #define RX_DSCP_PRI_MAP2    0x38 /* Rx DSCP Priority to Rx Packet Mapping */
 259 #define RX_DSCP_PRI_MAP3    0x3c /* Rx DSCP Priority to Rx Packet Mapping */
 260 #define RX_DSCP_PRI_MAP4    0x40 /* Rx DSCP Priority to Rx Packet Mapping */
 261 #define RX_DSCP_PRI_MAP5    0x44 /* Rx DSCP Priority to Rx Packet Mapping */
 262 #define RX_DSCP_PRI_MAP6    0x48 /* Rx DSCP Priority to Rx Packet Mapping */
 263 #define RX_DSCP_PRI_MAP7    0x4c /* Rx DSCP Priority to Rx Packet Mapping */
 264
 265 /* Bit definitions for the CPSW2_CONTROL register */
 266 #define PASS_PRI_TAGGED     BIT(24) /* Pass Priority Tagged */
 267 #define VLAN_LTYPE2_EN      BIT(21) /* VLAN LTYPE 2 enable */
 268 #define VLAN_LTYPE1_EN      BIT(20) /* VLAN LTYPE 1 enable */
 269 #define DSCP_PRI_EN         BIT(16) /* DSCP Priority Enable */
 270 #define TS_107              BIT(15) /* Tyme Sync Dest IP Address 107 */
 271 #define TS_320              BIT(14) /* Time Sync Dest Port 320 enable */
 272 #define TS_319              BIT(13) /* Time Sync Dest Port 319 enable */
 273 #define TS_132              BIT(12) /* Time Sync Dest IP Addr 132 enable */
 274 #define TS_131              BIT(11) /* Time Sync Dest IP Addr 131 enable */
 275 #define TS_130              BIT(10) /* Time Sync Dest IP Addr 130 enable */
 276 #define TS_129              BIT(9)  /* Time Sync Dest IP Addr 129 enable */
 277 #define TS_TTL_NONZERO      BIT(8)  /* Time Sync Time To Live Non-zero enable */
 278 #define TS_ANNEX_F_EN       BIT(6)  /* Time Sync Annex F enable */
 279 #define TS_ANNEX_D_EN       BIT(4)  /* Time Sync Annex D enable */
 280 #define TS_LTYPE2_EN        BIT(3)  /* Time Sync LTYPE 2 enable */
 281 #define TS_LTYPE1_EN        BIT(2)  /* Time Sync LTYPE 1 enable */
 282 #define TS_TX_EN            BIT(1)  /* Time Sync Transmit Enable */
 283 #define TS_RX_EN            BIT(0)  /* Time Sync Receive Enable */
 284
 285 #define CTRL_V2_TS_BITS \
 286         (TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
 287          TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN)
 288
 289 #define CTRL_V2_ALL_TS_MASK (CTRL_V2_TS_BITS | TS_TX_EN | TS_RX_EN)
 290 #define CTRL_V2_TX_TS_BITS  (CTRL_V2_TS_BITS | TS_TX_EN)
 291 #define CTRL_V2_RX_TS_BITS  (CTRL_V2_TS_BITS | TS_RX_EN)
 292
 293
 294 #define CTRL_V3_TS_BITS \
 295         (TS_107 | TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
 296          TS_TTL_NONZERO | TS_ANNEX_F_EN | TS_ANNEX_D_EN |\
 297          TS_LTYPE1_EN)
 298
 299 #define CTRL_V3_ALL_TS_MASK (CTRL_V3_TS_BITS | TS_TX_EN | TS_RX_EN)
 300 #define CTRL_V3_TX_TS_BITS  (CTRL_V3_TS_BITS | TS_TX_EN)
 301 #define CTRL_V3_RX_TS_BITS  (CTRL_V3_TS_BITS | TS_RX_EN)
 302
 303 /* Bit definitions for the CPSW2_TS_SEQ_MTYPE register */
 304 #define TS_SEQ_ID_OFFSET_SHIFT   (16)    /* Time Sync Sequence ID Offset */
 305 #define TS_SEQ_ID_OFFSET_MASK    (0x3f)
 306 #define TS_MSG_TYPE_EN_SHIFT     (0)     /* Time Sync Message Type Enable */
 307 #define TS_MSG_TYPE_EN_MASK      (0xffff)
 308
 309 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
 310 #define EVENT_MSG_BITS ((1<<0) | (1<<1) | (1<<2) | (1<<3))
 311
 312 /* Bit definitions for the CPSW1_TS_CTL register */
 313 #define CPSW_V1_TS_RX_EN                BIT(0)
 314 #define CPSW_V1_TS_TX_EN                BIT(4)
 315 #define CPSW_V1_MSG_TYPE_OFS            16
 316
 317 /* Bit definitions for the CPSW1_TS_SEQ_LTYPE register */
 318 #define CPSW_V1_SEQ_ID_OFS_SHIFT        16
 319
 320 #define CPSW_MAX_BLKS_TX                15
 321 #define CPSW_MAX_BLKS_TX_SHIFT          4
 322 #define CPSW_MAX_BLKS_RX                5
 323
 324 struct cpsw_host_regs {
 325         u32     max_blks;
 326         u32     blk_cnt;
 327         u32     tx_in_ctl;
 328         u32     port_vlan;
 329         u32     tx_pri_map;
 330         u32     cpdma_tx_pri_map;
 331         u32     cpdma_rx_chan_map;
 332 };
 333
 334 struct cpsw_sliver_regs {
 335         u32     id_ver;
 336         u32     mac_control;
 337         u32     mac_status;
 338         u32     soft_reset;
 339         u32     rx_maxlen;
 340         u32     __reserved_0;
 341         u32     rx_pause;
 342         u32     tx_pause;
 343         u32     __reserved_1;
 344         u32     rx_pri_map;
 345 };
 346
 347 struct cpsw_hw_stats {
 348         u32     rxgoodframes;
 349         u32     rxbroadcastframes;
 350         u32     rxmulticastframes;
 351         u32     rxpauseframes;
 352         u32     rxcrcerrors;
 353         u32     rxaligncodeerrors;
 354         u32     rxoversizedframes;
 355         u32     rxjabberframes;
 356         u32     rxundersizedframes;
 357         u32     rxfragments;
 358         u32     __pad_0[2];
 359         u32     rxoctets;
 360         u32     txgoodframes;
 361         u32     txbroadcastframes;
 362         u32     txmulticastframes;
 363         u32     txpauseframes;
 364         u32     txdeferredframes;
 365         u32     txcollisionframes;
 366         u32     txsinglecollframes;
 367         u32     txmultcollframes;
 368         u32     txexcessivecollisions;
 369         u32     txlatecollisions;
 370         u32     txunderrun;
 371         u32     txcarriersenseerrors;
 372         u32     txoctets;
 373         u32     octetframes64;
 374         u32     octetframes65t127;
 375         u32     octetframes128t255;
 376         u32     octetframes256t511;
 377         u32     octetframes512t1023;
 378         u32     octetframes1024tup;
 379         u32     netoctets;
 380         u32     rxsofoverruns;
 381         u32     rxmofoverruns;
 382         u32     rxdmaoverruns;
 383 };
 384
 385 struct cpsw_slave_data {
 386         struct device_node *phy_node;
 387         char            phy_id[MII_BUS_ID_SIZE];
 388         int             phy_if;
 389         u8              mac_addr[ETH_ALEN];
 390         u16             dual_emac_res_vlan;     /* Reserved VLAN for DualEMAC */
 391         struct phy      *ifphy;
 392 };
 393
 394 struct cpsw_platform_data {
 395         struct cpsw_slave_data  *slave_data;
 396         u32     ss_reg_ofs;     /* Subsystem control register offset */
 397         u32     channels;       /* number of cpdma channels (symmetric) */
 398         u32     slaves;         /* number of slave cpgmac ports */
 399         u32     active_slave; /* time stamping, ethtool and SIOCGMIIPHY slave */
 400         u32     ale_entries;    /* ale table size */
 401         u32     bd_ram_size;  /*buffer descriptor ram size */
 402         u32     mac_control;    /* Mac control register */
 403         u16     default_vlan;   /* Def VLAN for ALE lookup in VLAN aware mode*/
 404         bool    dual_emac;      /* Enable Dual EMAC mode */
 405 };
 406
 407 struct cpsw_slave {
 408         void __iomem                    *regs;
 409         struct cpsw_sliver_regs __iomem *sliver;
 410         int                             slave_num;
 411         u32                             mac_control;
 412         struct cpsw_slave_data          *data;
 413         struct phy_device               *phy;
 414         struct net_device               *ndev;
 415         u32                             port_vlan;
 416 };
 417
 418 static inline u32 slave_read(struct cpsw_slave *slave, u32 offset)
 419 {
 420         return readl_relaxed(slave->regs + offset);
 421 }
 422
 423 static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
 424 {
 425         writel_relaxed(val, slave->regs + offset);
 426 }
 427
 428 struct cpsw_vector {
 429         struct cpdma_chan *ch;
 430         int budget;
 431 };
 432
 433 struct cpsw_common {
 434         struct device                   *dev;
 435         struct cpsw_platform_data       data;
 436         struct napi_struct              napi_rx;
 437         struct napi_struct              napi_tx;
 438         struct cpsw_ss_regs __iomem     *regs;
 439         struct cpsw_wr_regs __iomem     *wr_regs;
 440         u8 __iomem                      *hw_stats;
 441         struct cpsw_host_regs __iomem   *host_port_regs;
 442         u32                             version;
 443         u32                             coal_intvl;
 444         u32                             bus_freq_mhz;
 445         int                             rx_packet_max;
 446         struct cpsw_slave               *slaves;
 447         struct cpdma_ctlr               *dma;
 448         struct cpsw_vector              txv[CPSW_MAX_QUEUES];
 449         struct cpsw_vector              rxv[CPSW_MAX_QUEUES];
 450         struct cpsw_ale                 *ale;
 451         bool                            quirk_irq;
 452         bool                            rx_irq_disabled;
 453         bool                            tx_irq_disabled;
 454         u32 irqs_table[IRQ_NUM];
 455         struct cpts                     *cpts;
 456         int                             rx_ch_num, tx_ch_num;
 457         int                             speed;
 458         int                             usage_count;
 459 };
 460
 461 struct cpsw_priv {
 462         struct net_device               *ndev;
 463         struct device                   *dev;
 464         u32                             msg_enable;
 465         u8                              mac_addr[ETH_ALEN];
 466         bool                            rx_pause;
 467         bool                            tx_pause;
 468         bool                            mqprio_hw;
 469         int                             fifo_bw[CPSW_TC_NUM];
 470         int                             shp_cfg_speed;
 471         u32 emac_port;
 472         struct cpsw_common *cpsw;
 473 };
 474
 475 struct cpsw_stats {
 476         char stat_string[ETH_GSTRING_LEN];
 477         int type;
 478         int sizeof_stat;
 479         int stat_offset;
 480 };
 481
 482 enum {
 483         CPSW_STATS,
 484         CPDMA_RX_STATS,
 485         CPDMA_TX_STATS,
 486 };
 487
 488 #define CPSW_STAT(m)            CPSW_STATS,                             \
 489                                 FIELD_SIZEOF(struct cpsw_hw_stats, m), \
 490                                 offsetof(struct cpsw_hw_stats, m)
 491 #define CPDMA_RX_STAT(m)        CPDMA_RX_STATS,                            \
 492                                 FIELD_SIZEOF(struct cpdma_chan_stats, m), \
 493                                 offsetof(struct cpdma_chan_stats, m)
 494 #define CPDMA_TX_STAT(m)        CPDMA_TX_STATS,                            \
 495                                 FIELD_SIZEOF(struct cpdma_chan_stats, m), \
 496                                 offsetof(struct cpdma_chan_stats, m)
 497
 498 static const struct cpsw_stats cpsw_gstrings_stats[] = {
 499         { "Good Rx Frames", CPSW_STAT(rxgoodframes) },
 500         { "Broadcast Rx Frames", CPSW_STAT(rxbroadcastframes) },
 501         { "Multicast Rx Frames", CPSW_STAT(rxmulticastframes) },
 502         { "Pause Rx Frames", CPSW_STAT(rxpauseframes) },
 503         { "Rx CRC Errors", CPSW_STAT(rxcrcerrors) },
 504         { "Rx Align/Code Errors", CPSW_STAT(rxaligncodeerrors) },
 505         { "Oversize Rx Frames", CPSW_STAT(rxoversizedframes) },
 506         { "Rx Jabbers", CPSW_STAT(rxjabberframes) },
 507         { "Undersize (Short) Rx Frames", CPSW_STAT(rxundersizedframes) },
 508         { "Rx Fragments", CPSW_STAT(rxfragments) },
 509         { "Rx Octets", CPSW_STAT(rxoctets) },
 510         { "Good Tx Frames", CPSW_STAT(txgoodframes) },
 511         { "Broadcast Tx Frames", CPSW_STAT(txbroadcastframes) },
 512         { "Multicast Tx Frames", CPSW_STAT(txmulticastframes) },
 513         { "Pause Tx Frames", CPSW_STAT(txpauseframes) },
 514         { "Deferred Tx Frames", CPSW_STAT(txdeferredframes) },
 515         { "Collisions", CPSW_STAT(txcollisionframes) },
 516         { "Single Collision Tx Frames", CPSW_STAT(txsinglecollframes) },
 517         { "Multiple Collision Tx Frames", CPSW_STAT(txmultcollframes) },
 518         { "Excessive Collisions", CPSW_STAT(txexcessivecollisions) },
 519         { "Late Collisions", CPSW_STAT(txlatecollisions) },
 520         { "Tx Underrun", CPSW_STAT(txunderrun) },
 521         { "Carrier Sense Errors", CPSW_STAT(txcarriersenseerrors) },
 522         { "Tx Octets", CPSW_STAT(txoctets) },
 523         { "Rx + Tx 64 Octet Frames", CPSW_STAT(octetframes64) },
 524         { "Rx + Tx 65-127 Octet Frames", CPSW_STAT(octetframes65t127) },
 525         { "Rx + Tx 128-255 Octet Frames", CPSW_STAT(octetframes128t255) },
 526         { "Rx + Tx 256-511 Octet Frames", CPSW_STAT(octetframes256t511) },
 527         { "Rx + Tx 512-1023 Octet Frames", CPSW_STAT(octetframes512t1023) },
 528         { "Rx + Tx 1024-Up Octet Frames", CPSW_STAT(octetframes1024tup) },
 529         { "Net Octets", CPSW_STAT(netoctets) },
 530         { "Rx Start of Frame Overruns", CPSW_STAT(rxsofoverruns) },
 531         { "Rx Middle of Frame Overruns", CPSW_STAT(rxmofoverruns) },
 532         { "Rx DMA Overruns", CPSW_STAT(rxdmaoverruns) },
 533 };
 534
 535 static const struct cpsw_stats cpsw_gstrings_ch_stats[] = {
 536         { "head_enqueue", CPDMA_RX_STAT(head_enqueue) },
 537         { "tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
 538         { "pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
 539         { "misqueued", CPDMA_RX_STAT(misqueued) },
 540         { "desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
 541         { "pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
 542         { "runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
 543         { "runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
 544         { "empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
 545         { "busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
 546         { "good_dequeue", CPDMA_RX_STAT(good_dequeue) },
 547         { "requeue", CPDMA_RX_STAT(requeue) },
 548         { "teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
 549 };
 550
 551 #define CPSW_STATS_COMMON_LEN   ARRAY_SIZE(cpsw_gstrings_stats)
 552 #define CPSW_STATS_CH_LEN       ARRAY_SIZE(cpsw_gstrings_ch_stats)
 553
 554 #define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw)
 555 #define napi_to_cpsw(napi)      container_of(napi, struct cpsw_common, napi)
 556 #define for_each_slave(priv, func, arg...)                              \
 557         do {                                                            \
 558                 struct cpsw_slave *slave;                               \
 559                 struct cpsw_common *cpsw = (priv)->cpsw;                \
 560                 int n;                                                  \
 561                 if (cpsw->data.dual_emac)                               \
 562                         (func)((cpsw)->slaves + priv->emac_port, ##arg);\
 563                 else                                                    \
 564                         for (n = cpsw->data.slaves,                     \
 565                                         slave = cpsw->slaves;           \
 566                                         n; n--)                         \
 567                                 (func)(slave++, ##arg);                 \
 568         } while (0)
 569
 570 static inline int cpsw_get_slave_port(u32 slave_num)
 571 {
 572         return slave_num + 1;
 573 }
 574
 575 static void cpsw_add_mcast(struct cpsw_priv *priv, const u8 *addr)
 576 {
 577         struct cpsw_common *cpsw = priv->cpsw;
 578
 579         if (cpsw->data.dual_emac) {
 580                 struct cpsw_slave *slave = cpsw->slaves + priv->emac_port;
 581
 582                 cpsw_ale_add_mcast(cpsw->ale, addr, ALE_PORT_HOST,
 583                                    ALE_VLAN, slave->port_vlan, 0);
 584                 return;
 585         }
 586
 587         cpsw_ale_add_mcast(cpsw->ale, addr, ALE_ALL_PORTS, 0, 0, 0);
 588 }
 589
 590 static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
 591 {
 592         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 593         struct cpsw_ale *ale = cpsw->ale;
 594         int i;
 595
 596         if (cpsw->data.dual_emac) {
 597                 bool flag = false;
 598
 599                 /* Enabling promiscuous mode for one interface will be
 600                  * common for both the interface as the interface shares
 601                  * the same hardware resource.
 602                  */
 603                 for (i = 0; i < cpsw->data.slaves; i++)
 604                         if (cpsw->slaves[i].ndev->flags & IFF_PROMISC)
 605                                 flag = true;
 606
 607                 if (!enable && flag) {
 608                         enable = true;
 609                         dev_err(&ndev->dev, "promiscuity not disabled as the other interface is still in promiscuity mode\n");
 610                 }
 611
 612                 if (enable) {
 613                         /* Enable Bypass */
 614                         cpsw_ale_control_set(ale, 0, ALE_BYPASS, 1);
 615
 616                         dev_dbg(&ndev->dev, "promiscuity enabled\n");
 617                 } else {
 618                         /* Disable Bypass */
 619                         cpsw_ale_control_set(ale, 0, ALE_BYPASS, 0);
 620                         dev_dbg(&ndev->dev, "promiscuity disabled\n");
 621                 }
 622         } else {
 623                 if (enable) {
 624                         unsigned long timeout = jiffies + HZ;
 625
 626                         /* Disable Learn for all ports (host is port 0 and slaves are port 1 and up */
 627                         for (i = 0; i <= cpsw->data.slaves; i++) {
 628                                 cpsw_ale_control_set(ale, i,
 629                                                      ALE_PORT_NOLEARN, 1);
 630                                 cpsw_ale_control_set(ale, i,
 631                                                      ALE_PORT_NO_SA_UPDATE, 1);
 632                         }
 633
 634                         /* Clear All Untouched entries */
 635                         cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
 636                         do {
 637                                 cpu_relax();
 638                                 if (cpsw_ale_control_get(ale, 0, ALE_AGEOUT))
 639                                         break;
 640                         } while (time_after(timeout, jiffies));
 641                         cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
 642
 643                         /* Clear all mcast from ALE */
 644                         cpsw_ale_flush_multicast(ale, ALE_ALL_PORTS, -1);
 645                         __dev_mc_unsync(ndev, NULL);
 646
 647                         /* Flood All Unicast Packets to Host port */
 648                         cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 1);
 649                         dev_dbg(&ndev->dev, "promiscuity enabled\n");
 650                 } else {
 651                         /* Don't Flood All Unicast Packets to Host port */
 652                         cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
 653
 654                         /* Enable Learn for all ports (host is port 0 and slaves are port 1 and up */
 655                         for (i = 0; i <= cpsw->data.slaves; i++) {
 656                                 cpsw_ale_control_set(ale, i,
 657                                                      ALE_PORT_NOLEARN, 0);
 658                                 cpsw_ale_control_set(ale, i,
 659                                                      ALE_PORT_NO_SA_UPDATE, 0);
 660                         }
 661                         dev_dbg(&ndev->dev, "promiscuity disabled\n");
 662                 }
 663         }
 664 }
 665
 666 static int cpsw_add_mc_addr(struct net_device *ndev, const u8 *addr)
 667 {
 668         struct cpsw_priv *priv = netdev_priv(ndev);
 669
 670         cpsw_add_mcast(priv, addr);
 671         return 0;
 672 }
 673
 674 static int cpsw_del_mc_addr(struct net_device *ndev, const u8 *addr)
 675 {
 676         struct cpsw_priv *priv = netdev_priv(ndev);
 677         struct cpsw_common *cpsw = priv->cpsw;
 678         int vid, flags;
 679
 680         if (cpsw->data.dual_emac) {
 681                 vid = cpsw->slaves[priv->emac_port].port_vlan;
 682                 flags = ALE_VLAN;
 683         } else {
 684                 vid = 0;
 685                 flags = 0;
 686         }
 687
 688         cpsw_ale_del_mcast(cpsw->ale, addr, 0, flags, vid);
 689         return 0;
 690 }
 691
 692 static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
 693 {
 694         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 695
 696         if (ndev->flags & IFF_PROMISC) {
 697                 /* Enable promiscuous mode */
 698                 cpsw_set_promiscious(ndev, true);
 699                 cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI);
 700                 return;
 701         } else {
 702                 /* Disable promiscuous mode */
 703                 cpsw_set_promiscious(ndev, false);
 704         }
 705
 706         /* Restore allmulti on vlans if necessary */
 707         cpsw_ale_set_allmulti(cpsw->ale, ndev->flags & IFF_ALLMULTI);
 708
 709         __dev_mc_sync(ndev, cpsw_add_mc_addr, cpsw_del_mc_addr);
 710 }
 711
 712 static void cpsw_intr_enable(struct cpsw_common *cpsw)
 713 {
 714         writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);
 715         writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);
 716
 717         cpdma_ctlr_int_ctrl(cpsw->dma, true);
 718         return;
 719 }
 720
 721 static void cpsw_intr_disable(struct cpsw_common *cpsw)
 722 {
 723         writel_relaxed(0, &cpsw->wr_regs->tx_en);
 724         writel_relaxed(0, &cpsw->wr_regs->rx_en);
 725
 726         cpdma_ctlr_int_ctrl(cpsw->dma, false);
 727         return;
 728 }
 729
 730 static void cpsw_tx_handler(void *token, int len, int status)
 731 {
 732         struct netdev_queue     *txq;
 733         struct sk_buff          *skb = token;
 734         struct net_device       *ndev = skb->dev;
 735         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
 736
 737         /* Check whether the queue is stopped due to stalled tx dma, if the
 738          * queue is stopped then start the queue as we have free desc for tx
 739          */
 740         txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
 741         if (unlikely(netif_tx_queue_stopped(txq)))
 742                 netif_tx_wake_queue(txq);
 743
 744         cpts_tx_timestamp(cpsw->cpts, skb);
 745         ndev->stats.tx_packets++;
 746         ndev->stats.tx_bytes += len;
 747         dev_kfree_skb_any(skb);
 748 }
 749
 750 static void cpsw_rx_vlan_encap(struct sk_buff *skb)
 751 {
 752         struct cpsw_priv *priv = netdev_priv(skb->dev);
 753         struct cpsw_common *cpsw = priv->cpsw;
 754         u32 rx_vlan_encap_hdr = *((u32 *)skb->data);
 755         u16 vtag, vid, prio, pkt_type;
 756
 757         /* Remove VLAN header encapsulation word */
 758         skb_pull(skb, CPSW_RX_VLAN_ENCAP_HDR_SIZE);
 759
 760         pkt_type = (rx_vlan_encap_hdr >>
 761                     CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT) &
 762                     CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK;
 763         /* Ignore unknown & Priority-tagged packets*/
 764         if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV ||
 765             pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG)
 766                 return;
 767
 768         vid = (rx_vlan_encap_hdr >>
 769                CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT) &
 770                VLAN_VID_MASK;
 771         /* Ignore vid 0 and pass packet as is */
 772         if (!vid)
 773                 return;
 774         /* Ignore default vlans in dual mac mode */
 775         if (cpsw->data.dual_emac &&
 776             vid == cpsw->slaves[priv->emac_port].port_vlan)
 777                 return;
 778
 779         prio = (rx_vlan_encap_hdr >>
 780                 CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT) &
 781                 CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK;
 782
 783         vtag = (prio << VLAN_PRIO_SHIFT) | vid;
 784         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
 785
 786         /* strip vlan tag for VLAN-tagged packet */
 787         if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG) {
 788                 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
 789                 skb_pull(skb, VLAN_HLEN);
 790         }
 791 }
 792
 793 static void cpsw_rx_handler(void *token, int len, int status)
 794 {
 795         struct cpdma_chan       *ch;
 796         struct sk_buff          *skb = token;
 797         struct sk_buff          *new_skb;
 798         struct net_device       *ndev = skb->dev;
 799         int                     ret = 0, port;
 800         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
 801
 802         if (cpsw->data.dual_emac) {
 803                 port = CPDMA_RX_SOURCE_PORT(status);
 804                 if (port) {
 805                         ndev = cpsw->slaves[--port].ndev;
 806                         skb->dev = ndev;
 807                 }
 808         }
 809
 810         if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
 811                 /* In dual emac mode check for all interfaces */
 812                 if (cpsw->data.dual_emac && cpsw->usage_count &&
 813                     (status >= 0)) {
 814                         /* The packet received is for the interface which
 815                          * is already down and the other interface is up
 816                          * and running, instead of freeing which results
 817                          * in reducing of the number of rx descriptor in
 818                          * DMA engine, requeue skb back to cpdma.
 819                          */
 820                         new_skb = skb;
 821                         goto requeue;
 822                 }
 823
 824                 /* the interface is going down, skbs are purged */
 825                 dev_kfree_skb_any(skb);
 826                 return;
 827         }
 828
 829         new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
 830         if (new_skb) {
 831                 skb_copy_queue_mapping(new_skb, skb);
 832                 skb_put(skb, len);
 833                 if (status & CPDMA_RX_VLAN_ENCAP)
 834                         cpsw_rx_vlan_encap(skb);
 835                 cpts_rx_timestamp(cpsw->cpts, skb);
 836                 skb->protocol = eth_type_trans(skb, ndev);
 837                 netif_receive_skb(skb);
 838                 ndev->stats.rx_bytes += len;
 839                 ndev->stats.rx_packets++;
 840                 kmemleak_not_leak(new_skb);
 841         } else {
 842                 ndev->stats.rx_dropped++;
 843                 new_skb = skb;
 844         }
 845
 846 requeue:
 847         if (netif_dormant(ndev)) {
 848                 dev_kfree_skb_any(new_skb);
 849                 return;
 850         }
 851
 852         ch = cpsw->rxv[skb_get_queue_mapping(new_skb)].ch;
 853         ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
 854                                 skb_tailroom(new_skb), 0);
 855         if (WARN_ON(ret < 0))
 856                 dev_kfree_skb_any(new_skb);
 857 }
 858
 859 static void cpsw_split_res(struct net_device *ndev)
 860 {
 861         struct cpsw_priv *priv = netdev_priv(ndev);
 862         u32 consumed_rate = 0, bigest_rate = 0;
 863         struct cpsw_common *cpsw = priv->cpsw;
 864         struct cpsw_vector *txv = cpsw->txv;
 865         int i, ch_weight, rlim_ch_num = 0;
 866         int budget, bigest_rate_ch = 0;
 867         u32 ch_rate, max_rate;
 868         int ch_budget = 0;
 869
 870         for (i = 0; i < cpsw->tx_ch_num; i++) {
 871                 ch_rate = cpdma_chan_get_rate(txv[i].ch);
 872                 if (!ch_rate)
 873                         continue;
 874
 875                 rlim_ch_num++;
 876                 consumed_rate += ch_rate;
 877         }
 878
 879         if (cpsw->tx_ch_num == rlim_ch_num) {
 880                 max_rate = consumed_rate;
 881         } else if (!rlim_ch_num) {
 882                 ch_budget = CPSW_POLL_WEIGHT / cpsw->tx_ch_num;
 883                 bigest_rate = 0;
 884                 max_rate = consumed_rate;
 885         } else {
 886                 max_rate = cpsw->speed * 1000;
 887
 888                 /* if max_rate is less then expected due to reduced link speed,
 889                  * split proportionally according next potential max speed
 890                  */
 891                 if (max_rate < consumed_rate)
 892                         max_rate *= 10;
 893
 894                 if (max_rate < consumed_rate)
 895                         max_rate *= 10;
 896
 897                 ch_budget = (consumed_rate * CPSW_POLL_WEIGHT) / max_rate;
 898                 ch_budget = (CPSW_POLL_WEIGHT - ch_budget) /
 899                             (cpsw->tx_ch_num - rlim_ch_num);
 900                 bigest_rate = (max_rate - consumed_rate) /
 901                               (cpsw->tx_ch_num - rlim_ch_num);
 902         }
 903
 904         /* split tx weight/budget */
 905         budget = CPSW_POLL_WEIGHT;
 906         for (i = 0; i < cpsw->tx_ch_num; i++) {
 907                 ch_rate = cpdma_chan_get_rate(txv[i].ch);
 908                 if (ch_rate) {
 909                         txv[i].budget = (ch_rate * CPSW_POLL_WEIGHT) / max_rate;
 910                         if (!txv[i].budget)
 911                                 txv[i].budget++;
 912                         if (ch_rate > bigest_rate) {
 913                                 bigest_rate_ch = i;
 914                                 bigest_rate = ch_rate;
 915                         }
 916
 917                         ch_weight = (ch_rate * 100) / max_rate;
 918                         if (!ch_weight)
 919                                 ch_weight++;
 920                         cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);
 921                 } else {
 922                         txv[i].budget = ch_budget;
 923                         if (!bigest_rate_ch)
 924                                 bigest_rate_ch = i;
 925                         cpdma_chan_set_weight(cpsw->txv[i].ch, 0);
 926                 }
 927
 928                 budget -= txv[i].budget;
 929         }
 930
 931         if (budget)
 932                 txv[bigest_rate_ch].budget += budget;
 933
 934         /* split rx budget */
 935         budget = CPSW_POLL_WEIGHT;
 936         ch_budget = budget / cpsw->rx_ch_num;
 937         for (i = 0; i < cpsw->rx_ch_num; i++) {
 938                 cpsw->rxv[i].budget = ch_budget;
 939                 budget -= ch_budget;
 940         }
 941
 942         if (budget)
 943                 cpsw->rxv[0].budget += budget;
 944 }
 945
 946 static irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
 947 {
 948         struct cpsw_common *cpsw = dev_id;
 949
 950         writel(0, &cpsw->wr_regs->tx_en);
 951         cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
 952
 953         if (cpsw->quirk_irq) {
 954                 disable_irq_nosync(cpsw->irqs_table[1]);
 955                 cpsw->tx_irq_disabled = true;
 956         }
 957
 958         napi_schedule(&cpsw->napi_tx);
 959         return IRQ_HANDLED;
 960 }
 961
 962 static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
 963 {
 964         struct cpsw_common *cpsw = dev_id;
 965
 966         cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
 967         writel(0, &cpsw->wr_regs->rx_en);
 968
 969         if (cpsw->quirk_irq) {
 970                 disable_irq_nosync(cpsw->irqs_table[0]);
 971                 cpsw->rx_irq_disabled = true;
 972         }
 973
 974         napi_schedule(&cpsw->napi_rx);
 975         return IRQ_HANDLED;
 976 }
 977
 978 static int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
 979 {
 980         u32                     ch_map;
 981         int                     num_tx, cur_budget, ch;
 982         struct cpsw_common      *cpsw = napi_to_cpsw(napi_tx);
 983         struct cpsw_vector      *txv;
 984
 985         /* process every unprocessed channel */
 986         ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
 987         for (ch = 0, num_tx = 0; ch_map & 0xff; ch_map <<= 1, ch++) {
 988                 if (!(ch_map & 0x80))
 989                         continue;
 990
 991                 txv = &cpsw->txv[ch];
 992                 if (unlikely(txv->budget > budget - num_tx))
 993                         cur_budget = budget - num_tx;
 994                 else
 995                         cur_budget = txv->budget;
 996
 997                 num_tx += cpdma_chan_process(txv->ch, cur_budget);
 998                 if (num_tx >= budget)
 999                         break;
1000         }
1001
1002         if (num_tx < budget) {
1003                 napi_complete(napi_tx);
1004                 writel(0xff, &cpsw->wr_regs->tx_en);
1005         }
1006
1007         return num_tx;
1008 }
1009
1010 static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
1011 {
1012         struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
1013         int num_tx;
1014
1015         num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);
1016         if (num_tx < budget) {
1017                 napi_complete(napi_tx);
1018                 writel(0xff, &cpsw->wr_regs->tx_en);
1019                 if (cpsw->tx_irq_disabled) {
1020                         cpsw->tx_irq_disabled = false;
1021                         enable_irq(cpsw->irqs_table[1]);
1022                 }
1023         }
1024
1025         return num_tx;
1026 }
1027
1028 static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
1029 {
1030         u32                     ch_map;
1031         int                     num_rx, cur_budget, ch;
1032         struct cpsw_common      *cpsw = napi_to_cpsw(napi_rx);
1033         struct cpsw_vector      *rxv;
1034
1035         /* process every unprocessed channel */
1036         ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
1037         for (ch = 0, num_rx = 0; ch_map; ch_map >>= 1, ch++) {
1038                 if (!(ch_map & 0x01))
1039                         continue;
1040
1041                 rxv = &cpsw->rxv[ch];
1042                 if (unlikely(rxv->budget > budget - num_rx))
1043                         cur_budget = budget - num_rx;
1044                 else
1045                         cur_budget = rxv->budget;
1046
1047                 num_rx += cpdma_chan_process(rxv->ch, cur_budget);
1048                 if (num_rx >= budget)
1049                         break;
1050         }
1051
1052         if (num_rx < budget) {
1053                 napi_complete_done(napi_rx, num_rx);
1054                 writel(0xff, &cpsw->wr_regs->rx_en);
1055         }
1056
1057         return num_rx;
1058 }
1059
1060 static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
1061 {
1062         struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
1063         int num_rx;
1064
1065         num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);
1066         if (num_rx < budget) {
1067                 napi_complete_done(napi_rx, num_rx);
1068                 writel(0xff, &cpsw->wr_regs->rx_en);
1069                 if (cpsw->rx_irq_disabled) {
1070                         cpsw->rx_irq_disabled = false;
1071                         enable_irq(cpsw->irqs_table[0]);
1072                 }
1073         }
1074
1075         return num_rx;
1076 }
1077
1078 static inline void soft_reset(const char *module, void __iomem *reg)
1079 {
1080         unsigned long timeout = jiffies + HZ;
1081
1082         writel_relaxed(1, reg);
1083         do {
1084                 cpu_relax();
1085         } while ((readl_relaxed(reg) & 1) && time_after(timeout, jiffies));
1086
1087         WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module);
1088 }
1089
1090 static void cpsw_set_slave_mac(struct cpsw_slave *slave,
1091                                struct cpsw_priv *priv)
1092 {
1093         slave_write(slave, mac_hi(priv->mac_addr), SA_HI);
1094         slave_write(slave, mac_lo(priv->mac_addr), SA_LO);
1095 }
1096
1097 static bool cpsw_shp_is_off(struct cpsw_priv *priv)
1098 {
1099         struct cpsw_common *cpsw = priv->cpsw;
1100         struct cpsw_slave *slave;
1101         u32 shift, mask, val;
1102
1103         val = readl_relaxed(&cpsw->regs->ptype);
1104
1105         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1106         shift = CPSW_FIFO_SHAPE_EN_SHIFT + 3 * slave->slave_num;
1107         mask = 7 << shift;
1108         val = val & mask;
1109
1110         return !val;
1111 }
1112
1113 static void cpsw_fifo_shp_on(struct cpsw_priv *priv, int fifo, int on)
1114 {
1115         struct cpsw_common *cpsw = priv->cpsw;
1116         struct cpsw_slave *slave;
1117         u32 shift, mask, val;
1118
1119         val = readl_relaxed(&cpsw->regs->ptype);
1120
1121         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1122         shift = CPSW_FIFO_SHAPE_EN_SHIFT + 3 * slave->slave_num;
1123         mask = (1 << --fifo) << shift;
1124         val = on ? val | mask : val & ~mask;
1125
1126         writel_relaxed(val, &cpsw->regs->ptype);
1127 }
1128
1129 static void _cpsw_adjust_link(struct cpsw_slave *slave,
1130                               struct cpsw_priv *priv, bool *link)
1131 {
1132         struct phy_device       *phy = slave->phy;
1133         u32                     mac_control = 0;
1134         u32                     slave_port;
1135         struct cpsw_common *cpsw = priv->cpsw;
1136
1137         if (!phy)
1138                 return;
1139
1140         slave_port = cpsw_get_slave_port(slave->slave_num);
1141
1142         if (phy->link) {
1143                 mac_control = cpsw->data.mac_control;
1144
1145                 /* enable forwarding */
1146                 cpsw_ale_control_set(cpsw->ale, slave_port,
1147                                      ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1148
1149                 if (phy->speed == 1000)
1150                         mac_control |= BIT(7);  /* GIGABITEN    */
1151                 if (phy->duplex)
1152                         mac_control |= BIT(0);  /* FULLDUPLEXEN */
1153
1154                 /* set speed_in input in case RMII mode is used in 100Mbps */
1155                 if (phy->speed == 100)
1156                         mac_control |= BIT(15);
1157                 /* in band mode only works in 10Mbps RGMII mode */
1158                 else if ((phy->speed == 10) && phy_interface_is_rgmii(phy))
1159                         mac_control |= BIT(18); /* In Band mode */
1160
1161                 if (priv->rx_pause)
1162                         mac_control |= BIT(3);
1163
1164                 if (priv->tx_pause)
1165                         mac_control |= BIT(4);
1166
1167                 *link = true;
1168
1169                 if (priv->shp_cfg_speed &&
1170                     priv->shp_cfg_speed != slave->phy->speed &&
1171                     !cpsw_shp_is_off(priv))
1172                         dev_warn(priv->dev,
1173                                  "Speed was changed, CBS shaper speeds are changed!");
1174         } else {
1175                 mac_control = 0;
1176                 /* disable forwarding */
1177                 cpsw_ale_control_set(cpsw->ale, slave_port,
1178                                      ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1179         }
1180
1181         if (mac_control != slave->mac_control) {
1182                 phy_print_status(phy);
1183                 writel_relaxed(mac_control, &slave->sliver->mac_control);
1184         }
1185
1186         slave->mac_control = mac_control;
1187 }
1188
1189 static int cpsw_get_common_speed(struct cpsw_common *cpsw)
1190 {
1191         int i, speed;
1192
1193         for (i = 0, speed = 0; i < cpsw->data.slaves; i++)
1194                 if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)
1195                         speed += cpsw->slaves[i].phy->speed;
1196
1197         return speed;
1198 }
1199
1200 static int cpsw_need_resplit(struct cpsw_common *cpsw)
1201 {
1202         int i, rlim_ch_num;
1203         int speed, ch_rate;
1204
1205         /* re-split resources only in case speed was changed */
1206         speed = cpsw_get_common_speed(cpsw);
1207         if (speed == cpsw->speed || !speed)
1208                 return 0;
1209
1210         cpsw->speed = speed;
1211
1212         for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {
1213                 ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);
1214                 if (!ch_rate)
1215                         break;
1216
1217                 rlim_ch_num++;
1218         }
1219
1220         /* cases not dependent on speed */
1221         if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)
1222                 return 0;
1223
1224         return 1;
1225 }
1226
1227 static void cpsw_adjust_link(struct net_device *ndev)
1228 {
1229         struct cpsw_priv        *priv = netdev_priv(ndev);
1230         struct cpsw_common      *cpsw = priv->cpsw;
1231         bool                    link = false;
1232
1233         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
1234
1235         if (link) {
1236                 if (cpsw_need_resplit(cpsw))
1237                         cpsw_split_res(ndev);
1238
1239                 netif_carrier_on(ndev);
1240                 if (netif_running(ndev))
1241                         netif_tx_wake_all_queues(ndev);
1242         } else {
1243                 netif_carrier_off(ndev);
1244                 netif_tx_stop_all_queues(ndev);
1245         }
1246 }
1247
1248 static int cpsw_get_coalesce(struct net_device *ndev,
1249                                 struct ethtool_coalesce *coal)
1250 {
1251         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1252
1253         coal->rx_coalesce_usecs = cpsw->coal_intvl;
1254         return 0;
1255 }
1256
1257 static int cpsw_set_coalesce(struct net_device *ndev,
1258                                 struct ethtool_coalesce *coal)
1259 {
1260         struct cpsw_priv *priv = netdev_priv(ndev);
1261         u32 int_ctrl;
1262         u32 num_interrupts = 0;
1263         u32 prescale = 0;
1264         u32 addnl_dvdr = 1;
1265         u32 coal_intvl = 0;
1266         struct cpsw_common *cpsw = priv->cpsw;
1267
1268         coal_intvl = coal->rx_coalesce_usecs;
1269
1270         int_ctrl =  readl(&cpsw->wr_regs->int_control);
1271         prescale = cpsw->bus_freq_mhz * 4;
1272
1273         if (!coal->rx_coalesce_usecs) {
1274                 int_ctrl &= ~(CPSW_INTPRESCALE_MASK | CPSW_INTPACEEN);
1275                 goto update_return;
1276         }
1277
1278         if (coal_intvl < CPSW_CMINTMIN_INTVL)
1279                 coal_intvl = CPSW_CMINTMIN_INTVL;
1280
1281         if (coal_intvl > CPSW_CMINTMAX_INTVL) {
1282                 /* Interrupt pacer works with 4us Pulse, we can
1283                  * throttle further by dilating the 4us pulse.
1284                  */
1285                 addnl_dvdr = CPSW_INTPRESCALE_MASK / prescale;
1286
1287                 if (addnl_dvdr > 1) {
1288                         prescale *= addnl_dvdr;
1289                         if (coal_intvl > (CPSW_CMINTMAX_INTVL * addnl_dvdr))
1290                                 coal_intvl = (CPSW_CMINTMAX_INTVL
1291                                                 * addnl_dvdr);
1292                 } else {
1293                         addnl_dvdr = 1;
1294                         coal_intvl = CPSW_CMINTMAX_INTVL;
1295                 }
1296         }
1297
1298         num_interrupts = (1000 * addnl_dvdr) / coal_intvl;
1299         writel(num_interrupts, &cpsw->wr_regs->rx_imax);
1300         writel(num_interrupts, &cpsw->wr_regs->tx_imax);
1301
1302         int_ctrl |= CPSW_INTPACEEN;
1303         int_ctrl &= (~CPSW_INTPRESCALE_MASK);
1304         int_ctrl |= (prescale & CPSW_INTPRESCALE_MASK);
1305
1306 update_return:
1307         writel(int_ctrl, &cpsw->wr_regs->int_control);
1308
1309         cpsw_notice(priv, timer, "Set coalesce to %d usecs.\n", coal_intvl);
1310         cpsw->coal_intvl = coal_intvl;
1311
1312         return 0;
1313 }
1314
1315 static int cpsw_get_sset_count(struct net_device *ndev, int sset)
1316 {
1317         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1318
1319         switch (sset) {
1320         case ETH_SS_STATS:
1321                 return (CPSW_STATS_COMMON_LEN +
1322                        (cpsw->rx_ch_num + cpsw->tx_ch_num) *
1323                        CPSW_STATS_CH_LEN);
1324         default:
1325                 return -EOPNOTSUPP;
1326         }
1327 }
1328
1329 static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
1330 {
1331         int ch_stats_len;
1332         int line;
1333         int i;
1334
1335         ch_stats_len = CPSW_STATS_CH_LEN * ch_num;
1336         for (i = 0; i < ch_stats_len; i++) {
1337                 line = i % CPSW_STATS_CH_LEN;
1338                 snprintf(*p, ETH_GSTRING_LEN,
1339                          "%s DMA chan %ld: %s", rx_dir ? "Rx" : "Tx",
1340                          (long)(i / CPSW_STATS_CH_LEN),
1341                          cpsw_gstrings_ch_stats[line].stat_string);
1342                 *p += ETH_GSTRING_LEN;
1343         }
1344 }
1345
1346 static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1347 {
1348         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1349         u8 *p = data;
1350         int i;
1351
1352         switch (stringset) {
1353         case ETH_SS_STATS:
1354                 for (i = 0; i < CPSW_STATS_COMMON_LEN; i++) {
1355                         memcpy(p, cpsw_gstrings_stats[i].stat_string,
1356                                ETH_GSTRING_LEN);
1357                         p += ETH_GSTRING_LEN;
1358                 }
1359
1360                 cpsw_add_ch_strings(&p, cpsw->rx_ch_num, 1);
1361                 cpsw_add_ch_strings(&p, cpsw->tx_ch_num, 0);
1362                 break;
1363         }
1364 }
1365
1366 static void cpsw_get_ethtool_stats(struct net_device *ndev,
1367                                     struct ethtool_stats *stats, u64 *data)
1368 {
1369         u8 *p;
1370         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1371         struct cpdma_chan_stats ch_stats;
1372         int i, l, ch;
1373
1374         /* Collect Davinci CPDMA stats for Rx and Tx Channel */
1375         for (l = 0; l < CPSW_STATS_COMMON_LEN; l++)
1376                 data[l] = readl(cpsw->hw_stats +
1377                                 cpsw_gstrings_stats[l].stat_offset);
1378
1379         for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1380                 cpdma_chan_get_stats(cpsw->rxv[ch].ch, &ch_stats);
1381                 for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1382                         p = (u8 *)&ch_stats +
1383                                 cpsw_gstrings_ch_stats[i].stat_offset;
1384                         data[l] = *(u32 *)p;
1385                 }
1386         }
1387
1388         for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1389                 cpdma_chan_get_stats(cpsw->txv[ch].ch, &ch_stats);
1390                 for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1391                         p = (u8 *)&ch_stats +
1392                                 cpsw_gstrings_ch_stats[i].stat_offset;
1393                         data[l] = *(u32 *)p;
1394                 }
1395         }
1396 }
1397
1398 static inline int cpsw_tx_packet_submit(struct cpsw_priv *priv,
1399                                         struct sk_buff *skb,
1400                                         struct cpdma_chan *txch)
1401 {
1402         struct cpsw_common *cpsw = priv->cpsw;
1403
1404         skb_tx_timestamp(skb);
1405         return cpdma_chan_submit(txch, skb, skb->data, skb->len,
1406                                  priv->emac_port + cpsw->data.dual_emac);
1407 }
1408
1409 static inline void cpsw_add_dual_emac_def_ale_entries(
1410                 struct cpsw_priv *priv, struct cpsw_slave *slave,
1411                 u32 slave_port)
1412 {
1413         struct cpsw_common *cpsw = priv->cpsw;
1414         u32 port_mask = 1 << slave_port | ALE_PORT_HOST;
1415
1416         if (cpsw->version == CPSW_VERSION_1)
1417                 slave_write(slave, slave->port_vlan, CPSW1_PORT_VLAN);
1418         else
1419                 slave_write(slave, slave->port_vlan, CPSW2_PORT_VLAN);
1420         cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
1421                           port_mask, port_mask, 0);
1422         cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1423                            ALE_PORT_HOST, ALE_VLAN, slave->port_vlan, 0);
1424         cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1425                            HOST_PORT_NUM, ALE_VLAN |
1426                            ALE_SECURE, slave->port_vlan);
1427         cpsw_ale_control_set(cpsw->ale, slave_port,
1428                              ALE_PORT_DROP_UNKNOWN_VLAN, 1);
1429 }
1430
1431 static void soft_reset_slave(struct cpsw_slave *slave)
1432 {
1433         char name[32];
1434
1435         snprintf(name, sizeof(name), "slave-%d", slave->slave_num);
1436         soft_reset(name, &slave->sliver->soft_reset);
1437 }
1438
1439 static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
1440 {
1441         u32 slave_port;
1442         struct phy_device *phy;
1443         struct cpsw_common *cpsw = priv->cpsw;
1444
1445         soft_reset_slave(slave);
1446
1447         /* setup priority mapping */
1448         writel_relaxed(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
1449
1450         switch (cpsw->version) {
1451         case CPSW_VERSION_1:
1452                 slave_write(slave, TX_PRIORITY_MAPPING, CPSW1_TX_PRI_MAP);
1453                 /* Increase RX FIFO size to 5 for supporting fullduplex
1454                  * flow control mode
1455                  */
1456                 slave_write(slave,
1457                             (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1458                             CPSW_MAX_BLKS_RX, CPSW1_MAX_BLKS);
1459                 break;
1460         case CPSW_VERSION_2:
1461         case CPSW_VERSION_3:
1462         case CPSW_VERSION_4:
1463                 slave_write(slave, TX_PRIORITY_MAPPING, CPSW2_TX_PRI_MAP);
1464                 /* Increase RX FIFO size to 5 for supporting fullduplex
1465                  * flow control mode
1466                  */
1467                 slave_write(slave,
1468                             (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1469                             CPSW_MAX_BLKS_RX, CPSW2_MAX_BLKS);
1470                 break;
1471         }
1472
1473         /* setup max packet size, and mac address */
1474         writel_relaxed(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
1475         cpsw_set_slave_mac(slave, priv);
1476
1477         slave->mac_control = 0; /* no link yet */
1478
1479         slave_port = cpsw_get_slave_port(slave->slave_num);
1480
1481         if (cpsw->data.dual_emac)
1482                 cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
1483         else
1484                 cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1485                                    1 << slave_port, 0, 0, ALE_MCAST_FWD_2);
1486
1487         if (slave->data->phy_node) {
1488                 phy = of_phy_connect(priv->ndev, slave->data->phy_node,
1489                                  &cpsw_adjust_link, 0, slave->data->phy_if);
1490                 if (!phy) {
1491                         dev_err(priv->dev, "phy \"%pOF\" not found on slave %d\n",
1492                                 slave->data->phy_node,
1493                                 slave->slave_num);
1494                         return;
1495                 }
1496         } else {
1497                 phy = phy_connect(priv->ndev, slave->data->phy_id,
1498                                  &cpsw_adjust_link, slave->data->phy_if);
1499                 if (IS_ERR(phy)) {
1500                         dev_err(priv->dev,
1501                                 "phy \"%s\" not found on slave %d, err %ld\n",
1502                                 slave->data->phy_id, slave->slave_num,
1503                                 PTR_ERR(phy));
1504                         return;
1505                 }
1506         }
1507
1508         slave->phy = phy;
1509
1510         phy_attached_info(slave->phy);
1511
1512         phy_start(slave->phy);
1513
1514         /* Configure GMII_SEL register */
1515         if (!IS_ERR(slave->data->ifphy))
1516                 phy_set_mode_ext(slave->data->ifphy, PHY_MODE_ETHERNET,
1517                                  slave->data->phy_if);
1518         else
1519                 cpsw_phy_sel(cpsw->dev, slave->phy->interface,
1520                              slave->slave_num);
1521 }
1522
1523 static inline void cpsw_add_default_vlan(struct cpsw_priv *priv)
1524 {
1525         struct cpsw_common *cpsw = priv->cpsw;
1526         const int vlan = cpsw->data.default_vlan;
1527         u32 reg;
1528         int i;
1529         int unreg_mcast_mask;
1530
1531         reg = (cpsw->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
1532                CPSW2_PORT_VLAN;
1533
1534         writel(vlan, &cpsw->host_port_regs->port_vlan);
1535
1536         for (i = 0; i < cpsw->data.slaves; i++)
1537                 slave_write(cpsw->slaves + i, vlan, reg);
1538
1539         if (priv->ndev->flags & IFF_ALLMULTI)
1540                 unreg_mcast_mask = ALE_ALL_PORTS;
1541         else
1542                 unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1543
1544         cpsw_ale_add_vlan(cpsw->ale, vlan, ALE_ALL_PORTS,
1545                           ALE_ALL_PORTS, ALE_ALL_PORTS,
1546                           unreg_mcast_mask);
1547 }
1548
1549 static void cpsw_init_host_port(struct cpsw_priv *priv)
1550 {
1551         u32 fifo_mode;
1552         u32 control_reg;
1553         struct cpsw_common *cpsw = priv->cpsw;
1554
1555         /* soft reset the controller and initialize ale */
1556         soft_reset("cpsw", &cpsw->regs->soft_reset);
1557         cpsw_ale_start(cpsw->ale);
1558
1559         /* switch to vlan unaware mode */
1560         cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
1561                              CPSW_ALE_VLAN_AWARE);
1562         control_reg = readl(&cpsw->regs->control);
1563         control_reg |= CPSW_VLAN_AWARE | CPSW_RX_VLAN_ENCAP;
1564         writel(control_reg, &cpsw->regs->control);
1565         fifo_mode = (cpsw->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
1566                      CPSW_FIFO_NORMAL_MODE;
1567         writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
1568
1569         /* setup host port priority mapping */
1570         writel_relaxed(CPDMA_TX_PRIORITY_MAP,
1571                        &cpsw->host_port_regs->cpdma_tx_pri_map);
1572         writel_relaxed(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
1573
1574         cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
1575                              ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1576
1577         if (!cpsw->data.dual_emac) {
1578                 cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1579                                    0, 0);
1580                 cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1581                                    ALE_PORT_HOST, 0, 0, ALE_MCAST_FWD_2);
1582         }
1583 }
1584
1585 static int cpsw_fill_rx_channels(struct cpsw_priv *priv)
1586 {
1587         struct cpsw_common *cpsw = priv->cpsw;
1588         struct sk_buff *skb;
1589         int ch_buf_num;
1590         int ch, i, ret;
1591
1592         for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1593                 ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
1594                 for (i = 0; i < ch_buf_num; i++) {
1595                         skb = __netdev_alloc_skb_ip_align(priv->ndev,
1596                                                           cpsw->rx_packet_max,
1597                                                           GFP_KERNEL);
1598                         if (!skb) {
1599                                 cpsw_err(priv, ifup, "cannot allocate skb\n");
1600                                 return -ENOMEM;
1601                         }
1602
1603                         skb_set_queue_mapping(skb, ch);
1604                         ret = cpdma_chan_submit(cpsw->rxv[ch].ch, skb,
1605                                                 skb->data, skb_tailroom(skb),
1606                                                 0);
1607                         if (ret < 0) {
1608                                 cpsw_err(priv, ifup,
1609                                          "cannot submit skb to channel %d rx, error %d\n",
1610                                          ch, ret);
1611                                 kfree_skb(skb);
1612                                 return ret;
1613                         }
1614                         kmemleak_not_leak(skb);
1615                 }
1616
1617                 cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
1618                           ch, ch_buf_num);
1619         }
1620
1621         return 0;
1622 }
1623
1624 static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_common *cpsw)
1625 {
1626         u32 slave_port;
1627
1628         slave_port = cpsw_get_slave_port(slave->slave_num);
1629
1630         if (!slave->phy)
1631                 return;
1632         phy_stop(slave->phy);
1633         phy_disconnect(slave->phy);
1634         slave->phy = NULL;
1635         cpsw_ale_control_set(cpsw->ale, slave_port,
1636                              ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1637         soft_reset_slave(slave);
1638 }
1639
1640 static int cpsw_tc_to_fifo(int tc, int num_tc)
1641 {
1642         if (tc == num_tc - 1)
1643                 return 0;
1644
1645         return CPSW_FIFO_SHAPERS_NUM - tc;
1646 }
1647
1648 static int cpsw_set_fifo_bw(struct cpsw_priv *priv, int fifo, int bw)
1649 {
1650         struct cpsw_common *cpsw = priv->cpsw;
1651         u32 val = 0, send_pct, shift;
1652         struct cpsw_slave *slave;
1653         int pct = 0, i;
1654
1655         if (bw > priv->shp_cfg_speed * 1000)
1656                 goto err;
1657
1658         /* shaping has to stay enabled for highest fifos linearly
1659          * and fifo bw no more then interface can allow
1660          */
1661         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1662         send_pct = slave_read(slave, SEND_PERCENT);
1663         for (i = CPSW_FIFO_SHAPERS_NUM; i > 0; i--) {
1664                 if (!bw) {
1665                         if (i >= fifo || !priv->fifo_bw[i])
1666                                 continue;
1667
1668                         dev_warn(priv->dev, "Prev FIFO%d is shaped", i);
1669                         continue;
1670                 }
1671
1672                 if (!priv->fifo_bw[i] && i > fifo) {
1673                         dev_err(priv->dev, "Upper FIFO%d is not shaped", i);
1674                         return -EINVAL;
1675                 }
1676
1677                 shift = (i - 1) * 8;
1678                 if (i == fifo) {
1679                         send_pct &= ~(CPSW_PCT_MASK << shift);
1680                         val = DIV_ROUND_UP(bw, priv->shp_cfg_speed * 10);
1681                         if (!val)
1682                                 val = 1;
1683
1684                         send_pct |= val << shift;
1685                         pct += val;
1686                         continue;
1687                 }
1688
1689                 if (priv->fifo_bw[i])
1690                         pct += (send_pct >> shift) & CPSW_PCT_MASK;
1691         }
1692
1693         if (pct >= 100)
1694                 goto err;
1695
1696         slave_write(slave, send_pct, SEND_PERCENT);
1697         priv->fifo_bw[fifo] = bw;
1698
1699         dev_warn(priv->dev, "set FIFO%d bw = %d\n", fifo,
1700                  DIV_ROUND_CLOSEST(val * priv->shp_cfg_speed, 100));
1701
1702         return 0;
1703 err:
1704         dev_err(priv->dev, "Bandwidth doesn't fit in tc configuration");
1705         return -EINVAL;
1706 }
1707
1708 static int cpsw_set_fifo_rlimit(struct cpsw_priv *priv, int fifo, int bw)
1709 {
1710         struct cpsw_common *cpsw = priv->cpsw;
1711         struct cpsw_slave *slave;
1712         u32 tx_in_ctl_rg, val;
1713         int ret;
1714
1715         ret = cpsw_set_fifo_bw(priv, fifo, bw);
1716         if (ret)
1717                 return ret;
1718
1719         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1720         tx_in_ctl_rg = cpsw->version == CPSW_VERSION_1 ?
1721                        CPSW1_TX_IN_CTL : CPSW2_TX_IN_CTL;
1722
1723         if (!bw)
1724                 cpsw_fifo_shp_on(priv, fifo, bw);
1725
1726         val = slave_read(slave, tx_in_ctl_rg);
1727         if (cpsw_shp_is_off(priv)) {
1728                 /* disable FIFOs rate limited queues */
1729                 val &= ~(0xf << CPSW_FIFO_RATE_EN_SHIFT);
1730
1731                 /* set type of FIFO queues to normal priority mode */
1732                 val &= ~(3 << CPSW_FIFO_QUEUE_TYPE_SHIFT);
1733
1734                 /* set type of FIFO queues to be rate limited */
1735                 if (bw)
1736                         val |= 2 << CPSW_FIFO_QUEUE_TYPE_SHIFT;
1737                 else
1738                         priv->shp_cfg_speed = 0;
1739         }
1740
1741         /* toggle a FIFO rate limited queue */
1742         if (bw)
1743                 val |= BIT(fifo + CPSW_FIFO_RATE_EN_SHIFT);
1744         else
1745                 val &= ~BIT(fifo + CPSW_FIFO_RATE_EN_SHIFT);
1746         slave_write(slave, val, tx_in_ctl_rg);
1747
1748         /* FIFO transmit shape enable */
1749         cpsw_fifo_shp_on(priv, fifo, bw);
1750         return 0;
1751 }
1752
1753 /* Defaults:
1754  * class A - prio 3
1755  * class B - prio 2
1756  * shaping for class A should be set first
1757  */
1758 static int cpsw_set_cbs(struct net_device *ndev,
1759                         struct tc_cbs_qopt_offload *qopt)
1760 {
1761         struct cpsw_priv *priv = netdev_priv(ndev);
1762         struct cpsw_common *cpsw = priv->cpsw;
1763         struct cpsw_slave *slave;
1764         int prev_speed = 0;
1765         int tc, ret, fifo;
1766         u32 bw = 0;
1767
1768         tc = netdev_txq_to_tc(priv->ndev, qopt->queue);
1769
1770         /* enable channels in backward order, as highest FIFOs must be rate
1771          * limited first and for compliance with CPDMA rate limited channels
1772          * that also used in bacward order. FIFO0 cannot be rate limited.
1773          */
1774         fifo = cpsw_tc_to_fifo(tc, ndev->num_tc);
1775         if (!fifo) {
1776                 dev_err(priv->dev, "Last tc%d can't be rate limited", tc);
1777                 return -EINVAL;
1778         }
1779
1780         /* do nothing, it's disabled anyway */
1781         if (!qopt->enable && !priv->fifo_bw[fifo])
1782                 return 0;
1783
1784         /* shapers can be set if link speed is known */
1785         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1786         if (slave->phy && slave->phy->link) {
1787                 if (priv->shp_cfg_speed &&
1788                     priv->shp_cfg_speed != slave->phy->speed)
1789                         prev_speed = priv->shp_cfg_speed;
1790
1791                 priv->shp_cfg_speed = slave->phy->speed;
1792         }
1793
1794         if (!priv->shp_cfg_speed) {
1795                 dev_err(priv->dev, "Link speed is not known");
1796                 return -1;
1797         }
1798
1799         ret = pm_runtime_get_sync(cpsw->dev);
1800         if (ret < 0) {
1801                 pm_runtime_put_noidle(cpsw->dev);
1802                 return ret;
1803         }
1804
1805         bw = qopt->enable ? qopt->idleslope : 0;
1806         ret = cpsw_set_fifo_rlimit(priv, fifo, bw);
1807         if (ret) {
1808                 priv->shp_cfg_speed = prev_speed;
1809                 prev_speed = 0;
1810         }
1811
1812         if (bw && prev_speed)
1813                 dev_warn(priv->dev,
1814                          "Speed was changed, CBS shaper speeds are changed!");
1815
1816         pm_runtime_put_sync(cpsw->dev);
1817         return ret;
1818 }
1819
1820 static void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv)
1821 {
1822         int fifo, bw;
1823
1824         for (fifo = CPSW_FIFO_SHAPERS_NUM; fifo > 0; fifo--) {
1825                 bw = priv->fifo_bw[fifo];
1826                 if (!bw)
1827                         continue;
1828
1829                 cpsw_set_fifo_rlimit(priv, fifo, bw);
1830         }
1831 }
1832
1833 static void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv)
1834 {
1835         struct cpsw_common *cpsw = priv->cpsw;
1836         u32 tx_prio_map = 0;
1837         int i, tc, fifo;
1838         u32 tx_prio_rg;
1839
1840         if (!priv->mqprio_hw)
1841                 return;
1842
1843         for (i = 0; i < 8; i++) {
1844                 tc = netdev_get_prio_tc_map(priv->ndev, i);
1845                 fifo = CPSW_FIFO_SHAPERS_NUM - tc;
1846                 tx_prio_map |= fifo << (4 * i);
1847         }
1848
1849         tx_prio_rg = cpsw->version == CPSW_VERSION_1 ?
1850                      CPSW1_TX_PRI_MAP : CPSW2_TX_PRI_MAP;
1851
1852         slave_write(slave, tx_prio_map, tx_prio_rg);
1853 }
1854
1855 /* restore resources after port reset */
1856 static void cpsw_restore(struct cpsw_priv *priv)
1857 {
1858         /* restore MQPRIO offload */
1859         for_each_slave(priv, cpsw_mqprio_resume, priv);
1860
1861         /* restore CBS offload */
1862         for_each_slave(priv, cpsw_cbs_resume, priv);
1863 }
1864
1865 static int cpsw_ndo_open(struct net_device *ndev)
1866 {
1867         struct cpsw_priv *priv = netdev_priv(ndev);
1868         struct cpsw_common *cpsw = priv->cpsw;
1869         int ret;
1870         u32 reg;
1871
1872         ret = pm_runtime_get_sync(cpsw->dev);
1873         if (ret < 0) {
1874                 pm_runtime_put_noidle(cpsw->dev);
1875                 return ret;
1876         }
1877
1878         netif_carrier_off(ndev);
1879
1880         /* Notify the stack of the actual queue counts. */
1881         ret = netif_set_real_num_tx_queues(ndev, cpsw->tx_ch_num);
1882         if (ret) {
1883                 dev_err(priv->dev, "cannot set real number of tx queues\n");
1884                 goto err_cleanup;
1885         }
1886
1887         ret = netif_set_real_num_rx_queues(ndev, cpsw->rx_ch_num);
1888         if (ret) {
1889                 dev_err(priv->dev, "cannot set real number of rx queues\n");
1890                 goto err_cleanup;
1891         }
1892
1893         reg = cpsw->version;
1894
1895         dev_info(priv->dev, "initializing cpsw version %d.%d (%d)\n",
1896                  CPSW_MAJOR_VERSION(reg), CPSW_MINOR_VERSION(reg),
1897                  CPSW_RTL_VERSION(reg));
1898
1899         /* Initialize host and slave ports */
1900         if (!cpsw->usage_count)
1901                 cpsw_init_host_port(priv);
1902         for_each_slave(priv, cpsw_slave_open, priv);
1903
1904         /* Add default VLAN */
1905         if (!cpsw->data.dual_emac)
1906                 cpsw_add_default_vlan(priv);
1907         else
1908                 cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
1909                                   ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
1910
1911         /* initialize shared resources for every ndev */
1912         if (!cpsw->usage_count) {
1913                 /* disable priority elevation */
1914                 writel_relaxed(0, &cpsw->regs->ptype);
1915
1916                 /* enable statistics collection only on all ports */
1917                 writel_relaxed(0x7, &cpsw->regs->stat_port_en);
1918
1919                 /* Enable internal fifo flow control */
1920                 writel(0x7, &cpsw->regs->flow_control);
1921
1922                 napi_enable(&cpsw->napi_rx);
1923                 napi_enable(&cpsw->napi_tx);
1924
1925                 if (cpsw->tx_irq_disabled) {
1926                         cpsw->tx_irq_disabled = false;
1927                         enable_irq(cpsw->irqs_table[1]);
1928                 }
1929
1930                 if (cpsw->rx_irq_disabled) {
1931                         cpsw->rx_irq_disabled = false;
1932                         enable_irq(cpsw->irqs_table[0]);
1933                 }
1934
1935                 ret = cpsw_fill_rx_channels(priv);
1936                 if (ret < 0)
1937                         goto err_cleanup;
1938
1939                 if (cpts_register(cpsw->cpts))
1940                         dev_err(priv->dev, "error registering cpts device\n");
1941
1942         }
1943
1944         cpsw_restore(priv);
1945
1946         /* Enable Interrupt pacing if configured */
1947         if (cpsw->coal_intvl != 0) {
1948                 struct ethtool_coalesce coal;
1949
1950                 coal.rx_coalesce_usecs = cpsw->coal_intvl;
1951                 cpsw_set_coalesce(ndev, &coal);
1952         }
1953
1954         cpdma_ctlr_start(cpsw->dma);
1955         cpsw_intr_enable(cpsw);
1956         cpsw->usage_count++;
1957
1958         return 0;
1959
1960 err_cleanup:
1961         cpdma_ctlr_stop(cpsw->dma);
1962         for_each_slave(priv, cpsw_slave_stop, cpsw);
1963         pm_runtime_put_sync(cpsw->dev);
1964         netif_carrier_off(priv->ndev);
1965         return ret;
1966 }
1967
1968 static int cpsw_ndo_stop(struct net_device *ndev)
1969 {
1970         struct cpsw_priv *priv = netdev_priv(ndev);
1971         struct cpsw_common *cpsw = priv->cpsw;
1972
1973         cpsw_info(priv, ifdown, "shutting down cpsw device\n");
1974         __dev_mc_unsync(priv->ndev, cpsw_del_mc_addr);
1975         netif_tx_stop_all_queues(priv->ndev);
1976         netif_carrier_off(priv->ndev);
1977
1978         if (cpsw->usage_count <= 1) {
1979                 napi_disable(&cpsw->napi_rx);
1980                 napi_disable(&cpsw->napi_tx);
1981                 cpts_unregister(cpsw->cpts);
1982                 cpsw_intr_disable(cpsw);
1983                 cpdma_ctlr_stop(cpsw->dma);
1984                 cpsw_ale_stop(cpsw->ale);
1985         }
1986         for_each_slave(priv, cpsw_slave_stop, cpsw);
1987
1988         if (cpsw_need_resplit(cpsw))
1989                 cpsw_split_res(ndev);
1990
1991         cpsw->usage_count--;
1992         pm_runtime_put_sync(cpsw->dev);
1993         return 0;
1994 }
1995
1996 static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
1997                                        struct net_device *ndev)
1998 {
1999         struct cpsw_priv *priv = netdev_priv(ndev);
2000         struct cpsw_common *cpsw = priv->cpsw;
2001         struct cpts *cpts = cpsw->cpts;
2002         struct netdev_queue *txq;
2003         struct cpdma_chan *txch;
2004         int ret, q_idx;
2005
2006         if (skb_padto(skb, CPSW_MIN_PACKET_SIZE)) {
2007                 cpsw_err(priv, tx_err, "packet pad failed\n");
2008                 ndev->stats.tx_dropped++;
2009                 return NET_XMIT_DROP;
2010         }
2011
2012         if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
2013             cpts_is_tx_enabled(cpts) && cpts_can_timestamp(cpts, skb))
2014                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2015
2016         q_idx = skb_get_queue_mapping(skb);
2017         if (q_idx >= cpsw->tx_ch_num)
2018                 q_idx = q_idx % cpsw->tx_ch_num;
2019
2020         txch = cpsw->txv[q_idx].ch;
2021         txq = netdev_get_tx_queue(ndev, q_idx);
2022         ret = cpsw_tx_packet_submit(priv, skb, txch);
2023         if (unlikely(ret != 0)) {
2024                 cpsw_err(priv, tx_err, "desc submit failed\n");
2025                 goto fail;
2026         }
2027
2028         /* If there is no more tx desc left free then we need to
2029          * tell the kernel to stop sending us tx frames.
2030          */
2031         if (unlikely(!cpdma_check_free_tx_desc(txch))) {
2032                 netif_tx_stop_queue(txq);
2033
2034                 /* Barrier, so that stop_queue visible to other cpus */
2035                 smp_mb__after_atomic();
2036
2037                 if (cpdma_check_free_tx_desc(txch))
2038                         netif_tx_wake_queue(txq);
2039         }
2040
2041         return NETDEV_TX_OK;
2042 fail:
2043         ndev->stats.tx_dropped++;
2044         netif_tx_stop_queue(txq);
2045
2046         /* Barrier, so that stop_queue visible to other cpus */
2047         smp_mb__after_atomic();
2048
2049         if (cpdma_check_free_tx_desc(txch))
2050                 netif_tx_wake_queue(txq);
2051
2052         return NETDEV_TX_BUSY;
2053 }
2054
2055 #if IS_ENABLED(CONFIG_TI_CPTS)
2056
2057 static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
2058 {
2059         struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
2060         u32 ts_en, seq_id;
2061
2062         if (!cpts_is_tx_enabled(cpsw->cpts) &&
2063             !cpts_is_rx_enabled(cpsw->cpts)) {
2064                 slave_write(slave, 0, CPSW1_TS_CTL);
2065                 return;
2066         }
2067
2068         seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
2069         ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
2070
2071         if (cpts_is_tx_enabled(cpsw->cpts))
2072                 ts_en |= CPSW_V1_TS_TX_EN;
2073
2074         if (cpts_is_rx_enabled(cpsw->cpts))
2075                 ts_en |= CPSW_V1_TS_RX_EN;
2076
2077         slave_write(slave, ts_en, CPSW1_TS_CTL);
2078         slave_write(slave, seq_id, CPSW1_TS_SEQ_LTYPE);
2079 }
2080
2081 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
2082 {
2083         struct cpsw_slave *slave;
2084         struct cpsw_common *cpsw = priv->cpsw;
2085         u32 ctrl, mtype;
2086
2087         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
2088
2089         ctrl = slave_read(slave, CPSW2_CONTROL);
2090         switch (cpsw->version) {
2091         case CPSW_VERSION_2:
2092                 ctrl &= ~CTRL_V2_ALL_TS_MASK;
2093
2094                 if (cpts_is_tx_enabled(cpsw->cpts))
2095                         ctrl |= CTRL_V2_TX_TS_BITS;
2096
2097                 if (cpts_is_rx_enabled(cpsw->cpts))
2098                         ctrl |= CTRL_V2_RX_TS_BITS;
2099                 break;
2100         case CPSW_VERSION_3:
2101         default:
2102                 ctrl &= ~CTRL_V3_ALL_TS_MASK;
2103
2104                 if (cpts_is_tx_enabled(cpsw->cpts))
2105                         ctrl |= CTRL_V3_TX_TS_BITS;
2106
2107                 if (cpts_is_rx_enabled(cpsw->cpts))
2108                         ctrl |= CTRL_V3_RX_TS_BITS;
2109                 break;
2110         }
2111
2112         mtype = (30 << TS_SEQ_ID_OFFSET_SHIFT) | EVENT_MSG_BITS;
2113
2114         slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
2115         slave_write(slave, ctrl, CPSW2_CONTROL);
2116         writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);
2117 }
2118
2119 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
2120 {
2121         struct cpsw_priv *priv = netdev_priv(dev);
2122         struct hwtstamp_config cfg;
2123         struct cpsw_common *cpsw = priv->cpsw;
2124         struct cpts *cpts = cpsw->cpts;
2125
2126         if (cpsw->version != CPSW_VERSION_1 &&
2127             cpsw->version != CPSW_VERSION_2 &&
2128             cpsw->version != CPSW_VERSION_3)
2129                 return -EOPNOTSUPP;
2130
2131         if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
2132                 return -EFAULT;
2133
2134         /* reserved for future extensions */
2135         if (cfg.flags)
2136                 return -EINVAL;
2137
2138         if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
2139                 return -ERANGE;
2140
2141         switch (cfg.rx_filter) {
2142         case HWTSTAMP_FILTER_NONE:
2143                 cpts_rx_enable(cpts, 0);
2144                 break;
2145         case HWTSTAMP_FILTER_ALL:
2146         case HWTSTAMP_FILTER_NTP_ALL:
2147                 return -ERANGE;
2148         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2149         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2150         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2151                 cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V1_L4_EVENT);
2152                 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
2153                 break;
2154         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2155         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2156         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2157         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2158         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2159         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2160         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2161         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2162         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2163                 cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V2_EVENT);
2164                 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
2165                 break;
2166         default:
2167                 return -ERANGE;
2168         }
2169
2170         cpts_tx_enable(cpts, cfg.tx_type == HWTSTAMP_TX_ON);
2171
2172         switch (cpsw->version) {
2173         case CPSW_VERSION_1:
2174                 cpsw_hwtstamp_v1(cpsw);
2175                 break;
2176         case CPSW_VERSION_2:
2177         case CPSW_VERSION_3:
2178                 cpsw_hwtstamp_v2(priv);
2179                 break;
2180         default:
2181                 WARN_ON(1);
2182         }
2183
2184         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
2185 }
2186
2187 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
2188 {
2189         struct cpsw_common *cpsw = ndev_to_cpsw(dev);
2190         struct cpts *cpts = cpsw->cpts;
2191         struct hwtstamp_config cfg;
2192
2193         if (cpsw->version != CPSW_VERSION_1 &&
2194             cpsw->version != CPSW_VERSION_2 &&
2195             cpsw->version != CPSW_VERSION_3)
2196                 return -EOPNOTSUPP;
2197
2198         cfg.flags = 0;
2199         cfg.tx_type = cpts_is_tx_enabled(cpts) ?
2200                       HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
2201         cfg.rx_filter = (cpts_is_rx_enabled(cpts) ?
2202                          cpts->rx_enable : HWTSTAMP_FILTER_NONE);
2203
2204         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
2205 }
2206 #else
2207 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
2208 {
2209         return -EOPNOTSUPP;
2210 }
2211
2212 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
2213 {
2214         return -EOPNOTSUPP;
2215 }
2216 #endif /*CONFIG_TI_CPTS*/
2217
2218 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2219 {
2220         struct cpsw_priv *priv = netdev_priv(dev);
2221         struct cpsw_common *cpsw = priv->cpsw;
2222         int slave_no = cpsw_slave_index(cpsw, priv);
2223
2224         if (!netif_running(dev))
2225                 return -EINVAL;
2226
2227         switch (cmd) {
2228         case SIOCSHWTSTAMP:
2229                 return cpsw_hwtstamp_set(dev, req);
2230         case SIOCGHWTSTAMP:
2231                 return cpsw_hwtstamp_get(dev, req);
2232         }
2233
2234         if (!cpsw->slaves[slave_no].phy)
2235                 return -EOPNOTSUPP;
2236         return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
2237 }
2238
2239 static void cpsw_ndo_tx_timeout(struct net_device *ndev)
2240 {
2241         struct cpsw_priv *priv = netdev_priv(ndev);
2242         struct cpsw_common *cpsw = priv->cpsw;
2243         int ch;
2244
2245         cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
2246         ndev->stats.tx_errors++;
2247         cpsw_intr_disable(cpsw);
2248         for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
2249                 cpdma_chan_stop(cpsw->txv[ch].ch);
2250                 cpdma_chan_start(cpsw->txv[ch].ch);
2251         }
2252
2253         cpsw_intr_enable(cpsw);
2254         netif_trans_update(ndev);
2255         netif_tx_wake_all_queues(ndev);
2256 }
2257
2258 static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
2259 {
2260         struct cpsw_priv *priv = netdev_priv(ndev);
2261         struct sockaddr *addr = (struct sockaddr *)p;
2262         struct cpsw_common *cpsw = priv->cpsw;
2263         int flags = 0;
2264         u16 vid = 0;
2265         int ret;
2266
2267         if (!is_valid_ether_addr(addr->sa_data))
2268                 return -EADDRNOTAVAIL;
2269
2270         ret = pm_runtime_get_sync(cpsw->dev);
2271         if (ret < 0) {
2272                 pm_runtime_put_noidle(cpsw->dev);
2273                 return ret;
2274         }
2275
2276         if (cpsw->data.dual_emac) {
2277                 vid = cpsw->slaves[priv->emac_port].port_vlan;
2278                 flags = ALE_VLAN;
2279         }
2280
2281         cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
2282                            flags, vid);
2283         cpsw_ale_add_ucast(cpsw->ale, addr->sa_data, HOST_PORT_NUM,
2284                            flags, vid);
2285
2286         memcpy(priv->mac_addr, addr->sa_data, ETH_ALEN);
2287         memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
2288         for_each_slave(priv, cpsw_set_slave_mac, priv);
2289
2290         pm_runtime_put(cpsw->dev);
2291
2292         return 0;
2293 }
2294
2295 #ifdef CONFIG_NET_POLL_CONTROLLER
2296 static void cpsw_ndo_poll_controller(struct net_device *ndev)
2297 {
2298         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2299
2300         cpsw_intr_disable(cpsw);
2301         cpsw_rx_interrupt(cpsw->irqs_table[0], cpsw);
2302         cpsw_tx_interrupt(cpsw->irqs_table[1], cpsw);
2303         cpsw_intr_enable(cpsw);
2304 }
2305 #endif
2306
2307 static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
2308                                 unsigned short vid)
2309 {
2310         int ret;
2311         int unreg_mcast_mask = 0;
2312         int mcast_mask;
2313         u32 port_mask;
2314         struct cpsw_common *cpsw = priv->cpsw;
2315
2316         if (cpsw->data.dual_emac) {
2317                 port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
2318
2319                 mcast_mask = ALE_PORT_HOST;
2320                 if (priv->ndev->flags & IFF_ALLMULTI)
2321                         unreg_mcast_mask = mcast_mask;
2322         } else {
2323                 port_mask = ALE_ALL_PORTS;
2324                 mcast_mask = port_mask;
2325
2326                 if (priv->ndev->flags & IFF_ALLMULTI)
2327                         unreg_mcast_mask = ALE_ALL_PORTS;
2328                 else
2329                         unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
2330         }
2331
2332         ret = cpsw_ale_add_vlan(cpsw->ale, vid, port_mask, 0, port_mask,
2333                                 unreg_mcast_mask);
2334         if (ret != 0)
2335                 return ret;
2336
2337         ret = cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
2338                                  HOST_PORT_NUM, ALE_VLAN, vid);
2339         if (ret != 0)
2340                 goto clean_vid;
2341
2342         ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
2343                                  mcast_mask, ALE_VLAN, vid, 0);
2344         if (ret != 0)
2345                 goto clean_vlan_ucast;
2346         return 0;
2347
2348 clean_vlan_ucast:
2349         cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2350                            HOST_PORT_NUM, ALE_VLAN, vid);
2351 clean_vid:
2352         cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2353         return ret;
2354 }
2355
2356 static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
2357                                     __be16 proto, u16 vid)
2358 {
2359         struct cpsw_priv *priv = netdev_priv(ndev);
2360         struct cpsw_common *cpsw = priv->cpsw;
2361         int ret;
2362
2363         if (vid == cpsw->data.default_vlan)
2364                 return 0;
2365
2366         ret = pm_runtime_get_sync(cpsw->dev);
2367         if (ret < 0) {
2368                 pm_runtime_put_noidle(cpsw->dev);
2369                 return ret;
2370         }
2371
2372         if (cpsw->data.dual_emac) {
2373                 /* In dual EMAC, reserved VLAN id should not be used for
2374                  * creating VLAN interfaces as this can break the dual
2375                  * EMAC port separation
2376                  */
2377                 int i;
2378
2379                 for (i = 0; i < cpsw->data.slaves; i++) {
2380                         if (vid == cpsw->slaves[i].port_vlan) {
2381                                 ret = -EINVAL;
2382                                 goto err;
2383                         }
2384                 }
2385         }
2386
2387         dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
2388         ret = cpsw_add_vlan_ale_entry(priv, vid);
2389 err:
2390         pm_runtime_put(cpsw->dev);
2391         return ret;
2392 }
2393
2394 static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
2395                                      __be16 proto, u16 vid)
2396 {
2397         struct cpsw_priv *priv = netdev_priv(ndev);
2398         struct cpsw_common *cpsw = priv->cpsw;
2399         int ret;
2400
2401         if (vid == cpsw->data.default_vlan)
2402                 return 0;
2403
2404         ret = pm_runtime_get_sync(cpsw->dev);
2405         if (ret < 0) {
2406                 pm_runtime_put_noidle(cpsw->dev);
2407                 return ret;
2408         }
2409
2410         if (cpsw->data.dual_emac) {
2411                 int i;
2412
2413                 for (i = 0; i < cpsw->data.slaves; i++) {
2414                         if (vid == cpsw->slaves[i].port_vlan)
2415                                 goto err;
2416                 }
2417         }
2418
2419         dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid);
2420         ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2421         ret |= cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2422                                   HOST_PORT_NUM, ALE_VLAN, vid);
2423         ret |= cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast,
2424                                   0, ALE_VLAN, vid);
2425 err:
2426         pm_runtime_put(cpsw->dev);
2427         return ret;
2428 }
2429
2430 static int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate)
2431 {
2432         struct cpsw_priv *priv = netdev_priv(ndev);
2433         struct cpsw_common *cpsw = priv->cpsw;
2434         struct cpsw_slave *slave;
2435         u32 min_rate;
2436         u32 ch_rate;
2437         int i, ret;
2438
2439         ch_rate = netdev_get_tx_queue(ndev, queue)->tx_maxrate;
2440         if (ch_rate == rate)
2441                 return 0;
2442
2443         ch_rate = rate * 1000;
2444         min_rate = cpdma_chan_get_min_rate(cpsw->dma);
2445         if ((ch_rate < min_rate && ch_rate)) {
2446                 dev_err(priv->dev, "The channel rate cannot be less than %dMbps",
2447                         min_rate);
2448                 return -EINVAL;
2449         }
2450
2451         if (rate > cpsw->speed) {
2452                 dev_err(priv->dev, "The channel rate cannot be more than 2Gbps");
2453                 return -EINVAL;
2454         }
2455
2456         ret = pm_runtime_get_sync(cpsw->dev);
2457         if (ret < 0) {
2458                 pm_runtime_put_noidle(cpsw->dev);
2459                 return ret;
2460         }
2461
2462         ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);
2463         pm_runtime_put(cpsw->dev);
2464
2465         if (ret)
2466                 return ret;
2467
2468         /* update rates for slaves tx queues */
2469         for (i = 0; i < cpsw->data.slaves; i++) {
2470                 slave = &cpsw->slaves[i];
2471                 if (!slave->ndev)
2472                         continue;
2473
2474                 netdev_get_tx_queue(slave->ndev, queue)->tx_maxrate = rate;
2475         }
2476
2477         cpsw_split_res(ndev);
2478         return ret;
2479 }
2480
2481 static int cpsw_set_mqprio(struct net_device *ndev, void *type_data)
2482 {
2483         struct tc_mqprio_qopt_offload *mqprio = type_data;
2484         struct cpsw_priv *priv = netdev_priv(ndev);
2485         struct cpsw_common *cpsw = priv->cpsw;
2486         int fifo, num_tc, count, offset;
2487         struct cpsw_slave *slave;
2488         u32 tx_prio_map = 0;
2489         int i, tc, ret;
2490
2491         num_tc = mqprio->qopt.num_tc;
2492         if (num_tc > CPSW_TC_NUM)
2493                 return -EINVAL;
2494
2495         if (mqprio->mode != TC_MQPRIO_MODE_DCB)
2496                 return -EINVAL;
2497
2498         ret = pm_runtime_get_sync(cpsw->dev);
2499         if (ret < 0) {
2500                 pm_runtime_put_noidle(cpsw->dev);
2501                 return ret;
2502         }
2503
2504         if (num_tc) {
2505                 for (i = 0; i < 8; i++) {
2506                         tc = mqprio->qopt.prio_tc_map[i];
2507                         fifo = cpsw_tc_to_fifo(tc, num_tc);
2508                         tx_prio_map |= fifo << (4 * i);
2509                 }
2510
2511                 netdev_set_num_tc(ndev, num_tc);
2512                 for (i = 0; i < num_tc; i++) {
2513                         count = mqprio->qopt.count[i];
2514                         offset = mqprio->qopt.offset[i];
2515                         netdev_set_tc_queue(ndev, i, count, offset);
2516                 }
2517         }
2518
2519         if (!mqprio->qopt.hw) {
2520                 /* restore default configuration */
2521                 netdev_reset_tc(ndev);
2522                 tx_prio_map = TX_PRIORITY_MAPPING;
2523         }
2524
2525         priv->mqprio_hw = mqprio->qopt.hw;
2526
2527         offset = cpsw->version == CPSW_VERSION_1 ?
2528                  CPSW1_TX_PRI_MAP : CPSW2_TX_PRI_MAP;
2529
2530         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
2531         slave_write(slave, tx_prio_map, offset);
2532
2533         pm_runtime_put_sync(cpsw->dev);
2534
2535         return 0;
2536 }
2537
2538 static int cpsw_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type,
2539                              void *type_data)
2540 {
2541         switch (type) {
2542         case TC_SETUP_QDISC_CBS:
2543                 return cpsw_set_cbs(ndev, type_data);
2544
2545         case TC_SETUP_QDISC_MQPRIO:
2546                 return cpsw_set_mqprio(ndev, type_data);
2547
2548         default:
2549                 return -EOPNOTSUPP;
2550         }
2551 }
2552
2553 static const struct net_device_ops cpsw_netdev_ops = {
2554         .ndo_open               = cpsw_ndo_open,
2555         .ndo_stop               = cpsw_ndo_stop,
2556         .ndo_start_xmit         = cpsw_ndo_start_xmit,
2557         .ndo_set_mac_address    = cpsw_ndo_set_mac_address,
2558         .ndo_do_ioctl           = cpsw_ndo_ioctl,
2559         .ndo_validate_addr      = eth_validate_addr,
2560         .ndo_tx_timeout         = cpsw_ndo_tx_timeout,
2561         .ndo_set_rx_mode        = cpsw_ndo_set_rx_mode,
2562         .ndo_set_tx_maxrate     = cpsw_ndo_set_tx_maxrate,
2563 #ifdef CONFIG_NET_POLL_CONTROLLER
2564         .ndo_poll_controller    = cpsw_ndo_poll_controller,
2565 #endif
2566         .ndo_vlan_rx_add_vid    = cpsw_ndo_vlan_rx_add_vid,
2567         .ndo_vlan_rx_kill_vid   = cpsw_ndo_vlan_rx_kill_vid,
2568         .ndo_setup_tc           = cpsw_ndo_setup_tc,
2569 };
2570
2571 static int cpsw_get_regs_len(struct net_device *ndev)
2572 {
2573         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2574
2575         return cpsw->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
2576 }
2577
2578 static void cpsw_get_regs(struct net_device *ndev,
2579                           struct ethtool_regs *regs, void *p)
2580 {
2581         u32 *reg = p;
2582         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2583
2584         /* update CPSW IP version */
2585         regs->version = cpsw->version;
2586
2587         cpsw_ale_dump(cpsw->ale, reg);
2588 }
2589
2590 static void cpsw_get_drvinfo(struct net_device *ndev,
2591                              struct ethtool_drvinfo *info)
2592 {
2593         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2594         struct platform_device  *pdev = to_platform_device(cpsw->dev);
2595
2596         strlcpy(info->driver, "cpsw", sizeof(info->driver));
2597         strlcpy(info->version, "1.0", sizeof(info->version));
2598         strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
2599 }
2600
2601 static u32 cpsw_get_msglevel(struct net_device *ndev)
2602 {
2603         struct cpsw_priv *priv = netdev_priv(ndev);
2604         return priv->msg_enable;
2605 }
2606
2607 static void cpsw_set_msglevel(struct net_device *ndev, u32 value)
2608 {
2609         struct cpsw_priv *priv = netdev_priv(ndev);
2610         priv->msg_enable = value;
2611 }
2612
2613 #if IS_ENABLED(CONFIG_TI_CPTS)
2614 static int cpsw_get_ts_info(struct net_device *ndev,
2615                             struct ethtool_ts_info *info)
2616 {
2617         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2618
2619         info->so_timestamping =
2620                 SOF_TIMESTAMPING_TX_HARDWARE |
2621                 SOF_TIMESTAMPING_TX_SOFTWARE |
2622                 SOF_TIMESTAMPING_RX_HARDWARE |
2623                 SOF_TIMESTAMPING_RX_SOFTWARE |
2624                 SOF_TIMESTAMPING_SOFTWARE |
2625                 SOF_TIMESTAMPING_RAW_HARDWARE;
2626         info->phc_index = cpsw->cpts->phc_index;
2627         info->tx_types =
2628                 (1 << HWTSTAMP_TX_OFF) |
2629                 (1 << HWTSTAMP_TX_ON);
2630         info->rx_filters =
2631                 (1 << HWTSTAMP_FILTER_NONE) |
2632                 (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
2633                 (1 << HWTSTAMP_FILTER_PTP_V2_EVENT);
2634         return 0;
2635 }
2636 #else
2637 static int cpsw_get_ts_info(struct net_device *ndev,
2638                             struct ethtool_ts_info *info)
2639 {
2640         info->so_timestamping =
2641                 SOF_TIMESTAMPING_TX_SOFTWARE |
2642                 SOF_TIMESTAMPING_RX_SOFTWARE |
2643                 SOF_TIMESTAMPING_SOFTWARE;
2644         info->phc_index = -1;
2645         info->tx_types = 0;
2646         info->rx_filters = 0;
2647         return 0;
2648 }
2649 #endif
2650
2651 static int cpsw_get_link_ksettings(struct net_device *ndev,
2652                                    struct ethtool_link_ksettings *ecmd)
2653 {
2654         struct cpsw_priv *priv = netdev_priv(ndev);
2655         struct cpsw_common *cpsw = priv->cpsw;
2656         int slave_no = cpsw_slave_index(cpsw, priv);
2657
2658         if (!cpsw->slaves[slave_no].phy)
2659                 return -EOPNOTSUPP;
2660
2661         phy_ethtool_ksettings_get(cpsw->slaves[slave_no].phy, ecmd);
2662         return 0;
2663 }
2664
2665 static int cpsw_set_link_ksettings(struct net_device *ndev,
2666                                    const struct ethtool_link_ksettings *ecmd)
2667 {
2668         struct cpsw_priv *priv = netdev_priv(ndev);
2669         struct cpsw_common *cpsw = priv->cpsw;
2670         int slave_no = cpsw_slave_index(cpsw, priv);
2671
2672         if (cpsw->slaves[slave_no].phy)
2673                 return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy,
2674                                                  ecmd);
2675         else
2676                 return -EOPNOTSUPP;
2677 }
2678
2679 static void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2680 {
2681         struct cpsw_priv *priv = netdev_priv(ndev);
2682         struct cpsw_common *cpsw = priv->cpsw;
2683         int slave_no = cpsw_slave_index(cpsw, priv);
2684
2685         wol->supported = 0;
2686         wol->wolopts = 0;
2687
2688         if (cpsw->slaves[slave_no].phy)
2689                 phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol);
2690 }
2691
2692 static int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2693 {
2694         struct cpsw_priv *priv = netdev_priv(ndev);
2695         struct cpsw_common *cpsw = priv->cpsw;
2696         int slave_no = cpsw_slave_index(cpsw, priv);
2697
2698         if (cpsw->slaves[slave_no].phy)
2699                 return phy_ethtool_set_wol(cpsw->slaves[slave_no].phy, wol);
2700         else
2701                 return -EOPNOTSUPP;
2702 }
2703
2704 static void cpsw_get_pauseparam(struct net_device *ndev,
2705                                 struct ethtool_pauseparam *pause)
2706 {
2707         struct cpsw_priv *priv = netdev_priv(ndev);
2708
2709         pause->autoneg = AUTONEG_DISABLE;
2710         pause->rx_pause = priv->rx_pause ? true : false;
2711         pause->tx_pause = priv->tx_pause ? true : false;
2712 }
2713
2714 static int cpsw_set_pauseparam(struct net_device *ndev,
2715                                struct ethtool_pauseparam *pause)
2716 {
2717         struct cpsw_priv *priv = netdev_priv(ndev);
2718         bool link;
2719
2720         priv->rx_pause = pause->rx_pause ? true : false;
2721         priv->tx_pause = pause->tx_pause ? true : false;
2722
2723         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
2724         return 0;
2725 }
2726
2727 static int cpsw_ethtool_op_begin(struct net_device *ndev)
2728 {
2729         struct cpsw_priv *priv = netdev_priv(ndev);
2730         struct cpsw_common *cpsw = priv->cpsw;
2731         int ret;
2732
2733         ret = pm_runtime_get_sync(cpsw->dev);
2734         if (ret < 0) {
2735                 cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
2736                 pm_runtime_put_noidle(cpsw->dev);
2737         }
2738
2739         return ret;
2740 }
2741
2742 static void cpsw_ethtool_op_complete(struct net_device *ndev)
2743 {
2744         struct cpsw_priv *priv = netdev_priv(ndev);
2745         int ret;
2746
2747         ret = pm_runtime_put(priv->cpsw->dev);
2748         if (ret < 0)
2749                 cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
2750 }
2751
2752 static void cpsw_get_channels(struct net_device *ndev,
2753                               struct ethtool_channels *ch)
2754 {
2755         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2756
2757         ch->max_rx = cpsw->quirk_irq ? 1 : CPSW_MAX_QUEUES;
2758         ch->max_tx = cpsw->quirk_irq ? 1 : CPSW_MAX_QUEUES;
2759         ch->max_combined = 0;
2760         ch->max_other = 0;
2761         ch->other_count = 0;
2762         ch->rx_count = cpsw->rx_ch_num;
2763         ch->tx_count = cpsw->tx_ch_num;
2764         ch->combined_count = 0;
2765 }
2766
2767 static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
2768                                   struct ethtool_channels *ch)
2769 {
2770         if (cpsw->quirk_irq) {
2771                 dev_err(cpsw->dev, "Maximum one tx/rx queue is allowed");
2772                 return -EOPNOTSUPP;
2773         }
2774
2775         if (ch->combined_count)
2776                 return -EINVAL;
2777
2778         /* verify we have at least one channel in each direction */
2779         if (!ch->rx_count || !ch->tx_count)
2780                 return -EINVAL;
2781
2782         if (ch->rx_count > cpsw->data.channels ||
2783             ch->tx_count > cpsw->data.channels)
2784                 return -EINVAL;
2785
2786         return 0;
2787 }
2788
2789 static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
2790 {
2791         struct cpsw_common *cpsw = priv->cpsw;
2792         void (*handler)(void *, int, int);
2793         struct netdev_queue *queue;
2794         struct cpsw_vector *vec;
2795         int ret, *ch, vch;
2796
2797         if (rx) {
2798                 ch = &cpsw->rx_ch_num;
2799                 vec = cpsw->rxv;
2800                 handler = cpsw_rx_handler;
2801         } else {
2802                 ch = &cpsw->tx_ch_num;
2803                 vec = cpsw->txv;
2804                 handler = cpsw_tx_handler;
2805         }
2806
2807         while (*ch < ch_num) {
2808                 vch = rx ? *ch : 7 - *ch;
2809                 vec[*ch].ch = cpdma_chan_create(cpsw->dma, vch, handler, rx);
2810                 queue = netdev_get_tx_queue(priv->ndev, *ch);
2811                 queue->tx_maxrate = 0;
2812
2813                 if (IS_ERR(vec[*ch].ch))
2814                         return PTR_ERR(vec[*ch].ch);
2815
2816                 if (!vec[*ch].ch)
2817                         return -EINVAL;
2818
2819                 cpsw_info(priv, ifup, "created new %d %s channel\n", *ch,
2820                           (rx ? "rx" : "tx"));
2821                 (*ch)++;
2822         }
2823
2824         while (*ch > ch_num) {
2825                 (*ch)--;
2826
2827                 ret = cpdma_chan_destroy(vec[*ch].ch);
2828                 if (ret)
2829                         return ret;
2830
2831                 cpsw_info(priv, ifup, "destroyed %d %s channel\n", *ch,
2832                           (rx ? "rx" : "tx"));
2833         }
2834
2835         return 0;
2836 }
2837
2838 static int cpsw_update_channels(struct cpsw_priv *priv,
2839                                 struct ethtool_channels *ch)
2840 {
2841         int ret;
2842
2843         ret = cpsw_update_channels_res(priv, ch->rx_count, 1);
2844         if (ret)
2845                 return ret;
2846
2847         ret = cpsw_update_channels_res(priv, ch->tx_count, 0);
2848         if (ret)
2849                 return ret;
2850
2851         return 0;
2852 }
2853
2854 static void cpsw_suspend_data_pass(struct net_device *ndev)
2855 {
2856         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2857         struct cpsw_slave *slave;
2858         int i;
2859
2860         /* Disable NAPI scheduling */
2861         cpsw_intr_disable(cpsw);
2862
2863         /* Stop all transmit queues for every network device.
2864          * Disable re-using rx descriptors with dormant_on.
2865          */
2866         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2867                 if (!(slave->ndev && netif_running(slave->ndev)))
2868                         continue;
2869
2870                 netif_tx_stop_all_queues(slave->ndev);
2871                 netif_dormant_on(slave->ndev);
2872         }
2873
2874         /* Handle rest of tx packets and stop cpdma channels */
2875         cpdma_ctlr_stop(cpsw->dma);
2876 }
2877
2878 static int cpsw_resume_data_pass(struct net_device *ndev)
2879 {
2880         struct cpsw_priv *priv = netdev_priv(ndev);
2881         struct cpsw_common *cpsw = priv->cpsw;
2882         struct cpsw_slave *slave;
2883         int i, ret;
2884
2885         /* Allow rx packets handling */
2886         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2887                 if (slave->ndev && netif_running(slave->ndev))
2888                         netif_dormant_off(slave->ndev);
2889
2890         /* After this receive is started */
2891         if (cpsw->usage_count) {
2892                 ret = cpsw_fill_rx_channels(priv);
2893                 if (ret)
2894                         return ret;
2895
2896                 cpdma_ctlr_start(cpsw->dma);
2897                 cpsw_intr_enable(cpsw);
2898         }
2899
2900         /* Resume transmit for every affected interface */
2901         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2902                 if (slave->ndev && netif_running(slave->ndev))
2903                         netif_tx_start_all_queues(slave->ndev);
2904
2905         return 0;
2906 }
2907
2908 static int cpsw_set_channels(struct net_device *ndev,
2909                              struct ethtool_channels *chs)
2910 {
2911         struct cpsw_priv *priv = netdev_priv(ndev);
2912         struct cpsw_common *cpsw = priv->cpsw;
2913         struct cpsw_slave *slave;
2914         int i, ret;
2915
2916         ret = cpsw_check_ch_settings(cpsw, chs);
2917         if (ret < 0)
2918                 return ret;
2919
2920         cpsw_suspend_data_pass(ndev);
2921         ret = cpsw_update_channels(priv, chs);
2922         if (ret)
2923                 goto err;
2924
2925         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2926                 if (!(slave->ndev && netif_running(slave->ndev)))
2927                         continue;
2928
2929                 /* Inform stack about new count of queues */
2930                 ret = netif_set_real_num_tx_queues(slave->ndev,
2931                                                    cpsw->tx_ch_num);
2932                 if (ret) {
2933                         dev_err(priv->dev, "cannot set real number of tx queues\n");
2934                         goto err;
2935                 }
2936
2937                 ret = netif_set_real_num_rx_queues(slave->ndev,
2938                                                    cpsw->rx_ch_num);
2939                 if (ret) {
2940                         dev_err(priv->dev, "cannot set real number of rx queues\n");
2941                         goto err;
2942                 }
2943         }
2944
2945         if (cpsw->usage_count)
2946                 cpsw_split_res(ndev);
2947
2948         ret = cpsw_resume_data_pass(ndev);
2949         if (!ret)
2950                 return 0;
2951 err:
2952         dev_err(priv->dev, "cannot update channels number, closing device\n");
2953         dev_close(ndev);
2954         return ret;
2955 }
2956
2957 static int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
2958 {
2959         struct cpsw_priv *priv = netdev_priv(ndev);
2960         struct cpsw_common *cpsw = priv->cpsw;
2961         int slave_no = cpsw_slave_index(cpsw, priv);
2962
2963         if (cpsw->slaves[slave_no].phy)
2964                 return phy_ethtool_get_eee(cpsw->slaves[slave_no].phy, edata);
2965         else
2966                 return -EOPNOTSUPP;
2967 }
2968
2969 static int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
2970 {
2971         struct cpsw_priv *priv = netdev_priv(ndev);
2972         struct cpsw_common *cpsw = priv->cpsw;
2973         int slave_no = cpsw_slave_index(cpsw, priv);
2974
2975         if (cpsw->slaves[slave_no].phy)
2976                 return phy_ethtool_set_eee(cpsw->slaves[slave_no].phy, edata);
2977         else
2978                 return -EOPNOTSUPP;
2979 }
2980
2981 static int cpsw_nway_reset(struct net_device *ndev)
2982 {
2983         struct cpsw_priv *priv = netdev_priv(ndev);
2984         struct cpsw_common *cpsw = priv->cpsw;
2985         int slave_no = cpsw_slave_index(cpsw, priv);
2986
2987         if (cpsw->slaves[slave_no].phy)
2988                 return genphy_restart_aneg(cpsw->slaves[slave_no].phy);
2989         else
2990                 return -EOPNOTSUPP;
2991 }
2992
2993 static void cpsw_get_ringparam(struct net_device *ndev,
2994                                struct ethtool_ringparam *ering)
2995 {
2996         struct cpsw_priv *priv = netdev_priv(ndev);
2997         struct cpsw_common *cpsw = priv->cpsw;
2998
2999         /* not supported */
3000         ering->tx_max_pending = 0;
3001         ering->tx_pending = cpdma_get_num_tx_descs(cpsw->dma);
3002         ering->rx_max_pending = descs_pool_size - CPSW_MAX_QUEUES;
3003         ering->rx_pending = cpdma_get_num_rx_descs(cpsw->dma);
3004 }
3005
3006 static int cpsw_set_ringparam(struct net_device *ndev,
3007                               struct ethtool_ringparam *ering)
3008 {
3009         struct cpsw_priv *priv = netdev_priv(ndev);
3010         struct cpsw_common *cpsw = priv->cpsw;
3011         int ret;
3012
3013         /* ignore ering->tx_pending - only rx_pending adjustment is supported */
3014
3015         if (ering->rx_mini_pending || ering->rx_jumbo_pending ||
3016             ering->rx_pending < CPSW_MAX_QUEUES ||
3017             ering->rx_pending > (descs_pool_size - CPSW_MAX_QUEUES))
3018                 return -EINVAL;
3019
3020         if (ering->rx_pending == cpdma_get_num_rx_descs(cpsw->dma))
3021                 return 0;
3022
3023         cpsw_suspend_data_pass(ndev);
3024
3025         cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
3026
3027         if (cpsw->usage_count)
3028                 cpdma_chan_split_pool(cpsw->dma);
3029
3030         ret = cpsw_resume_data_pass(ndev);
3031         if (!ret)
3032                 return 0;
3033
3034         dev_err(&ndev->dev, "cannot set ring params, closing device\n");
3035         dev_close(ndev);
3036         return ret;
3037 }
3038
3039 static const struct ethtool_ops cpsw_ethtool_ops = {
3040         .get_drvinfo    = cpsw_get_drvinfo,
3041         .get_msglevel   = cpsw_get_msglevel,
3042         .set_msglevel   = cpsw_set_msglevel,
3043         .get_link       = ethtool_op_get_link,
3044         .get_ts_info    = cpsw_get_ts_info,
3045         .get_coalesce   = cpsw_get_coalesce,
3046         .set_coalesce   = cpsw_set_coalesce,
3047         .get_sset_count         = cpsw_get_sset_count,
3048         .get_strings            = cpsw_get_strings,
3049         .get_ethtool_stats      = cpsw_get_ethtool_stats,
3050         .get_pauseparam         = cpsw_get_pauseparam,
3051         .set_pauseparam         = cpsw_set_pauseparam,
3052         .get_wol        = cpsw_get_wol,
3053         .set_wol        = cpsw_set_wol,
3054         .get_regs_len   = cpsw_get_regs_len,
3055         .get_regs       = cpsw_get_regs,
3056         .begin          = cpsw_ethtool_op_begin,
3057         .complete       = cpsw_ethtool_op_complete,
3058         .get_channels   = cpsw_get_channels,
3059         .set_channels   = cpsw_set_channels,
3060         .get_link_ksettings     = cpsw_get_link_ksettings,
3061         .set_link_ksettings     = cpsw_set_link_ksettings,
3062         .get_eee        = cpsw_get_eee,
3063         .set_eee        = cpsw_set_eee,
3064         .nway_reset     = cpsw_nway_reset,
3065         .get_ringparam = cpsw_get_ringparam,
3066         .set_ringparam = cpsw_set_ringparam,
3067 };
3068
3069 static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_common *cpsw,
3070                             u32 slave_reg_ofs, u32 sliver_reg_ofs)
3071 {
3072         void __iomem            *regs = cpsw->regs;
3073         int                     slave_num = slave->slave_num;
3074         struct cpsw_slave_data  *data = cpsw->data.slave_data + slave_num;
3075
3076         slave->data     = data;
3077         slave->regs     = regs + slave_reg_ofs;
3078         slave->sliver   = regs + sliver_reg_ofs;
3079         slave->port_vlan = data->dual_emac_res_vlan;
3080 }
3081
3082 static int cpsw_probe_dt(struct cpsw_platform_data *data,
3083                          struct platform_device *pdev)
3084 {
3085         struct device_node *node = pdev->dev.of_node;
3086         struct device_node *slave_node;
3087         int i = 0, ret;
3088         u32 prop;
3089
3090         if (!node)
3091                 return -EINVAL;
3092
3093         if (of_property_read_u32(node, "slaves", &prop)) {
3094                 dev_err(&pdev->dev, "Missing slaves property in the DT.\n");
3095                 return -EINVAL;
3096         }
3097         data->slaves = prop;
3098
3099         if (of_property_read_u32(node, "active_slave", &prop)) {
3100                 dev_err(&pdev->dev, "Missing active_slave property in the DT.\n");
3101                 return -EINVAL;
3102         }
3103         data->active_slave = prop;
3104
3105         data->slave_data = devm_kcalloc(&pdev->dev,
3106                                         data->slaves,
3107                                         sizeof(struct cpsw_slave_data),
3108                                         GFP_KERNEL);
3109         if (!data->slave_data)
3110                 return -ENOMEM;
3111
3112         if (of_property_read_u32(node, "cpdma_channels", &prop)) {
3113                 dev_err(&pdev->dev, "Missing cpdma_channels property in the DT.\n");
3114                 return -EINVAL;
3115         }
3116         data->channels = prop;
3117
3118         if (of_property_read_u32(node, "ale_entries", &prop)) {
3119                 dev_err(&pdev->dev, "Missing ale_entries property in the DT.\n");
3120                 return -EINVAL;
3121         }
3122         data->ale_entries = prop;
3123
3124         if (of_property_read_u32(node, "bd_ram_size", &prop)) {
3125                 dev_err(&pdev->dev, "Missing bd_ram_size property in the DT.\n");
3126                 return -EINVAL;
3127         }
3128         data->bd_ram_size = prop;
3129
3130         if (of_property_read_u32(node, "mac_control", &prop)) {
3131                 dev_err(&pdev->dev, "Missing mac_control property in the DT.\n");
3132                 return -EINVAL;
3133         }
3134         data->mac_control = prop;
3135
3136         if (of_property_read_bool(node, "dual_emac"))
3137                 data->dual_emac = 1;
3138
3139         /*
3140          * Populate all the child nodes here...
3141          */
3142         ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
3143         /* We do not want to force this, as in some cases may not have child */
3144         if (ret)
3145                 dev_warn(&pdev->dev, "Doesn't have any child node\n");
3146
3147         for_each_available_child_of_node(node, slave_node) {
3148                 struct cpsw_slave_data *slave_data = data->slave_data + i;
3149                 const void *mac_addr = NULL;
3150                 int lenp;
3151                 const __be32 *parp;
3152
3153                 /* This is no slave child node, continue */
3154                 if (strcmp(slave_node->name, "slave"))
3155                         continue;
3156
3157                 slave_data->ifphy = devm_of_phy_get(&pdev->dev, slave_node,
3158                                                     NULL);
3159                 if (!IS_ENABLED(CONFIG_TI_CPSW_PHY_SEL) &&
3160                     IS_ERR(slave_data->ifphy)) {
3161                         ret = PTR_ERR(slave_data->ifphy);
3162                         dev_err(&pdev->dev,
3163                                 "%d: Error retrieving port phy: %d\n", i, ret);
3164                         return ret;
3165                 }
3166
3167                 slave_data->phy_node = of_parse_phandle(slave_node,
3168                                                         "phy-handle", 0);
3169                 parp = of_get_property(slave_node, "phy_id", &lenp);
3170                 if (slave_data->phy_node) {
3171                         dev_dbg(&pdev->dev,
3172                                 "slave[%d] using phy-handle=\"%pOF\"\n",
3173                                 i, slave_data->phy_node);
3174                 } else if (of_phy_is_fixed_link(slave_node)) {
3175                         /* In the case of a fixed PHY, the DT node associated
3176                          * to the PHY is the Ethernet MAC DT node.
3177                          */
3178                         ret = of_phy_register_fixed_link(slave_node);
3179                         if (ret) {
3180                                 if (ret != -EPROBE_DEFER)
3181                                         dev_err(&pdev->dev, "failed to register fixed-link phy: %d\n", ret);
3182                                 return ret;
3183                         }
3184                         slave_data->phy_node = of_node_get(slave_node);
3185                 } else if (parp) {
3186                         u32 phyid;
3187                         struct device_node *mdio_node;
3188                         struct platform_device *mdio;
3189
3190                         if (lenp != (sizeof(__be32) * 2)) {
3191                                 dev_err(&pdev->dev, "Invalid slave[%d] phy_id property\n", i);
3192                                 goto no_phy_slave;
3193                         }
3194                         mdio_node = of_find_node_by_phandle(be32_to_cpup(parp));
3195                         phyid = be32_to_cpup(parp+1);
3196                         mdio = of_find_device_by_node(mdio_node);
3197                         of_node_put(mdio_node);
3198                         if (!mdio) {
3199                                 dev_err(&pdev->dev, "Missing mdio platform device\n");
3200                                 return -EINVAL;
3201                         }
3202                         snprintf(slave_data->phy_id, sizeof(slave_data->phy_id),
3203                                  PHY_ID_FMT, mdio->name, phyid);
3204                         put_device(&mdio->dev);
3205                 } else {
3206                         dev_err(&pdev->dev,
3207                                 "No slave[%d] phy_id, phy-handle, or fixed-link property\n",
3208                                 i);
3209                         goto no_phy_slave;
3210                 }
3211                 slave_data->phy_if = of_get_phy_mode(slave_node);
3212                 if (slave_data->phy_if < 0) {
3213                         dev_err(&pdev->dev, "Missing or malformed slave[%d] phy-mode property\n",
3214                                 i);
3215                         return slave_data->phy_if;
3216                 }
3217
3218 no_phy_slave:
3219                 mac_addr = of_get_mac_address(slave_node);
3220                 if (mac_addr) {
3221                         memcpy(slave_data->mac_addr, mac_addr, ETH_ALEN);
3222                 } else {
3223                         ret = ti_cm_get_macid(&pdev->dev, i,
3224                                               slave_data->mac_addr);
3225                         if (ret)
3226                                 return ret;
3227                 }
3228                 if (data->dual_emac) {
3229                         if (of_property_read_u32(slave_node, "dual_emac_res_vlan",
3230                                                  &prop)) {
3231                                 dev_err(&pdev->dev, "Missing dual_emac_res_vlan in DT.\n");
3232                                 slave_data->dual_emac_res_vlan = i+1;
3233                                 dev_err(&pdev->dev, "Using %d as Reserved VLAN for %d slave\n",
3234                                         slave_data->dual_emac_res_vlan, i);
3235                         } else {
3236                                 slave_data->dual_emac_res_vlan = prop;
3237                         }
3238                 }
3239
3240                 i++;
3241                 if (i == data->slaves)
3242                         break;
3243         }
3244
3245         return 0;
3246 }
3247
3248 static void cpsw_remove_dt(struct platform_device *pdev)
3249 {
3250         struct net_device *ndev = platform_get_drvdata(pdev);
3251         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
3252         struct cpsw_platform_data *data = &cpsw->data;
3253         struct device_node *node = pdev->dev.of_node;
3254         struct device_node *slave_node;
3255         int i = 0;
3256
3257         for_each_available_child_of_node(node, slave_node) {
3258                 struct cpsw_slave_data *slave_data = &data->slave_data[i];
3259
3260                 if (strcmp(slave_node->name, "slave"))
3261                         continue;
3262
3263                 if (of_phy_is_fixed_link(slave_node))
3264                         of_phy_deregister_fixed_link(slave_node);
3265
3266                 of_node_put(slave_data->phy_node);
3267
3268                 i++;
3269                 if (i == data->slaves)
3270                         break;
3271         }
3272
3273         of_platform_depopulate(&pdev->dev);
3274 }
3275
3276 static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
3277 {
3278         struct cpsw_common              *cpsw = priv->cpsw;
3279         struct cpsw_platform_data       *data = &cpsw->data;
3280         struct net_device               *ndev;
3281         struct cpsw_priv                *priv_sl2;
3282         int ret = 0;
3283
3284         ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
3285         if (!ndev) {
3286                 dev_err(cpsw->dev, "cpsw: error allocating net_device\n");
3287                 return -ENOMEM;
3288         }
3289
3290         priv_sl2 = netdev_priv(ndev);
3291         priv_sl2->cpsw = cpsw;
3292         priv_sl2->ndev = ndev;
3293         priv_sl2->dev  = &ndev->dev;
3294         priv_sl2->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
3295
3296         if (is_valid_ether_addr(data->slave_data[1].mac_addr)) {
3297                 memcpy(priv_sl2->mac_addr, data->slave_data[1].mac_addr,
3298                         ETH_ALEN);
3299                 dev_info(cpsw->dev, "cpsw: Detected MACID = %pM\n",
3300                          priv_sl2->mac_addr);
3301         } else {
3302                 eth_random_addr(priv_sl2->mac_addr);
3303                 dev_info(cpsw->dev, "cpsw: Random MACID = %pM\n",
3304                          priv_sl2->mac_addr);
3305         }
3306         memcpy(ndev->dev_addr, priv_sl2->mac_addr, ETH_ALEN);
3307
3308         priv_sl2->emac_port = 1;
3309         cpsw->slaves[1].ndev = ndev;
3310         ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
3311
3312         ndev->netdev_ops = &cpsw_netdev_ops;
3313         ndev->ethtool_ops = &cpsw_ethtool_ops;
3314
3315         /* register the network device */
3316         SET_NETDEV_DEV(ndev, cpsw->dev);
3317         ret = register_netdev(ndev);
3318         if (ret) {
3319                 dev_err(cpsw->dev, "cpsw: error registering net device\n");
3320                 free_netdev(ndev);
3321                 ret = -ENODEV;
3322         }
3323
3324         return ret;
3325 }
3326
3327 static const struct of_device_id cpsw_of_mtable[] = {
3328         { .compatible = "ti,cpsw"},
3329         { .compatible = "ti,am335x-cpsw"},
3330         { .compatible = "ti,am4372-cpsw"},
3331         { .compatible = "ti,dra7-cpsw"},
3332         { /* sentinel */ },
3333 };
3334 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
3335
3336 static const struct soc_device_attribute cpsw_soc_devices[] = {
3337         { .family = "AM33xx", .revision = "ES1.0"},
3338         { /* sentinel */ }
3339 };
3340
3341 static int cpsw_probe(struct platform_device *pdev)
3342 {
3343         struct clk                      *clk;
3344         struct cpsw_platform_data       *data;
3345         struct net_device               *ndev;
3346         struct cpsw_priv                *priv;
3347         struct cpdma_params             dma_params;
3348         struct cpsw_ale_params          ale_params;
3349         void __iomem                    *ss_regs;
3350         void __iomem                    *cpts_regs;
3351         struct resource                 *res, *ss_res;
3352         struct gpio_descs               *mode;
3353         u32 slave_offset, sliver_offset, slave_size;
3354         const struct soc_device_attribute *soc;
3355         struct cpsw_common              *cpsw;
3356         int ret = 0, i, ch;
3357         int irq;
3358
3359         cpsw = devm_kzalloc(&pdev->dev, sizeof(struct cpsw_common), GFP_KERNEL);
3360         if (!cpsw)
3361                 return -ENOMEM;
3362
3363         cpsw->dev = &pdev->dev;
3364
3365         ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
3366         if (!ndev) {
3367                 dev_err(&pdev->dev, "error allocating net_device\n");
3368                 return -ENOMEM;
3369         }
3370
3371         platform_set_drvdata(pdev, ndev);
3372         priv = netdev_priv(ndev);
3373         priv->cpsw = cpsw;
3374         priv->ndev = ndev;
3375         priv->dev  = &ndev->dev;
3376         priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
3377         cpsw->rx_packet_max = max(rx_packet_max, 128);
3378
3379         mode = devm_gpiod_get_array_optional(&pdev->dev, "mode", GPIOD_OUT_LOW);
3380         if (IS_ERR(mode)) {
3381                 ret = PTR_ERR(mode);
3382                 dev_err(&pdev->dev, "gpio request failed, ret %d\n", ret);
3383                 goto clean_ndev_ret;
3384         }
3385
3386         /*
3387          * This may be required here for child devices.
3388          */
3389         pm_runtime_enable(&pdev->dev);
3390
3391         /* Select default pin state */
3392         pinctrl_pm_select_default_state(&pdev->dev);
3393
3394         /* Need to enable clocks with runtime PM api to access module
3395          * registers
3396          */
3397         ret = pm_runtime_get_sync(&pdev->dev);
3398         if (ret < 0) {
3399                 pm_runtime_put_noidle(&pdev->dev);
3400                 goto clean_runtime_disable_ret;
3401         }
3402
3403         ret = cpsw_probe_dt(&cpsw->data, pdev);
3404         if (ret)
3405                 goto clean_dt_ret;
3406
3407         data = &cpsw->data;
3408         cpsw->rx_ch_num = 1;
3409         cpsw->tx_ch_num = 1;
3410
3411         if (is_valid_ether_addr(data->slave_data[0].mac_addr)) {
3412                 memcpy(priv->mac_addr, data->slave_data[0].mac_addr, ETH_ALEN);
3413                 dev_info(&pdev->dev, "Detected MACID = %pM\n", priv->mac_addr);
3414         } else {
3415                 eth_random_addr(priv->mac_addr);
3416                 dev_info(&pdev->dev, "Random MACID = %pM\n", priv->mac_addr);
3417         }
3418
3419         memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
3420
3421         cpsw->slaves = devm_kcalloc(&pdev->dev,
3422                                     data->slaves, sizeof(struct cpsw_slave),
3423                                     GFP_KERNEL);
3424         if (!cpsw->slaves) {
3425                 ret = -ENOMEM;
3426                 goto clean_dt_ret;
3427         }
3428         for (i = 0; i < data->slaves; i++)
3429                 cpsw->slaves[i].slave_num = i;
3430
3431         cpsw->slaves[0].ndev = ndev;
3432         priv->emac_port = 0;
3433
3434         clk = devm_clk_get(&pdev->dev, "fck");
3435         if (IS_ERR(clk)) {
3436                 dev_err(priv->dev, "fck is not found\n");
3437                 ret = -ENODEV;
3438                 goto clean_dt_ret;
3439         }
3440         cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000;
3441
3442         ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3443         ss_regs = devm_ioremap_resource(&pdev->dev, ss_res);
3444         if (IS_ERR(ss_regs)) {
3445                 ret = PTR_ERR(ss_regs);
3446                 goto clean_dt_ret;
3447         }
3448         cpsw->regs = ss_regs;
3449
3450         cpsw->version = readl(&cpsw->regs->id_ver);
3451
3452         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
3453         cpsw->wr_regs = devm_ioremap_resource(&pdev->dev, res);
3454         if (IS_ERR(cpsw->wr_regs)) {
3455                 ret = PTR_ERR(cpsw->wr_regs);
3456                 goto clean_dt_ret;
3457         }
3458
3459         memset(&dma_params, 0, sizeof(dma_params));
3460         memset(&ale_params, 0, sizeof(ale_params));
3461
3462         switch (cpsw->version) {
3463         case CPSW_VERSION_1:
3464                 cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
3465                 cpts_regs               = ss_regs + CPSW1_CPTS_OFFSET;
3466                 cpsw->hw_stats       = ss_regs + CPSW1_HW_STATS;
3467                 dma_params.dmaregs   = ss_regs + CPSW1_CPDMA_OFFSET;
3468                 dma_params.txhdp     = ss_regs + CPSW1_STATERAM_OFFSET;
3469                 ale_params.ale_regs  = ss_regs + CPSW1_ALE_OFFSET;
3470                 slave_offset         = CPSW1_SLAVE_OFFSET;
3471                 slave_size           = CPSW1_SLAVE_SIZE;
3472                 sliver_offset        = CPSW1_SLIVER_OFFSET;
3473                 dma_params.desc_mem_phys = 0;
3474                 break;
3475         case CPSW_VERSION_2:
3476         case CPSW_VERSION_3:
3477         case CPSW_VERSION_4:
3478                 cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
3479                 cpts_regs               = ss_regs + CPSW2_CPTS_OFFSET;
3480                 cpsw->hw_stats       = ss_regs + CPSW2_HW_STATS;
3481                 dma_params.dmaregs   = ss_regs + CPSW2_CPDMA_OFFSET;
3482                 dma_params.txhdp     = ss_regs + CPSW2_STATERAM_OFFSET;
3483                 ale_params.ale_regs  = ss_regs + CPSW2_ALE_OFFSET;
3484                 slave_offset         = CPSW2_SLAVE_OFFSET;
3485                 slave_size           = CPSW2_SLAVE_SIZE;
3486                 sliver_offset        = CPSW2_SLIVER_OFFSET;
3487                 dma_params.desc_mem_phys =
3488                         (u32 __force) ss_res->start + CPSW2_BD_OFFSET;
3489                 break;
3490         default:
3491                 dev_err(priv->dev, "unknown version 0x%08x\n", cpsw->version);
3492                 ret = -ENODEV;
3493                 goto clean_dt_ret;
3494         }
3495         for (i = 0; i < cpsw->data.slaves; i++) {
3496                 struct cpsw_slave *slave = &cpsw->slaves[i];
3497
3498                 cpsw_slave_init(slave, cpsw, slave_offset, sliver_offset);
3499                 slave_offset  += slave_size;
3500                 sliver_offset += SLIVER_SIZE;
3501         }
3502
3503         dma_params.dev          = &pdev->dev;
3504         dma_params.rxthresh     = dma_params.dmaregs + CPDMA_RXTHRESH;
3505         dma_params.rxfree       = dma_params.dmaregs + CPDMA_RXFREE;
3506         dma_params.rxhdp        = dma_params.txhdp + CPDMA_RXHDP;
3507         dma_params.txcp         = dma_params.txhdp + CPDMA_TXCP;
3508         dma_params.rxcp         = dma_params.txhdp + CPDMA_RXCP;
3509
3510         dma_params.num_chan             = data->channels;
3511         dma_params.has_soft_reset       = true;
3512         dma_params.min_packet_size      = CPSW_MIN_PACKET_SIZE;
3513         dma_params.desc_mem_size        = data->bd_ram_size;
3514         dma_params.desc_align           = 16;
3515         dma_params.has_ext_regs         = true;
3516         dma_params.desc_hw_addr         = dma_params.desc_mem_phys;
3517         dma_params.bus_freq_mhz         = cpsw->bus_freq_mhz;
3518         dma_params.descs_pool_size      = descs_pool_size;
3519
3520         cpsw->dma = cpdma_ctlr_create(&dma_params);
3521         if (!cpsw->dma) {
3522                 dev_err(priv->dev, "error initializing dma\n");
3523                 ret = -ENOMEM;
3524                 goto clean_dt_ret;
3525         }
3526
3527         soc = soc_device_match(cpsw_soc_devices);
3528         if (soc)
3529                 cpsw->quirk_irq = 1;
3530
3531         ch = cpsw->quirk_irq ? 0 : 7;
3532         cpsw->txv[0].ch = cpdma_chan_create(cpsw->dma, ch, cpsw_tx_handler, 0);
3533         if (IS_ERR(cpsw->txv[0].ch)) {
3534                 dev_err(priv->dev, "error initializing tx dma channel\n");
3535                 ret = PTR_ERR(cpsw->txv[0].ch);
3536                 goto clean_dma_ret;
3537         }
3538
3539         cpsw->rxv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_rx_handler, 1);
3540         if (IS_ERR(cpsw->rxv[0].ch)) {
3541                 dev_err(priv->dev, "error initializing rx dma channel\n");
3542                 ret = PTR_ERR(cpsw->rxv[0].ch);
3543                 goto clean_dma_ret;
3544         }
3545
3546         ale_params.dev                  = &pdev->dev;
3547         ale_params.ale_ageout           = ale_ageout;
3548         ale_params.ale_entries          = data->ale_entries;
3549         ale_params.ale_ports            = CPSW_ALE_PORTS_NUM;
3550
3551         cpsw->ale = cpsw_ale_create(&ale_params);
3552         if (!cpsw->ale) {
3553                 dev_err(priv->dev, "error initializing ale engine\n");
3554                 ret = -ENODEV;
3555                 goto clean_dma_ret;
3556         }
3557
3558         cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpsw->dev->of_node);
3559         if (IS_ERR(cpsw->cpts)) {
3560                 ret = PTR_ERR(cpsw->cpts);
3561                 goto clean_dma_ret;
3562         }
3563
3564         ndev->irq = platform_get_irq(pdev, 1);
3565         if (ndev->irq < 0) {
3566                 dev_err(priv->dev, "error getting irq resource\n");
3567                 ret = ndev->irq;
3568                 goto clean_dma_ret;
3569         }
3570
3571         ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
3572
3573         ndev->netdev_ops = &cpsw_netdev_ops;
3574         ndev->ethtool_ops = &cpsw_ethtool_ops;
3575         netif_napi_add(ndev, &cpsw->napi_rx,
3576                        cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll,
3577                        CPSW_POLL_WEIGHT);
3578         netif_tx_napi_add(ndev, &cpsw->napi_tx,
3579                           cpsw->quirk_irq ? cpsw_tx_poll : cpsw_tx_mq_poll,
3580                           CPSW_POLL_WEIGHT);
3581         cpsw_split_res(ndev);
3582
3583         /* register the network device */
3584         SET_NETDEV_DEV(ndev, &pdev->dev);
3585         ret = register_netdev(ndev);
3586         if (ret) {
3587                 dev_err(priv->dev, "error registering net device\n");
3588                 ret = -ENODEV;
3589                 goto clean_dma_ret;
3590         }
3591
3592         if (cpsw->data.dual_emac) {
3593                 ret = cpsw_probe_dual_emac(priv);
3594                 if (ret) {
3595                         cpsw_err(priv, probe, "error probe slave 2 emac interface\n");
3596                         goto clean_unregister_netdev_ret;
3597                 }
3598         }
3599
3600         /* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and
3601          * MISC IRQs which are always kept disabled with this driver so
3602          * we will not request them.
3603          *
3604          * If anyone wants to implement support for those, make sure to
3605          * first request and append them to irqs_table array.
3606          */
3607
3608         /* RX IRQ */
3609         irq = platform_get_irq(pdev, 1);
3610         if (irq < 0) {
3611                 ret = irq;
3612                 goto clean_dma_ret;
3613         }
3614
3615         cpsw->irqs_table[0] = irq;
3616         ret = devm_request_irq(&pdev->dev, irq, cpsw_rx_interrupt,
3617                                0, dev_name(&pdev->dev), cpsw);
3618         if (ret < 0) {
3619                 dev_err(priv->dev, "error attaching irq (%d)\n", ret);
3620                 goto clean_dma_ret;
3621         }
3622
3623         /* TX IRQ */
3624         irq = platform_get_irq(pdev, 2);
3625         if (irq < 0) {
3626                 ret = irq;
3627                 goto clean_dma_ret;
3628         }
3629
3630         cpsw->irqs_table[1] = irq;
3631         ret = devm_request_irq(&pdev->dev, irq, cpsw_tx_interrupt,
3632                                0, dev_name(&pdev->dev), cpsw);
3633         if (ret < 0) {
3634                 dev_err(priv->dev, "error attaching irq (%d)\n", ret);
3635                 goto clean_dma_ret;
3636         }
3637
3638         cpsw_notice(priv, probe,
3639                     "initialized device (regs %pa, irq %d, pool size %d)\n",
3640                     &ss_res->start, ndev->irq, dma_params.descs_pool_size);
3641
3642         pm_runtime_put(&pdev->dev);
3643
3644         return 0;
3645
3646 clean_unregister_netdev_ret:
3647         unregister_netdev(ndev);
3648 clean_dma_ret:
3649         cpdma_ctlr_destroy(cpsw->dma);
3650 clean_dt_ret:
3651         cpsw_remove_dt(pdev);
3652         pm_runtime_put_sync(&pdev->dev);
3653 clean_runtime_disable_ret:
3654         pm_runtime_disable(&pdev->dev);
3655 clean_ndev_ret:
3656         free_netdev(priv->ndev);
3657         return ret;
3658 }
3659
3660 static int cpsw_remove(struct platform_device *pdev)
3661 {
3662         struct net_device *ndev = platform_get_drvdata(pdev);
3663         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
3664         int ret;
3665
3666         ret = pm_runtime_get_sync(&pdev->dev);
3667         if (ret < 0) {
3668                 pm_runtime_put_noidle(&pdev->dev);
3669                 return ret;
3670         }
3671
3672         if (cpsw->data.dual_emac)
3673                 unregister_netdev(cpsw->slaves[1].ndev);
3674         unregister_netdev(ndev);
3675
3676         cpts_release(cpsw->cpts);
3677         cpdma_ctlr_destroy(cpsw->dma);
3678         cpsw_remove_dt(pdev);
3679         pm_runtime_put_sync(&pdev->dev);
3680         pm_runtime_disable(&pdev->dev);
3681         if (cpsw->data.dual_emac)
3682                 free_netdev(cpsw->slaves[1].ndev);
3683         free_netdev(ndev);
3684         return 0;
3685 }
3686
3687 #ifdef CONFIG_PM_SLEEP
3688 static int cpsw_suspend(struct device *dev)
3689 {
3690         struct net_device       *ndev = dev_get_drvdata(dev);
3691         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
3692
3693         if (cpsw->data.dual_emac) {
3694                 int i;
3695
3696                 for (i = 0; i < cpsw->data.slaves; i++) {
3697                         if (netif_running(cpsw->slaves[i].ndev))
3698                                 cpsw_ndo_stop(cpsw->slaves[i].ndev);
3699                 }
3700         } else {
3701                 if (netif_running(ndev))
3702                         cpsw_ndo_stop(ndev);
3703         }
3704
3705         /* Select sleep pin state */
3706         pinctrl_pm_select_sleep_state(dev);
3707
3708         return 0;
3709 }
3710
3711 static int cpsw_resume(struct device *dev)
3712 {
3713         struct net_device       *ndev = dev_get_drvdata(dev);
3714         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
3715
3716         /* Select default pin state */
3717         pinctrl_pm_select_default_state(dev);
3718
3719         /* shut up ASSERT_RTNL() warning in netif_set_real_num_tx/rx_queues */
3720         rtnl_lock();
3721         if (cpsw->data.dual_emac) {
3722                 int i;
3723
3724                 for (i = 0; i < cpsw->data.slaves; i++) {
3725                         if (netif_running(cpsw->slaves[i].ndev))
3726                                 cpsw_ndo_open(cpsw->slaves[i].ndev);
3727                 }
3728         } else {
3729                 if (netif_running(ndev))
3730                         cpsw_ndo_open(ndev);
3731         }
3732         rtnl_unlock();
3733
3734         return 0;
3735 }
3736 #endif
3737
3738 static SIMPLE_DEV_PM_OPS(cpsw_pm_ops, cpsw_suspend, cpsw_resume);
3739
3740 static struct platform_driver cpsw_driver = {
3741         .driver = {
3742                 .name    = "cpsw",
3743                 .pm      = &cpsw_pm_ops,
3744                 .of_match_table = cpsw_of_mtable,
3745         },
3746         .probe = cpsw_probe,
3747         .remove = cpsw_remove,
3748 };
3749
3750 module_platform_driver(cpsw_driver);
3751
3752 MODULE_LICENSE("GPL");
3753 MODULE_AUTHOR("Cyril Chemparathy <cyril@ti.com>");
3754 MODULE_AUTHOR("Mugunthan V N <mugunthanvnm@ti.com>");
3755 MODULE_DESCRIPTION("TI CPSW Ethernet driver");