drivers/net/ethernet/ibm/emac/core.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * drivers/net/ethernet/ibm/emac/core.c
   4  *
   5  * Driver for PowerPC 4xx on-chip ethernet controller.
   6  *
   7  * Copyright 2007 Benjamin Herrenschmidt, IBM Corp.
   8  *                <benh@kernel.crashing.org>
   9  *
  10  * Based on the arch/ppc version of the driver:
  11  *
  12  * Copyright (c) 2004, 2005 Zultys Technologies.
  13  * Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>
  14  *
  15  * Based on original work by
  16  *      Matt Porter <mporter@kernel.crashing.org>
  17  *      (c) 2003 Benjamin Herrenschmidt <benh@kernel.crashing.org>
  18  *      Armin Kuster <akuster@mvista.com>
  19  *      Johnnie Peters <jpeters@mvista.com>
  20  */
  21
  22 #include <linux/module.h>
  23 #include <linux/sched.h>
  24 #include <linux/string.h>
  25 #include <linux/errno.h>
  26 #include <linux/delay.h>
  27 #include <linux/types.h>
  28 #include <linux/pci.h>
  29 #include <linux/etherdevice.h>
  30 #include <linux/skbuff.h>
  31 #include <linux/crc32.h>
  32 #include <linux/ethtool.h>
  33 #include <linux/mii.h>
  34 #include <linux/bitops.h>
  35 #include <linux/of.h>
  36 #include <linux/of_address.h>
  37 #include <linux/of_irq.h>
  38 #include <linux/of_net.h>
  39 #include <linux/of_mdio.h>
  40 #include <linux/of_platform.h>
  41 #include <linux/platform_device.h>
  42 #include <linux/slab.h>
  43
  44 #include <asm/processor.h>
  45 #include <asm/io.h>
  46 #include <asm/dma.h>
  47 #include <linux/uaccess.h>
  48 #include <asm/dcr.h>
  49 #include <asm/dcr-regs.h>
  50
  51 #include "core.h"
  52
  53 /*
  54  * Lack of dma_unmap_???? calls is intentional.
  55  *
  56  * API-correct usage requires additional support state information to be
  57  * maintained for every RX and TX buffer descriptor (BD). Unfortunately, due to
  58  * EMAC design (e.g. TX buffer passed from network stack can be split into
  59  * several BDs, dma_map_single/dma_map_page can be used to map particular BD),
  60  * maintaining such information will add additional overhead.
  61  * Current DMA API implementation for 4xx processors only ensures cache coherency
  62  * and dma_unmap_???? routines are empty and are likely to stay this way.
  63  * I decided to omit dma_unmap_??? calls because I don't want to add additional
  64  * complexity just for the sake of following some abstract API, when it doesn't
  65  * add any real benefit to the driver. I understand that this decision maybe
  66  * controversial, but I really tried to make code API-correct and efficient
  67  * at the same time and didn't come up with code I liked :(.                --ebs
  68  */
  69
  70 #define DRV_NAME        "emac"
  71 #define DRV_VERSION     "3.54"
  72 #define DRV_DESC        "PPC 4xx OCP EMAC driver"
  73
  74 MODULE_DESCRIPTION(DRV_DESC);
  75 MODULE_AUTHOR
  76     ("Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>");
  77 MODULE_LICENSE("GPL");
  78
  79 /* minimum number of free TX descriptors required to wake up TX process */
  80 #define EMAC_TX_WAKEUP_THRESH           (NUM_TX_BUFF / 4)
  81
  82 /* If packet size is less than this number, we allocate small skb and copy packet
  83  * contents into it instead of just sending original big skb up
  84  */
  85 #define EMAC_RX_COPY_THRESH             CONFIG_IBM_EMAC_RX_COPY_THRESHOLD
  86
  87 /* Since multiple EMACs share MDIO lines in various ways, we need
  88  * to avoid re-using the same PHY ID in cases where the arch didn't
  89  * setup precise phy_map entries
  90  *
  91  * XXX This is something that needs to be reworked as we can have multiple
  92  * EMAC "sets" (multiple ASICs containing several EMACs) though we can
  93  * probably require in that case to have explicit PHY IDs in the device-tree
  94  */
  95 static u32 busy_phy_map;
  96 static DEFINE_MUTEX(emac_phy_map_lock);
  97
  98 /* Having stable interface names is a doomed idea. However, it would be nice
  99  * if we didn't have completely random interface names at boot too :-) It's
 100  * just a matter of making everybody's life easier. Since we are doing
 101  * threaded probing, it's a bit harder though. The base idea here is that
 102  * we make up a list of all emacs in the device-tree before we register the
 103  * driver. Every emac will then wait for the previous one in the list to
 104  * initialize before itself. We should also keep that list ordered by
 105  * cell_index.
 106  * That list is only 4 entries long, meaning that additional EMACs don't
 107  * get ordering guarantees unless EMAC_BOOT_LIST_SIZE is increased.
 108  */
 109
 110 #define EMAC_BOOT_LIST_SIZE     4
 111 static struct device_node *emac_boot_list[EMAC_BOOT_LIST_SIZE];
 112
 113 /* I don't want to litter system log with timeout errors
 114  * when we have brain-damaged PHY.
 115  */
 116 static inline void emac_report_timeout_error(struct emac_instance *dev,
 117                                              const char *error)
 118 {
 119         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX |
 120                                   EMAC_FTR_460EX_PHY_CLK_FIX |
 121                                   EMAC_FTR_440EP_PHY_CLK_FIX))
 122                 DBG(dev, "%s" NL, error);
 123         else if (net_ratelimit())
 124                 printk(KERN_ERR "%pOF: %s\n", dev->ofdev->dev.of_node, error);
 125 }
 126
 127 /* EMAC PHY clock workaround:
 128  * 440EP/440GR has more sane SDR0_MFR register implementation than 440GX,
 129  * which allows controlling each EMAC clock
 130  */
 131 static inline void emac_rx_clk_tx(struct emac_instance *dev)
 132 {
 133 #ifdef CONFIG_PPC_DCR_NATIVE
 134         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 135                 dcri_clrset(SDR0, SDR0_MFR,
 136                             0, SDR0_MFR_ECS >> dev->cell_index);
 137 #endif
 138 }
 139
 140 static inline void emac_rx_clk_default(struct emac_instance *dev)
 141 {
 142 #ifdef CONFIG_PPC_DCR_NATIVE
 143         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 144                 dcri_clrset(SDR0, SDR0_MFR,
 145                             SDR0_MFR_ECS >> dev->cell_index, 0);
 146 #endif
 147 }
 148
 149 /* PHY polling intervals */
 150 #define PHY_POLL_LINK_ON        HZ
 151 #define PHY_POLL_LINK_OFF       (HZ / 5)
 152
 153 /* Graceful stop timeouts in us.
 154  * We should allow up to 1 frame time (full-duplex, ignoring collisions)
 155  */
 156 #define STOP_TIMEOUT_10         1230
 157 #define STOP_TIMEOUT_100        124
 158 #define STOP_TIMEOUT_1000       13
 159 #define STOP_TIMEOUT_1000_JUMBO 73
 160
 161 static unsigned char default_mcast_addr[] = {
 162         0x01, 0x80, 0xC2, 0x00, 0x00, 0x01
 163 };
 164
 165 /* Please, keep in sync with struct ibm_emac_stats/ibm_emac_error_stats */
 166 static const char emac_stats_keys[EMAC_ETHTOOL_STATS_COUNT][ETH_GSTRING_LEN] = {
 167         "rx_packets", "rx_bytes", "tx_packets", "tx_bytes", "rx_packets_csum",
 168         "tx_packets_csum", "tx_undo", "rx_dropped_stack", "rx_dropped_oom",
 169         "rx_dropped_error", "rx_dropped_resize", "rx_dropped_mtu",
 170         "rx_stopped", "rx_bd_errors", "rx_bd_overrun", "rx_bd_bad_packet",
 171         "rx_bd_runt_packet", "rx_bd_short_event", "rx_bd_alignment_error",
 172         "rx_bd_bad_fcs", "rx_bd_packet_too_long", "rx_bd_out_of_range",
 173         "rx_bd_in_range", "rx_parity", "rx_fifo_overrun", "rx_overrun",
 174         "rx_bad_packet", "rx_runt_packet", "rx_short_event",
 175         "rx_alignment_error", "rx_bad_fcs", "rx_packet_too_long",
 176         "rx_out_of_range", "rx_in_range", "tx_dropped", "tx_bd_errors",
 177         "tx_bd_bad_fcs", "tx_bd_carrier_loss", "tx_bd_excessive_deferral",
 178         "tx_bd_excessive_collisions", "tx_bd_late_collision",
 179         "tx_bd_multple_collisions", "tx_bd_single_collision",
 180         "tx_bd_underrun", "tx_bd_sqe", "tx_parity", "tx_underrun", "tx_sqe",
 181         "tx_errors"
 182 };
 183
 184 static irqreturn_t emac_irq(int irq, void *dev_instance);
 185 static void emac_clean_tx_ring(struct emac_instance *dev);
 186 static void __emac_set_multicast_list(struct emac_instance *dev);
 187
 188 static inline int emac_phy_supports_gige(int phy_mode)
 189 {
 190         return  phy_interface_mode_is_rgmii(phy_mode) ||
 191                 phy_mode == PHY_INTERFACE_MODE_GMII ||
 192                 phy_mode == PHY_INTERFACE_MODE_SGMII ||
 193                 phy_mode == PHY_INTERFACE_MODE_TBI ||
 194                 phy_mode == PHY_INTERFACE_MODE_RTBI;
 195 }
 196
 197 static inline int emac_phy_gpcs(int phy_mode)
 198 {
 199         return  phy_mode == PHY_INTERFACE_MODE_SGMII ||
 200                 phy_mode == PHY_INTERFACE_MODE_TBI ||
 201                 phy_mode == PHY_INTERFACE_MODE_RTBI;
 202 }
 203
 204 static inline void emac_tx_enable(struct emac_instance *dev)
 205 {
 206         struct emac_regs __iomem *p = dev->emacp;
 207         u32 r;
 208
 209         DBG(dev, "tx_enable" NL);
 210
 211         r = in_be32(&p->mr0);
 212         if (!(r & EMAC_MR0_TXE))
 213                 out_be32(&p->mr0, r | EMAC_MR0_TXE);
 214 }
 215
 216 static void emac_tx_disable(struct emac_instance *dev)
 217 {
 218         struct emac_regs __iomem *p = dev->emacp;
 219         u32 r;
 220
 221         DBG(dev, "tx_disable" NL);
 222
 223         r = in_be32(&p->mr0);
 224         if (r & EMAC_MR0_TXE) {
 225                 int n = dev->stop_timeout;
 226                 out_be32(&p->mr0, r & ~EMAC_MR0_TXE);
 227                 while (!(in_be32(&p->mr0) & EMAC_MR0_TXI) && n) {
 228                         udelay(1);
 229                         --n;
 230                 }
 231                 if (unlikely(!n))
 232                         emac_report_timeout_error(dev, "TX disable timeout");
 233         }
 234 }
 235
 236 static void emac_rx_enable(struct emac_instance *dev)
 237 {
 238         struct emac_regs __iomem *p = dev->emacp;
 239         u32 r;
 240
 241         if (unlikely(test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags)))
 242                 goto out;
 243
 244         DBG(dev, "rx_enable" NL);
 245
 246         r = in_be32(&p->mr0);
 247         if (!(r & EMAC_MR0_RXE)) {
 248                 if (unlikely(!(r & EMAC_MR0_RXI))) {
 249                         /* Wait if previous async disable is still in progress */
 250                         int n = dev->stop_timeout;
 251                         while (!(r = in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 252                                 udelay(1);
 253                                 --n;
 254                         }
 255                         if (unlikely(!n))
 256                                 emac_report_timeout_error(dev,
 257                                                           "RX disable timeout");
 258                 }
 259                 out_be32(&p->mr0, r | EMAC_MR0_RXE);
 260         }
 261  out:
 262         ;
 263 }
 264
 265 static void emac_rx_disable(struct emac_instance *dev)
 266 {
 267         struct emac_regs __iomem *p = dev->emacp;
 268         u32 r;
 269
 270         DBG(dev, "rx_disable" NL);
 271
 272         r = in_be32(&p->mr0);
 273         if (r & EMAC_MR0_RXE) {
 274                 int n = dev->stop_timeout;
 275                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 276                 while (!(in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 277                         udelay(1);
 278                         --n;
 279                 }
 280                 if (unlikely(!n))
 281                         emac_report_timeout_error(dev, "RX disable timeout");
 282         }
 283 }
 284
 285 static inline void emac_netif_stop(struct emac_instance *dev)
 286 {
 287         netif_tx_lock_bh(dev->ndev);
 288         netif_addr_lock(dev->ndev);
 289         dev->no_mcast = 1;
 290         netif_addr_unlock(dev->ndev);
 291         netif_tx_unlock_bh(dev->ndev);
 292         netif_trans_update(dev->ndev);  /* prevent tx timeout */
 293         mal_poll_disable(dev->mal, &dev->commac);
 294         netif_tx_disable(dev->ndev);
 295 }
 296
 297 static inline void emac_netif_start(struct emac_instance *dev)
 298 {
 299         netif_tx_lock_bh(dev->ndev);
 300         netif_addr_lock(dev->ndev);
 301         dev->no_mcast = 0;
 302         if (dev->mcast_pending && netif_running(dev->ndev))
 303                 __emac_set_multicast_list(dev);
 304         netif_addr_unlock(dev->ndev);
 305         netif_tx_unlock_bh(dev->ndev);
 306
 307         netif_wake_queue(dev->ndev);
 308
 309         /* NOTE: unconditional netif_wake_queue is only appropriate
 310          * so long as all callers are assured to have free tx slots
 311          * (taken from tg3... though the case where that is wrong is
 312          *  not terribly harmful)
 313          */
 314         mal_poll_enable(dev->mal, &dev->commac);
 315 }
 316
 317 static inline void emac_rx_disable_async(struct emac_instance *dev)
 318 {
 319         struct emac_regs __iomem *p = dev->emacp;
 320         u32 r;
 321
 322         DBG(dev, "rx_disable_async" NL);
 323
 324         r = in_be32(&p->mr0);
 325         if (r & EMAC_MR0_RXE)
 326                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 327 }
 328
 329 static int emac_reset(struct emac_instance *dev)
 330 {
 331         struct emac_regs __iomem *p = dev->emacp;
 332         int n = 20;
 333         bool __maybe_unused try_internal_clock = false;
 334
 335         DBG(dev, "reset" NL);
 336
 337         if (!dev->reset_failed) {
 338                 /* 40x erratum suggests stopping RX channel before reset,
 339                  * we stop TX as well
 340                  */
 341                 emac_rx_disable(dev);
 342                 emac_tx_disable(dev);
 343         }
 344
 345 #ifdef CONFIG_PPC_DCR_NATIVE
 346 do_retry:
 347         /*
 348          * PPC460EX/GT Embedded Processor Advanced User's Manual
 349          * section 28.10.1 Mode Register 0 (EMACx_MR0) states:
 350          * Note: The PHY must provide a TX Clk in order to perform a soft reset
 351          * of the EMAC. If none is present, select the internal clock
 352          * (SDR0_ETH_CFG[EMACx_PHY_CLK] = 1).
 353          * After a soft reset, select the external clock.
 354          *
 355          * The AR8035-A PHY Meraki MR24 does not provide a TX Clk if the
 356          * ethernet cable is not attached. This causes the reset to timeout
 357          * and the PHY detection code in emac_init_phy() is unable to
 358          * communicate and detect the AR8035-A PHY. As a result, the emac
 359          * driver bails out early and the user has no ethernet.
 360          * In order to stay compatible with existing configurations, the
 361          * driver will temporarily switch to the internal clock, after
 362          * the first reset fails.
 363          */
 364         if (emac_has_feature(dev, EMAC_FTR_460EX_PHY_CLK_FIX)) {
 365                 if (try_internal_clock || (dev->phy_address == 0xffffffff &&
 366                                            dev->phy_map == 0xffffffff)) {
 367                         /* No PHY: select internal loop clock before reset */
 368                         dcri_clrset(SDR0, SDR0_ETH_CFG,
 369                                     0, SDR0_ETH_CFG_ECS << dev->cell_index);
 370                 } else {
 371                         /* PHY present: select external clock before reset */
 372                         dcri_clrset(SDR0, SDR0_ETH_CFG,
 373                                     SDR0_ETH_CFG_ECS << dev->cell_index, 0);
 374                 }
 375         }
 376 #endif
 377
 378         out_be32(&p->mr0, EMAC_MR0_SRST);
 379         while ((in_be32(&p->mr0) & EMAC_MR0_SRST) && n)
 380                 --n;
 381
 382 #ifdef CONFIG_PPC_DCR_NATIVE
 383         if (emac_has_feature(dev, EMAC_FTR_460EX_PHY_CLK_FIX)) {
 384                 if (!n && !try_internal_clock) {
 385                         /* first attempt has timed out. */
 386                         n = 20;
 387                         try_internal_clock = true;
 388                         goto do_retry;
 389                 }
 390
 391                 if (try_internal_clock || (dev->phy_address == 0xffffffff &&
 392                                            dev->phy_map == 0xffffffff)) {
 393                         /* No PHY: restore external clock source after reset */
 394                         dcri_clrset(SDR0, SDR0_ETH_CFG,
 395                                     SDR0_ETH_CFG_ECS << dev->cell_index, 0);
 396                 }
 397         }
 398 #endif
 399
 400         if (n) {
 401                 dev->reset_failed = 0;
 402                 return 0;
 403         } else {
 404                 emac_report_timeout_error(dev, "reset timeout");
 405                 dev->reset_failed = 1;
 406                 return -ETIMEDOUT;
 407         }
 408 }
 409
 410 static void emac_hash_mc(struct emac_instance *dev)
 411 {
 412         u32 __iomem *gaht_base = emac_gaht_base(dev);
 413         const int regs = EMAC_XAHT_REGS(dev);
 414         u32 gaht_temp[EMAC_XAHT_MAX_REGS];
 415         struct netdev_hw_addr *ha;
 416         int i;
 417
 418         DBG(dev, "hash_mc %d" NL, netdev_mc_count(dev->ndev));
 419
 420         memset(gaht_temp, 0, sizeof (gaht_temp));
 421
 422         netdev_for_each_mc_addr(ha, dev->ndev) {
 423                 int slot, reg, mask;
 424                 DBG2(dev, "mc %pM" NL, ha->addr);
 425
 426                 slot = EMAC_XAHT_CRC_TO_SLOT(dev,
 427                                              ether_crc(ETH_ALEN, ha->addr));
 428                 reg = EMAC_XAHT_SLOT_TO_REG(dev, slot);
 429                 mask = EMAC_XAHT_SLOT_TO_MASK(dev, slot);
 430
 431                 gaht_temp[reg] |= mask;
 432         }
 433
 434         for (i = 0; i < regs; i++)
 435                 out_be32(gaht_base + i, gaht_temp[i]);
 436 }
 437
 438 static inline u32 emac_iff2rmr(struct net_device *ndev)
 439 {
 440         struct emac_instance *dev = netdev_priv(ndev);
 441         u32 r;
 442
 443         r = EMAC_RMR_SP | EMAC_RMR_SFCS | EMAC_RMR_IAE | EMAC_RMR_BAE;
 444
 445         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 446             r |= EMAC4_RMR_BASE;
 447         else
 448             r |= EMAC_RMR_BASE;
 449
 450         if (ndev->flags & IFF_PROMISC)
 451                 r |= EMAC_RMR_PME;
 452         else if (ndev->flags & IFF_ALLMULTI ||
 453                          (netdev_mc_count(ndev) > EMAC_XAHT_SLOTS(dev)))
 454                 r |= EMAC_RMR_PMME;
 455         else if (!netdev_mc_empty(ndev))
 456                 r |= EMAC_RMR_MAE;
 457
 458         if (emac_has_feature(dev, EMAC_APM821XX_REQ_JUMBO_FRAME_SIZE)) {
 459                 r &= ~EMAC4_RMR_MJS_MASK;
 460                 r |= EMAC4_RMR_MJS(ndev->mtu);
 461         }
 462
 463         return r;
 464 }
 465
 466 static u32 __emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 467 {
 468         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC_MR1_TR0_MULT;
 469
 470         DBG2(dev, "__emac_calc_base_mr1" NL);
 471
 472         switch(tx_size) {
 473         case 2048:
 474                 ret |= EMAC_MR1_TFS_2K;
 475                 break;
 476         default:
 477                 printk(KERN_WARNING "%s: Unknown Tx FIFO size %d\n",
 478                        dev->ndev->name, tx_size);
 479         }
 480
 481         switch(rx_size) {
 482         case 16384:
 483                 ret |= EMAC_MR1_RFS_16K;
 484                 break;
 485         case 4096:
 486                 ret |= EMAC_MR1_RFS_4K;
 487                 break;
 488         default:
 489                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 490                        dev->ndev->name, rx_size);
 491         }
 492
 493         return ret;
 494 }
 495
 496 static u32 __emac4_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 497 {
 498         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC4_MR1_TR |
 499                 EMAC4_MR1_OBCI(dev->opb_bus_freq / 1000000);
 500
 501         DBG2(dev, "__emac4_calc_base_mr1" NL);
 502
 503         switch(tx_size) {
 504         case 16384:
 505                 ret |= EMAC4_MR1_TFS_16K;
 506                 break;
 507         case 8192:
 508                 ret |= EMAC4_MR1_TFS_8K;
 509                 break;
 510         case 4096:
 511                 ret |= EMAC4_MR1_TFS_4K;
 512                 break;
 513         case 2048:
 514                 ret |= EMAC4_MR1_TFS_2K;
 515                 break;
 516         default:
 517                 printk(KERN_WARNING "%s: Unknown Tx FIFO size %d\n",
 518                        dev->ndev->name, tx_size);
 519         }
 520
 521         switch(rx_size) {
 522         case 16384:
 523                 ret |= EMAC4_MR1_RFS_16K;
 524                 break;
 525         case 8192:
 526                 ret |= EMAC4_MR1_RFS_8K;
 527                 break;
 528         case 4096:
 529                 ret |= EMAC4_MR1_RFS_4K;
 530                 break;
 531         case 2048:
 532                 ret |= EMAC4_MR1_RFS_2K;
 533                 break;
 534         default:
 535                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 536                        dev->ndev->name, rx_size);
 537         }
 538
 539         return ret;
 540 }
 541
 542 static u32 emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 543 {
 544         return emac_has_feature(dev, EMAC_FTR_EMAC4) ?
 545                 __emac4_calc_base_mr1(dev, tx_size, rx_size) :
 546                 __emac_calc_base_mr1(dev, tx_size, rx_size);
 547 }
 548
 549 static inline u32 emac_calc_trtr(struct emac_instance *dev, unsigned int size)
 550 {
 551         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 552                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT_EMAC4;
 553         else
 554                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT;
 555 }
 556
 557 static inline u32 emac_calc_rwmr(struct emac_instance *dev,
 558                                  unsigned int low, unsigned int high)
 559 {
 560         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 561                 return (low << 22) | ( (high & 0x3ff) << 6);
 562         else
 563                 return (low << 23) | ( (high & 0x1ff) << 7);
 564 }
 565
 566 static int emac_configure(struct emac_instance *dev)
 567 {
 568         struct emac_regs __iomem *p = dev->emacp;
 569         struct net_device *ndev = dev->ndev;
 570         int tx_size, rx_size, link = netif_carrier_ok(dev->ndev);
 571         u32 r, mr1 = 0;
 572
 573         DBG(dev, "configure" NL);
 574
 575         if (!link) {
 576                 out_be32(&p->mr1, in_be32(&p->mr1)
 577                          | EMAC_MR1_FDE | EMAC_MR1_ILE);
 578                 udelay(100);
 579         } else if (emac_reset(dev) < 0)
 580                 return -ETIMEDOUT;
 581
 582         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
 583                 tah_reset(dev->tah_dev);
 584
 585         DBG(dev, " link = %d duplex = %d, pause = %d, asym_pause = %d\n",
 586             link, dev->phy.duplex, dev->phy.pause, dev->phy.asym_pause);
 587
 588         /* Default fifo sizes */
 589         tx_size = dev->tx_fifo_size;
 590         rx_size = dev->rx_fifo_size;
 591
 592         /* No link, force loopback */
 593         if (!link)
 594                 mr1 = EMAC_MR1_FDE | EMAC_MR1_ILE;
 595
 596         /* Check for full duplex */
 597         else if (dev->phy.duplex == DUPLEX_FULL)
 598                 mr1 |= EMAC_MR1_FDE | EMAC_MR1_MWSW_001;
 599
 600         /* Adjust fifo sizes, mr1 and timeouts based on link speed */
 601         dev->stop_timeout = STOP_TIMEOUT_10;
 602         switch (dev->phy.speed) {
 603         case SPEED_1000:
 604                 if (emac_phy_gpcs(dev->phy.mode)) {
 605                         mr1 |= EMAC_MR1_MF_1000GPCS | EMAC_MR1_MF_IPPA(
 606                                 (dev->phy.gpcs_address != 0xffffffff) ?
 607                                  dev->phy.gpcs_address : dev->phy.address);
 608
 609                         /* Put some arbitrary OUI, Manuf & Rev IDs so we can
 610                          * identify this GPCS PHY later.
 611                          */
 612                         out_be32(&p->u1.emac4.ipcr, 0xdeadbeef);
 613                 } else
 614                         mr1 |= EMAC_MR1_MF_1000;
 615
 616                 /* Extended fifo sizes */
 617                 tx_size = dev->tx_fifo_size_gige;
 618                 rx_size = dev->rx_fifo_size_gige;
 619
 620                 if (dev->ndev->mtu > ETH_DATA_LEN) {
 621                         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 622                                 mr1 |= EMAC4_MR1_JPSM;
 623                         else
 624                                 mr1 |= EMAC_MR1_JPSM;
 625                         dev->stop_timeout = STOP_TIMEOUT_1000_JUMBO;
 626                 } else
 627                         dev->stop_timeout = STOP_TIMEOUT_1000;
 628                 break;
 629         case SPEED_100:
 630                 mr1 |= EMAC_MR1_MF_100;
 631                 dev->stop_timeout = STOP_TIMEOUT_100;
 632                 break;
 633         default: /* make gcc happy */
 634                 break;
 635         }
 636
 637         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 638                 rgmii_set_speed(dev->rgmii_dev, dev->rgmii_port,
 639                                 dev->phy.speed);
 640         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 641                 zmii_set_speed(dev->zmii_dev, dev->zmii_port, dev->phy.speed);
 642
 643         /* on 40x erratum forces us to NOT use integrated flow control,
 644          * let's hope it works on 44x ;)
 645          */
 646         if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x) &&
 647             dev->phy.duplex == DUPLEX_FULL) {
 648                 if (dev->phy.pause)
 649                         mr1 |= EMAC_MR1_EIFC | EMAC_MR1_APP;
 650                 else if (dev->phy.asym_pause)
 651                         mr1 |= EMAC_MR1_APP;
 652         }
 653
 654         /* Add base settings & fifo sizes & program MR1 */
 655         mr1 |= emac_calc_base_mr1(dev, tx_size, rx_size);
 656         out_be32(&p->mr1, mr1);
 657
 658         /* Set individual MAC address */
 659         out_be32(&p->iahr, (ndev->dev_addr[0] << 8) | ndev->dev_addr[1]);
 660         out_be32(&p->ialr, (ndev->dev_addr[2] << 24) |
 661                  (ndev->dev_addr[3] << 16) | (ndev->dev_addr[4] << 8) |
 662                  ndev->dev_addr[5]);
 663
 664         /* VLAN Tag Protocol ID */
 665         out_be32(&p->vtpid, 0x8100);
 666
 667         /* Receive mode register */
 668         r = emac_iff2rmr(ndev);
 669         if (r & EMAC_RMR_MAE)
 670                 emac_hash_mc(dev);
 671         out_be32(&p->rmr, r);
 672
 673         /* FIFOs thresholds */
 674         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 675                 r = EMAC4_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 676                                tx_size / 2 / dev->fifo_entry_size);
 677         else
 678                 r = EMAC_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 679                               tx_size / 2 / dev->fifo_entry_size);
 680         out_be32(&p->tmr1, r);
 681         out_be32(&p->trtr, emac_calc_trtr(dev, tx_size / 2));
 682
 683         /* PAUSE frame is sent when RX FIFO reaches its high-water mark,
 684            there should be still enough space in FIFO to allow the our link
 685            partner time to process this frame and also time to send PAUSE
 686            frame itself.
 687
 688            Here is the worst case scenario for the RX FIFO "headroom"
 689            (from "The Switch Book") (100Mbps, without preamble, inter-frame gap):
 690
 691            1) One maximum-length frame on TX                    1522 bytes
 692            2) One PAUSE frame time                                64 bytes
 693            3) PAUSE frame decode time allowance                   64 bytes
 694            4) One maximum-length frame on RX                    1522 bytes
 695            5) Round-trip propagation delay of the link (100Mb)    15 bytes
 696            ----------
 697            3187 bytes
 698
 699            I chose to set high-water mark to RX_FIFO_SIZE / 4 (1024 bytes)
 700            low-water mark  to RX_FIFO_SIZE / 8 (512 bytes)
 701          */
 702         r = emac_calc_rwmr(dev, rx_size / 8 / dev->fifo_entry_size,
 703                            rx_size / 4 / dev->fifo_entry_size);
 704         out_be32(&p->rwmr, r);
 705
 706         /* Set PAUSE timer to the maximum */
 707         out_be32(&p->ptr, 0xffff);
 708
 709         /* IRQ sources */
 710         r = EMAC_ISR_OVR | EMAC_ISR_BP | EMAC_ISR_SE |
 711                 EMAC_ISR_ALE | EMAC_ISR_BFCS | EMAC_ISR_PTLE | EMAC_ISR_ORE |
 712                 EMAC_ISR_IRE | EMAC_ISR_TE;
 713         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 714             r |= EMAC4_ISR_TXPE | EMAC4_ISR_RXPE /* | EMAC4_ISR_TXUE |
 715                                                   EMAC4_ISR_RXOE | */;
 716         out_be32(&p->iser,  r);
 717
 718         /* We need to take GPCS PHY out of isolate mode after EMAC reset */
 719         if (emac_phy_gpcs(dev->phy.mode)) {
 720                 if (dev->phy.gpcs_address != 0xffffffff)
 721                         emac_mii_reset_gpcs(&dev->phy);
 722                 else
 723                         emac_mii_reset_phy(&dev->phy);
 724         }
 725
 726         return 0;
 727 }
 728
 729 static void emac_reinitialize(struct emac_instance *dev)
 730 {
 731         DBG(dev, "reinitialize" NL);
 732
 733         emac_netif_stop(dev);
 734         if (!emac_configure(dev)) {
 735                 emac_tx_enable(dev);
 736                 emac_rx_enable(dev);
 737         }
 738         emac_netif_start(dev);
 739 }
 740
 741 static void emac_full_tx_reset(struct emac_instance *dev)
 742 {
 743         DBG(dev, "full_tx_reset" NL);
 744
 745         emac_tx_disable(dev);
 746         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
 747         emac_clean_tx_ring(dev);
 748         dev->tx_cnt = dev->tx_slot = dev->ack_slot = 0;
 749
 750         emac_configure(dev);
 751
 752         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
 753         emac_tx_enable(dev);
 754         emac_rx_enable(dev);
 755 }
 756
 757 static void emac_reset_work(struct work_struct *work)
 758 {
 759         struct emac_instance *dev = container_of(work, struct emac_instance, reset_work);
 760
 761         DBG(dev, "reset_work" NL);
 762
 763         mutex_lock(&dev->link_lock);
 764         if (dev->opened) {
 765                 emac_netif_stop(dev);
 766                 emac_full_tx_reset(dev);
 767                 emac_netif_start(dev);
 768         }
 769         mutex_unlock(&dev->link_lock);
 770 }
 771
 772 static void emac_tx_timeout(struct net_device *ndev, unsigned int txqueue)
 773 {
 774         struct emac_instance *dev = netdev_priv(ndev);
 775
 776         DBG(dev, "tx_timeout" NL);
 777
 778         schedule_work(&dev->reset_work);
 779 }
 780
 781
 782 static inline int emac_phy_done(struct emac_instance *dev, u32 stacr)
 783 {
 784         int done = !!(stacr & EMAC_STACR_OC);
 785
 786         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 787                 done = !done;
 788
 789         return done;
 790 };
 791
 792 static int __emac_mdio_read(struct emac_instance *dev, u8 id, u8 reg)
 793 {
 794         struct emac_regs __iomem *p = dev->emacp;
 795         u32 r = 0;
 796         int n, err = -ETIMEDOUT;
 797
 798         mutex_lock(&dev->mdio_lock);
 799
 800         DBG2(dev, "mdio_read(%02x,%02x)" NL, id, reg);
 801
 802         /* Enable proper MDIO port */
 803         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 804                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 805         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 806                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 807
 808         /* Wait for management interface to become idle */
 809         n = 20;
 810         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 811                 udelay(1);
 812                 if (!--n) {
 813                         DBG2(dev, " -> timeout wait idle\n");
 814                         goto bail;
 815                 }
 816         }
 817
 818         /* Issue read command */
 819         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 820                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 821         else
 822                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 823         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 824                 r |= EMAC_STACR_OC;
 825         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 826                 r |= EMACX_STACR_STAC_READ;
 827         else
 828                 r |= EMAC_STACR_STAC_READ;
 829         r |= (reg & EMAC_STACR_PRA_MASK)
 830                 | ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT);
 831         out_be32(&p->stacr, r);
 832
 833         /* Wait for read to complete */
 834         n = 200;
 835         while (!emac_phy_done(dev, (r = in_be32(&p->stacr)))) {
 836                 udelay(1);
 837                 if (!--n) {
 838                         DBG2(dev, " -> timeout wait complete\n");
 839                         goto bail;
 840                 }
 841         }
 842
 843         if (unlikely(r & EMAC_STACR_PHYE)) {
 844                 DBG(dev, "mdio_read(%02x, %02x) failed" NL, id, reg);
 845                 err = -EREMOTEIO;
 846                 goto bail;
 847         }
 848
 849         r = ((r >> EMAC_STACR_PHYD_SHIFT) & EMAC_STACR_PHYD_MASK);
 850
 851         DBG2(dev, "mdio_read -> %04x" NL, r);
 852         err = 0;
 853  bail:
 854         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 855                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 856         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 857                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 858         mutex_unlock(&dev->mdio_lock);
 859
 860         return err == 0 ? r : err;
 861 }
 862
 863 static void __emac_mdio_write(struct emac_instance *dev, u8 id, u8 reg,
 864                               u16 val)
 865 {
 866         struct emac_regs __iomem *p = dev->emacp;
 867         u32 r = 0;
 868         int n;
 869
 870         mutex_lock(&dev->mdio_lock);
 871
 872         DBG2(dev, "mdio_write(%02x,%02x,%04x)" NL, id, reg, val);
 873
 874         /* Enable proper MDIO port */
 875         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 876                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 877         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 878                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 879
 880         /* Wait for management interface to be idle */
 881         n = 20;
 882         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 883                 udelay(1);
 884                 if (!--n) {
 885                         DBG2(dev, " -> timeout wait idle\n");
 886                         goto bail;
 887                 }
 888         }
 889
 890         /* Issue write command */
 891         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 892                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 893         else
 894                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 895         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 896                 r |= EMAC_STACR_OC;
 897         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 898                 r |= EMACX_STACR_STAC_WRITE;
 899         else
 900                 r |= EMAC_STACR_STAC_WRITE;
 901         r |= (reg & EMAC_STACR_PRA_MASK) |
 902                 ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT) |
 903                 (val << EMAC_STACR_PHYD_SHIFT);
 904         out_be32(&p->stacr, r);
 905
 906         /* Wait for write to complete */
 907         n = 200;
 908         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 909                 udelay(1);
 910                 if (!--n) {
 911                         DBG2(dev, " -> timeout wait complete\n");
 912                         goto bail;
 913                 }
 914         }
 915  bail:
 916         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 917                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 918         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 919                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 920         mutex_unlock(&dev->mdio_lock);
 921 }
 922
 923 static int emac_mdio_read(struct net_device *ndev, int id, int reg)
 924 {
 925         struct emac_instance *dev = netdev_priv(ndev);
 926         int res;
 927
 928         res = __emac_mdio_read((dev->mdio_instance &&
 929                                 dev->phy.gpcs_address != id) ?
 930                                 dev->mdio_instance : dev,
 931                                (u8) id, (u8) reg);
 932         return res;
 933 }
 934
 935 static void emac_mdio_write(struct net_device *ndev, int id, int reg, int val)
 936 {
 937         struct emac_instance *dev = netdev_priv(ndev);
 938
 939         __emac_mdio_write((dev->mdio_instance &&
 940                            dev->phy.gpcs_address != id) ?
 941                            dev->mdio_instance : dev,
 942                           (u8) id, (u8) reg, (u16) val);
 943 }
 944
 945 /* Tx lock BH */
 946 static void __emac_set_multicast_list(struct emac_instance *dev)
 947 {
 948         struct emac_regs __iomem *p = dev->emacp;
 949         u32 rmr = emac_iff2rmr(dev->ndev);
 950
 951         DBG(dev, "__multicast %08x" NL, rmr);
 952
 953         /* I decided to relax register access rules here to avoid
 954          * full EMAC reset.
 955          *
 956          * There is a real problem with EMAC4 core if we use MWSW_001 bit
 957          * in MR1 register and do a full EMAC reset.
 958          * One TX BD status update is delayed and, after EMAC reset, it
 959          * never happens, resulting in TX hung (it'll be recovered by TX
 960          * timeout handler eventually, but this is just gross).
 961          * So we either have to do full TX reset or try to cheat here :)
 962          *
 963          * The only required change is to RX mode register, so I *think* all
 964          * we need is just to stop RX channel. This seems to work on all
 965          * tested SoCs.                                                --ebs
 966          *
 967          */
 968         dev->mcast_pending = 0;
 969         emac_rx_disable(dev);
 970         if (rmr & EMAC_RMR_MAE)
 971                 emac_hash_mc(dev);
 972         out_be32(&p->rmr, rmr);
 973         emac_rx_enable(dev);
 974 }
 975
 976 /* Tx lock BH */
 977 static void emac_set_multicast_list(struct net_device *ndev)
 978 {
 979         struct emac_instance *dev = netdev_priv(ndev);
 980
 981         DBG(dev, "multicast" NL);
 982
 983         BUG_ON(!netif_running(dev->ndev));
 984
 985         if (dev->no_mcast) {
 986                 dev->mcast_pending = 1;
 987                 return;
 988         }
 989
 990         mutex_lock(&dev->link_lock);
 991         __emac_set_multicast_list(dev);
 992         mutex_unlock(&dev->link_lock);
 993 }
 994
 995 static int emac_set_mac_address(struct net_device *ndev, void *sa)
 996 {
 997         struct emac_instance *dev = netdev_priv(ndev);
 998         struct sockaddr *addr = sa;
 999         struct emac_regs __iomem *p = dev->emacp;
1000
1001         if (!is_valid_ether_addr(addr->sa_data))
1002                return -EADDRNOTAVAIL;
1003
1004         mutex_lock(&dev->link_lock);
1005
1006         eth_hw_addr_set(ndev, addr->sa_data);
1007
1008         emac_rx_disable(dev);
1009         emac_tx_disable(dev);
1010         out_be32(&p->iahr, (ndev->dev_addr[0] << 8) | ndev->dev_addr[1]);
1011         out_be32(&p->ialr, (ndev->dev_addr[2] << 24) |
1012                 (ndev->dev_addr[3] << 16) | (ndev->dev_addr[4] << 8) |
1013                 ndev->dev_addr[5]);
1014         emac_tx_enable(dev);
1015         emac_rx_enable(dev);
1016
1017         mutex_unlock(&dev->link_lock);
1018
1019         return 0;
1020 }
1021
1022 static int emac_resize_rx_ring(struct emac_instance *dev, int new_mtu)
1023 {
1024         int rx_sync_size = emac_rx_sync_size(new_mtu);
1025         int rx_skb_size = emac_rx_skb_size(new_mtu);
1026         int i, ret = 0;
1027         int mr1_jumbo_bit_change = 0;
1028
1029         mutex_lock(&dev->link_lock);
1030         emac_netif_stop(dev);
1031         emac_rx_disable(dev);
1032         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
1033
1034         if (dev->rx_sg_skb) {
1035                 ++dev->estats.rx_dropped_resize;
1036                 dev_kfree_skb(dev->rx_sg_skb);
1037                 dev->rx_sg_skb = NULL;
1038         }
1039
1040         /* Make a first pass over RX ring and mark BDs ready, dropping
1041          * non-processed packets on the way. We need this as a separate pass
1042          * to simplify error recovery in the case of allocation failure later.
1043          */
1044         for (i = 0; i < NUM_RX_BUFF; ++i) {
1045                 if (dev->rx_desc[i].ctrl & MAL_RX_CTRL_FIRST)
1046                         ++dev->estats.rx_dropped_resize;
1047
1048                 dev->rx_desc[i].data_len = 0;
1049                 dev->rx_desc[i].ctrl = MAL_RX_CTRL_EMPTY |
1050                     (i == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1051         }
1052
1053         /* Reallocate RX ring only if bigger skb buffers are required */
1054         if (rx_skb_size <= dev->rx_skb_size)
1055                 goto skip;
1056
1057         /* Second pass, allocate new skbs */
1058         for (i = 0; i < NUM_RX_BUFF; ++i) {
1059                 struct sk_buff *skb;
1060
1061                 skb = netdev_alloc_skb_ip_align(dev->ndev, rx_skb_size);
1062                 if (!skb) {
1063                         ret = -ENOMEM;
1064                         goto oom;
1065                 }
1066
1067                 BUG_ON(!dev->rx_skb[i]);
1068                 dev_kfree_skb(dev->rx_skb[i]);
1069
1070                 dev->rx_desc[i].data_ptr =
1071                     dma_map_single(&dev->ofdev->dev, skb->data - NET_IP_ALIGN,
1072                                    rx_sync_size, DMA_FROM_DEVICE)
1073                                    + NET_IP_ALIGN;
1074                 dev->rx_skb[i] = skb;
1075         }
1076  skip:
1077         /* Check if we need to change "Jumbo" bit in MR1 */
1078         if (emac_has_feature(dev, EMAC_APM821XX_REQ_JUMBO_FRAME_SIZE)) {
1079                 mr1_jumbo_bit_change = (new_mtu > ETH_DATA_LEN) ||
1080                                 (dev->ndev->mtu > ETH_DATA_LEN);
1081         } else {
1082                 mr1_jumbo_bit_change = (new_mtu > ETH_DATA_LEN) ^
1083                                 (dev->ndev->mtu > ETH_DATA_LEN);
1084         }
1085
1086         if (mr1_jumbo_bit_change) {
1087                 /* This is to prevent starting RX channel in emac_rx_enable() */
1088                 set_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1089
1090                 WRITE_ONCE(dev->ndev->mtu, new_mtu);
1091                 emac_full_tx_reset(dev);
1092         }
1093
1094         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(new_mtu));
1095  oom:
1096         /* Restart RX */
1097         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1098         dev->rx_slot = 0;
1099         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1100         emac_rx_enable(dev);
1101         emac_netif_start(dev);
1102         mutex_unlock(&dev->link_lock);
1103
1104         return ret;
1105 }
1106
1107 /* Process ctx, rtnl_lock semaphore */
1108 static int emac_change_mtu(struct net_device *ndev, int new_mtu)
1109 {
1110         struct emac_instance *dev = netdev_priv(ndev);
1111         int ret = 0;
1112
1113         DBG(dev, "change_mtu(%d)" NL, new_mtu);
1114
1115         if (netif_running(ndev)) {
1116                 /* Check if we really need to reinitialize RX ring */
1117                 if (emac_rx_skb_size(ndev->mtu) != emac_rx_skb_size(new_mtu))
1118                         ret = emac_resize_rx_ring(dev, new_mtu);
1119         }
1120
1121         if (!ret) {
1122                 WRITE_ONCE(ndev->mtu, new_mtu);
1123                 dev->rx_skb_size = emac_rx_skb_size(new_mtu);
1124                 dev->rx_sync_size = emac_rx_sync_size(new_mtu);
1125         }
1126
1127         return ret;
1128 }
1129
1130 static void emac_clean_tx_ring(struct emac_instance *dev)
1131 {
1132         int i;
1133
1134         for (i = 0; i < NUM_TX_BUFF; ++i) {
1135                 if (dev->tx_skb[i]) {
1136                         dev_kfree_skb(dev->tx_skb[i]);
1137                         dev->tx_skb[i] = NULL;
1138                         if (dev->tx_desc[i].ctrl & MAL_TX_CTRL_READY)
1139                                 ++dev->estats.tx_dropped;
1140                 }
1141                 dev->tx_desc[i].ctrl = 0;
1142                 dev->tx_desc[i].data_ptr = 0;
1143         }
1144 }
1145
1146 static void emac_clean_rx_ring(struct emac_instance *dev)
1147 {
1148         int i;
1149
1150         for (i = 0; i < NUM_RX_BUFF; ++i)
1151                 if (dev->rx_skb[i]) {
1152                         dev->rx_desc[i].ctrl = 0;
1153                         dev_kfree_skb(dev->rx_skb[i]);
1154                         dev->rx_skb[i] = NULL;
1155                         dev->rx_desc[i].data_ptr = 0;
1156                 }
1157
1158         if (dev->rx_sg_skb) {
1159                 dev_kfree_skb(dev->rx_sg_skb);
1160                 dev->rx_sg_skb = NULL;
1161         }
1162 }
1163
1164 static int
1165 __emac_prepare_rx_skb(struct sk_buff *skb, struct emac_instance *dev, int slot)
1166 {
1167         if (unlikely(!skb))
1168                 return -ENOMEM;
1169
1170         dev->rx_skb[slot] = skb;
1171         dev->rx_desc[slot].data_len = 0;
1172
1173         dev->rx_desc[slot].data_ptr =
1174             dma_map_single(&dev->ofdev->dev, skb->data - NET_IP_ALIGN,
1175                            dev->rx_sync_size, DMA_FROM_DEVICE) + NET_IP_ALIGN;
1176         wmb();
1177         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1178             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1179
1180         return 0;
1181 }
1182
1183 static int
1184 emac_alloc_rx_skb(struct emac_instance *dev, int slot)
1185 {
1186         struct sk_buff *skb;
1187
1188         skb = __netdev_alloc_skb_ip_align(dev->ndev, dev->rx_skb_size,
1189                                           GFP_KERNEL);
1190
1191         return __emac_prepare_rx_skb(skb, dev, slot);
1192 }
1193
1194 static int
1195 emac_alloc_rx_skb_napi(struct emac_instance *dev, int slot)
1196 {
1197         struct sk_buff *skb;
1198
1199         skb = napi_alloc_skb(&dev->mal->napi, dev->rx_skb_size);
1200
1201         return __emac_prepare_rx_skb(skb, dev, slot);
1202 }
1203
1204 static void emac_print_link_status(struct emac_instance *dev)
1205 {
1206         if (netif_carrier_ok(dev->ndev))
1207                 printk(KERN_INFO "%s: link is up, %d %s%s\n",
1208                        dev->ndev->name, dev->phy.speed,
1209                        dev->phy.duplex == DUPLEX_FULL ? "FDX" : "HDX",
1210                        dev->phy.pause ? ", pause enabled" :
1211                        dev->phy.asym_pause ? ", asymmetric pause enabled" : "");
1212         else
1213                 printk(KERN_INFO "%s: link is down\n", dev->ndev->name);
1214 }
1215
1216 /* Process ctx, rtnl_lock semaphore */
1217 static int emac_open(struct net_device *ndev)
1218 {
1219         struct emac_instance *dev = netdev_priv(ndev);
1220         int i;
1221
1222         DBG(dev, "open" NL);
1223
1224         /* Allocate RX ring */
1225         for (i = 0; i < NUM_RX_BUFF; ++i)
1226                 if (emac_alloc_rx_skb(dev, i)) {
1227                         printk(KERN_ERR "%s: failed to allocate RX ring\n",
1228                                ndev->name);
1229                         goto oom;
1230                 }
1231
1232         dev->tx_cnt = dev->tx_slot = dev->ack_slot = dev->rx_slot = 0;
1233         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1234         dev->rx_sg_skb = NULL;
1235
1236         mutex_lock(&dev->link_lock);
1237         dev->opened = 1;
1238
1239         /* Start PHY polling now.
1240          */
1241         if (dev->phy.address >= 0) {
1242                 int link_poll_interval;
1243                 if (dev->phy.def->ops->poll_link(&dev->phy)) {
1244                         dev->phy.def->ops->read_link(&dev->phy);
1245                         emac_rx_clk_default(dev);
1246                         netif_carrier_on(dev->ndev);
1247                         link_poll_interval = PHY_POLL_LINK_ON;
1248                 } else {
1249                         emac_rx_clk_tx(dev);
1250                         netif_carrier_off(dev->ndev);
1251                         link_poll_interval = PHY_POLL_LINK_OFF;
1252                 }
1253                 dev->link_polling = 1;
1254                 wmb();
1255                 schedule_delayed_work(&dev->link_work, link_poll_interval);
1256                 emac_print_link_status(dev);
1257         } else
1258                 netif_carrier_on(dev->ndev);
1259
1260         /* Required for Pause packet support in EMAC */
1261         dev_mc_add_global(ndev, default_mcast_addr);
1262
1263         emac_configure(dev);
1264         mal_poll_add(dev->mal, &dev->commac);
1265         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
1266         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(ndev->mtu));
1267         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1268         emac_tx_enable(dev);
1269         emac_rx_enable(dev);
1270         emac_netif_start(dev);
1271
1272         mutex_unlock(&dev->link_lock);
1273
1274         return 0;
1275  oom:
1276         emac_clean_rx_ring(dev);
1277         return -ENOMEM;
1278 }
1279
1280 /* BHs disabled */
1281 #if 0
1282 static int emac_link_differs(struct emac_instance *dev)
1283 {
1284         u32 r = in_be32(&dev->emacp->mr1);
1285
1286         int duplex = r & EMAC_MR1_FDE ? DUPLEX_FULL : DUPLEX_HALF;
1287         int speed, pause, asym_pause;
1288
1289         if (r & EMAC_MR1_MF_1000)
1290                 speed = SPEED_1000;
1291         else if (r & EMAC_MR1_MF_100)
1292                 speed = SPEED_100;
1293         else
1294                 speed = SPEED_10;
1295
1296         switch (r & (EMAC_MR1_EIFC | EMAC_MR1_APP)) {
1297         case (EMAC_MR1_EIFC | EMAC_MR1_APP):
1298                 pause = 1;
1299                 asym_pause = 0;
1300                 break;
1301         case EMAC_MR1_APP:
1302                 pause = 0;
1303                 asym_pause = 1;
1304                 break;
1305         default:
1306                 pause = asym_pause = 0;
1307         }
1308         return speed != dev->phy.speed || duplex != dev->phy.duplex ||
1309             pause != dev->phy.pause || asym_pause != dev->phy.asym_pause;
1310 }
1311 #endif
1312
1313 static void emac_link_timer(struct work_struct *work)
1314 {
1315         struct emac_instance *dev =
1316                 container_of(to_delayed_work(work),
1317                              struct emac_instance, link_work);
1318         int link_poll_interval;
1319
1320         mutex_lock(&dev->link_lock);
1321         DBG2(dev, "link timer" NL);
1322
1323         if (!dev->opened)
1324                 goto bail;
1325
1326         if (dev->phy.def->ops->poll_link(&dev->phy)) {
1327                 if (!netif_carrier_ok(dev->ndev)) {
1328                         emac_rx_clk_default(dev);
1329                         /* Get new link parameters */
1330                         dev->phy.def->ops->read_link(&dev->phy);
1331
1332                         netif_carrier_on(dev->ndev);
1333                         emac_netif_stop(dev);
1334                         emac_full_tx_reset(dev);
1335                         emac_netif_start(dev);
1336                         emac_print_link_status(dev);
1337                 }
1338                 link_poll_interval = PHY_POLL_LINK_ON;
1339         } else {
1340                 if (netif_carrier_ok(dev->ndev)) {
1341                         emac_rx_clk_tx(dev);
1342                         netif_carrier_off(dev->ndev);
1343                         netif_tx_disable(dev->ndev);
1344                         emac_reinitialize(dev);
1345                         emac_print_link_status(dev);
1346                 }
1347                 link_poll_interval = PHY_POLL_LINK_OFF;
1348         }
1349         schedule_delayed_work(&dev->link_work, link_poll_interval);
1350  bail:
1351         mutex_unlock(&dev->link_lock);
1352 }
1353
1354 static void emac_force_link_update(struct emac_instance *dev)
1355 {
1356         netif_carrier_off(dev->ndev);
1357         smp_rmb();
1358         if (dev->link_polling) {
1359                 cancel_delayed_work_sync(&dev->link_work);
1360                 if (dev->link_polling)
1361                         schedule_delayed_work(&dev->link_work,  PHY_POLL_LINK_OFF);
1362         }
1363 }
1364
1365 /* Process ctx, rtnl_lock semaphore */
1366 static int emac_close(struct net_device *ndev)
1367 {
1368         struct emac_instance *dev = netdev_priv(ndev);
1369
1370         DBG(dev, "close" NL);
1371
1372         if (dev->phy.address >= 0) {
1373                 dev->link_polling = 0;
1374                 cancel_delayed_work_sync(&dev->link_work);
1375         }
1376         mutex_lock(&dev->link_lock);
1377         emac_netif_stop(dev);
1378         dev->opened = 0;
1379         mutex_unlock(&dev->link_lock);
1380
1381         emac_rx_disable(dev);
1382         emac_tx_disable(dev);
1383         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
1384         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
1385         mal_poll_del(dev->mal, &dev->commac);
1386
1387         emac_clean_tx_ring(dev);
1388         emac_clean_rx_ring(dev);
1389
1390         netif_carrier_off(ndev);
1391
1392         return 0;
1393 }
1394
1395 static inline u16 emac_tx_csum(struct emac_instance *dev,
1396                                struct sk_buff *skb)
1397 {
1398         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
1399                 (skb->ip_summed == CHECKSUM_PARTIAL)) {
1400                 ++dev->stats.tx_packets_csum;
1401                 return EMAC_TX_CTRL_TAH_CSUM;
1402         }
1403         return 0;
1404 }
1405
1406 static inline netdev_tx_t emac_xmit_finish(struct emac_instance *dev, int len)
1407 {
1408         struct emac_regs __iomem *p = dev->emacp;
1409         struct net_device *ndev = dev->ndev;
1410
1411         /* Send the packet out. If the if makes a significant perf
1412          * difference, then we can store the TMR0 value in "dev"
1413          * instead
1414          */
1415         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
1416                 out_be32(&p->tmr0, EMAC4_TMR0_XMIT);
1417         else
1418                 out_be32(&p->tmr0, EMAC_TMR0_XMIT);
1419
1420         if (unlikely(++dev->tx_cnt == NUM_TX_BUFF)) {
1421                 netif_stop_queue(ndev);
1422                 DBG2(dev, "stopped TX queue" NL);
1423         }
1424
1425         netif_trans_update(ndev);
1426         ++dev->stats.tx_packets;
1427         dev->stats.tx_bytes += len;
1428
1429         return NETDEV_TX_OK;
1430 }
1431
1432 /* Tx lock BH */
1433 static netdev_tx_t emac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
1434 {
1435         struct emac_instance *dev = netdev_priv(ndev);
1436         unsigned int len = skb->len;
1437         int slot;
1438
1439         u16 ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1440             MAL_TX_CTRL_LAST | emac_tx_csum(dev, skb);
1441
1442         slot = dev->tx_slot++;
1443         if (dev->tx_slot == NUM_TX_BUFF) {
1444                 dev->tx_slot = 0;
1445                 ctrl |= MAL_TX_CTRL_WRAP;
1446         }
1447
1448         DBG2(dev, "xmit(%u) %d" NL, len, slot);
1449
1450         dev->tx_skb[slot] = skb;
1451         dev->tx_desc[slot].data_ptr = dma_map_single(&dev->ofdev->dev,
1452                                                      skb->data, len,
1453                                                      DMA_TO_DEVICE);
1454         dev->tx_desc[slot].data_len = (u16) len;
1455         wmb();
1456         dev->tx_desc[slot].ctrl = ctrl;
1457
1458         return emac_xmit_finish(dev, len);
1459 }
1460
1461 static inline int emac_xmit_split(struct emac_instance *dev, int slot,
1462                                   u32 pd, int len, int last, u16 base_ctrl)
1463 {
1464         while (1) {
1465                 u16 ctrl = base_ctrl;
1466                 int chunk = min(len, MAL_MAX_TX_SIZE);
1467                 len -= chunk;
1468
1469                 slot = (slot + 1) % NUM_TX_BUFF;
1470
1471                 if (last && !len)
1472                         ctrl |= MAL_TX_CTRL_LAST;
1473                 if (slot == NUM_TX_BUFF - 1)
1474                         ctrl |= MAL_TX_CTRL_WRAP;
1475
1476                 dev->tx_skb[slot] = NULL;
1477                 dev->tx_desc[slot].data_ptr = pd;
1478                 dev->tx_desc[slot].data_len = (u16) chunk;
1479                 dev->tx_desc[slot].ctrl = ctrl;
1480                 ++dev->tx_cnt;
1481
1482                 if (!len)
1483                         break;
1484
1485                 pd += chunk;
1486         }
1487         return slot;
1488 }
1489
1490 /* Tx lock BH disabled (SG version for TAH equipped EMACs) */
1491 static netdev_tx_t
1492 emac_start_xmit_sg(struct sk_buff *skb, struct net_device *ndev)
1493 {
1494         struct emac_instance *dev = netdev_priv(ndev);
1495         int nr_frags = skb_shinfo(skb)->nr_frags;
1496         int len = skb->len, chunk;
1497         int slot, i;
1498         u16 ctrl;
1499         u32 pd;
1500
1501         /* This is common "fast" path */
1502         if (likely(!nr_frags && len <= MAL_MAX_TX_SIZE))
1503                 return emac_start_xmit(skb, ndev);
1504
1505         len -= skb->data_len;
1506
1507         /* Note, this is only an *estimation*, we can still run out of empty
1508          * slots because of the additional fragmentation into
1509          * MAL_MAX_TX_SIZE-sized chunks
1510          */
1511         if (unlikely(dev->tx_cnt + nr_frags + mal_tx_chunks(len) > NUM_TX_BUFF))
1512                 goto stop_queue;
1513
1514         ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1515             emac_tx_csum(dev, skb);
1516         slot = dev->tx_slot;
1517
1518         /* skb data */
1519         dev->tx_skb[slot] = NULL;
1520         chunk = min(len, MAL_MAX_TX_SIZE);
1521         dev->tx_desc[slot].data_ptr = pd =
1522             dma_map_single(&dev->ofdev->dev, skb->data, len, DMA_TO_DEVICE);
1523         dev->tx_desc[slot].data_len = (u16) chunk;
1524         len -= chunk;
1525         if (unlikely(len))
1526                 slot = emac_xmit_split(dev, slot, pd + chunk, len, !nr_frags,
1527                                        ctrl);
1528         /* skb fragments */
1529         for (i = 0; i < nr_frags; ++i) {
1530                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1531                 len = skb_frag_size(frag);
1532
1533                 if (unlikely(dev->tx_cnt + mal_tx_chunks(len) >= NUM_TX_BUFF))
1534                         goto undo_frame;
1535
1536                 pd = skb_frag_dma_map(&dev->ofdev->dev, frag, 0, len,
1537                                       DMA_TO_DEVICE);
1538
1539                 slot = emac_xmit_split(dev, slot, pd, len, i == nr_frags - 1,
1540                                        ctrl);
1541         }
1542
1543         DBG2(dev, "xmit_sg(%u) %d - %d" NL, skb->len, dev->tx_slot, slot);
1544
1545         /* Attach skb to the last slot so we don't release it too early */
1546         dev->tx_skb[slot] = skb;
1547
1548         /* Send the packet out */
1549         if (dev->tx_slot == NUM_TX_BUFF - 1)
1550                 ctrl |= MAL_TX_CTRL_WRAP;
1551         wmb();
1552         dev->tx_desc[dev->tx_slot].ctrl = ctrl;
1553         dev->tx_slot = (slot + 1) % NUM_TX_BUFF;
1554
1555         return emac_xmit_finish(dev, skb->len);
1556
1557  undo_frame:
1558         /* Well, too bad. Our previous estimation was overly optimistic.
1559          * Undo everything.
1560          */
1561         while (slot != dev->tx_slot) {
1562                 dev->tx_desc[slot].ctrl = 0;
1563                 --dev->tx_cnt;
1564                 if (--slot < 0)
1565                         slot = NUM_TX_BUFF - 1;
1566         }
1567         ++dev->estats.tx_undo;
1568
1569  stop_queue:
1570         netif_stop_queue(ndev);
1571         DBG2(dev, "stopped TX queue" NL);
1572         return NETDEV_TX_BUSY;
1573 }
1574
1575 /* Tx lock BHs */
1576 static void emac_parse_tx_error(struct emac_instance *dev, u16 ctrl)
1577 {
1578         struct emac_error_stats *st = &dev->estats;
1579
1580         DBG(dev, "BD TX error %04x" NL, ctrl);
1581
1582         ++st->tx_bd_errors;
1583         if (ctrl & EMAC_TX_ST_BFCS)
1584                 ++st->tx_bd_bad_fcs;
1585         if (ctrl & EMAC_TX_ST_LCS)
1586                 ++st->tx_bd_carrier_loss;
1587         if (ctrl & EMAC_TX_ST_ED)
1588                 ++st->tx_bd_excessive_deferral;
1589         if (ctrl & EMAC_TX_ST_EC)
1590                 ++st->tx_bd_excessive_collisions;
1591         if (ctrl & EMAC_TX_ST_LC)
1592                 ++st->tx_bd_late_collision;
1593         if (ctrl & EMAC_TX_ST_MC)
1594                 ++st->tx_bd_multple_collisions;
1595         if (ctrl & EMAC_TX_ST_SC)
1596                 ++st->tx_bd_single_collision;
1597         if (ctrl & EMAC_TX_ST_UR)
1598                 ++st->tx_bd_underrun;
1599         if (ctrl & EMAC_TX_ST_SQE)
1600                 ++st->tx_bd_sqe;
1601 }
1602
1603 static void emac_poll_tx(void *param)
1604 {
1605         struct emac_instance *dev = param;
1606         u32 bad_mask;
1607
1608         DBG2(dev, "poll_tx, %d %d" NL, dev->tx_cnt, dev->ack_slot);
1609
1610         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
1611                 bad_mask = EMAC_IS_BAD_TX_TAH;
1612         else
1613                 bad_mask = EMAC_IS_BAD_TX;
1614
1615         netif_tx_lock_bh(dev->ndev);
1616         if (dev->tx_cnt) {
1617                 u16 ctrl;
1618                 int slot = dev->ack_slot, n = 0;
1619         again:
1620                 ctrl = dev->tx_desc[slot].ctrl;
1621                 if (!(ctrl & MAL_TX_CTRL_READY)) {
1622                         struct sk_buff *skb = dev->tx_skb[slot];
1623                         ++n;
1624
1625                         if (skb) {
1626                                 dev_kfree_skb(skb);
1627                                 dev->tx_skb[slot] = NULL;
1628                         }
1629                         slot = (slot + 1) % NUM_TX_BUFF;
1630
1631                         if (unlikely(ctrl & bad_mask))
1632                                 emac_parse_tx_error(dev, ctrl);
1633
1634                         if (--dev->tx_cnt)
1635                                 goto again;
1636                 }
1637                 if (n) {
1638                         dev->ack_slot = slot;
1639                         if (netif_queue_stopped(dev->ndev) &&
1640                             dev->tx_cnt < EMAC_TX_WAKEUP_THRESH)
1641                                 netif_wake_queue(dev->ndev);
1642
1643                         DBG2(dev, "tx %d pkts" NL, n);
1644                 }
1645         }
1646         netif_tx_unlock_bh(dev->ndev);
1647 }
1648
1649 static inline void emac_recycle_rx_skb(struct emac_instance *dev, int slot,
1650                                        int len)
1651 {
1652         struct sk_buff *skb = dev->rx_skb[slot];
1653
1654         DBG2(dev, "recycle %d %d" NL, slot, len);
1655
1656         if (len)
1657                 dma_map_single(&dev->ofdev->dev, skb->data - NET_IP_ALIGN,
1658                                SKB_DATA_ALIGN(len + NET_IP_ALIGN),
1659                                DMA_FROM_DEVICE);
1660
1661         dev->rx_desc[slot].data_len = 0;
1662         wmb();
1663         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1664             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1665 }
1666
1667 static void emac_parse_rx_error(struct emac_instance *dev, u16 ctrl)
1668 {
1669         struct emac_error_stats *st = &dev->estats;
1670
1671         DBG(dev, "BD RX error %04x" NL, ctrl);
1672
1673         ++st->rx_bd_errors;
1674         if (ctrl & EMAC_RX_ST_OE)
1675                 ++st->rx_bd_overrun;
1676         if (ctrl & EMAC_RX_ST_BP)
1677                 ++st->rx_bd_bad_packet;
1678         if (ctrl & EMAC_RX_ST_RP)
1679                 ++st->rx_bd_runt_packet;
1680         if (ctrl & EMAC_RX_ST_SE)
1681                 ++st->rx_bd_short_event;
1682         if (ctrl & EMAC_RX_ST_AE)
1683                 ++st->rx_bd_alignment_error;
1684         if (ctrl & EMAC_RX_ST_BFCS)
1685                 ++st->rx_bd_bad_fcs;
1686         if (ctrl & EMAC_RX_ST_PTL)
1687                 ++st->rx_bd_packet_too_long;
1688         if (ctrl & EMAC_RX_ST_ORE)
1689                 ++st->rx_bd_out_of_range;
1690         if (ctrl & EMAC_RX_ST_IRE)
1691                 ++st->rx_bd_in_range;
1692 }
1693
1694 static inline void emac_rx_csum(struct emac_instance *dev,
1695                                 struct sk_buff *skb, u16 ctrl)
1696 {
1697 #ifdef CONFIG_IBM_EMAC_TAH
1698         if (!ctrl && dev->tah_dev) {
1699                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1700                 ++dev->stats.rx_packets_csum;
1701         }
1702 #endif
1703 }
1704
1705 static inline int emac_rx_sg_append(struct emac_instance *dev, int slot)
1706 {
1707         if (likely(dev->rx_sg_skb != NULL)) {
1708                 int len = dev->rx_desc[slot].data_len;
1709                 int tot_len = dev->rx_sg_skb->len + len;
1710
1711                 if (unlikely(tot_len + NET_IP_ALIGN > dev->rx_skb_size)) {
1712                         ++dev->estats.rx_dropped_mtu;
1713                         dev_kfree_skb(dev->rx_sg_skb);
1714                         dev->rx_sg_skb = NULL;
1715                 } else {
1716                         memcpy(skb_tail_pointer(dev->rx_sg_skb),
1717                                          dev->rx_skb[slot]->data, len);
1718                         skb_put(dev->rx_sg_skb, len);
1719                         emac_recycle_rx_skb(dev, slot, len);
1720                         return 0;
1721                 }
1722         }
1723         emac_recycle_rx_skb(dev, slot, 0);
1724         return -1;
1725 }
1726
1727 /* NAPI poll context */
1728 static int emac_poll_rx(void *param, int budget)
1729 {
1730         struct emac_instance *dev = param;
1731         int slot = dev->rx_slot, received = 0;
1732
1733         DBG2(dev, "poll_rx(%d)" NL, budget);
1734
1735  again:
1736         while (budget > 0) {
1737                 int len;
1738                 struct sk_buff *skb;
1739                 u16 ctrl = dev->rx_desc[slot].ctrl;
1740
1741                 if (ctrl & MAL_RX_CTRL_EMPTY)
1742                         break;
1743
1744                 skb = dev->rx_skb[slot];
1745                 mb();
1746                 len = dev->rx_desc[slot].data_len;
1747
1748                 if (unlikely(!MAL_IS_SINGLE_RX(ctrl)))
1749                         goto sg;
1750
1751                 ctrl &= EMAC_BAD_RX_MASK;
1752                 if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1753                         emac_parse_rx_error(dev, ctrl);
1754                         ++dev->estats.rx_dropped_error;
1755                         emac_recycle_rx_skb(dev, slot, 0);
1756                         len = 0;
1757                         goto next;
1758                 }
1759
1760                 if (len < ETH_HLEN) {
1761                         ++dev->estats.rx_dropped_stack;
1762                         emac_recycle_rx_skb(dev, slot, len);
1763                         goto next;
1764                 }
1765
1766                 if (len && len < EMAC_RX_COPY_THRESH) {
1767                         struct sk_buff *copy_skb;
1768
1769                         copy_skb = napi_alloc_skb(&dev->mal->napi, len);
1770                         if (unlikely(!copy_skb))
1771                                 goto oom;
1772
1773                         memcpy(copy_skb->data - NET_IP_ALIGN,
1774                                skb->data - NET_IP_ALIGN,
1775                                len + NET_IP_ALIGN);
1776                         emac_recycle_rx_skb(dev, slot, len);
1777                         skb = copy_skb;
1778                 } else if (unlikely(emac_alloc_rx_skb_napi(dev, slot)))
1779                         goto oom;
1780
1781                 skb_put(skb, len);
1782         push_packet:
1783                 skb->protocol = eth_type_trans(skb, dev->ndev);
1784                 emac_rx_csum(dev, skb, ctrl);
1785
1786                 if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
1787                         ++dev->estats.rx_dropped_stack;
1788         next:
1789                 ++dev->stats.rx_packets;
1790         skip:
1791                 dev->stats.rx_bytes += len;
1792                 slot = (slot + 1) % NUM_RX_BUFF;
1793                 --budget;
1794                 ++received;
1795                 continue;
1796         sg:
1797                 if (ctrl & MAL_RX_CTRL_FIRST) {
1798                         BUG_ON(dev->rx_sg_skb);
1799                         if (unlikely(emac_alloc_rx_skb_napi(dev, slot))) {
1800                                 DBG(dev, "rx OOM %d" NL, slot);
1801                                 ++dev->estats.rx_dropped_oom;
1802                                 emac_recycle_rx_skb(dev, slot, 0);
1803                         } else {
1804                                 dev->rx_sg_skb = skb;
1805                                 skb_put(skb, len);
1806                         }
1807                 } else if (!emac_rx_sg_append(dev, slot) &&
1808                            (ctrl & MAL_RX_CTRL_LAST)) {
1809
1810                         skb = dev->rx_sg_skb;
1811                         dev->rx_sg_skb = NULL;
1812
1813                         ctrl &= EMAC_BAD_RX_MASK;
1814                         if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1815                                 emac_parse_rx_error(dev, ctrl);
1816                                 ++dev->estats.rx_dropped_error;
1817                                 dev_kfree_skb(skb);
1818                                 len = 0;
1819                         } else
1820                                 goto push_packet;
1821                 }
1822                 goto skip;
1823         oom:
1824                 DBG(dev, "rx OOM %d" NL, slot);
1825                 /* Drop the packet and recycle skb */
1826                 ++dev->estats.rx_dropped_oom;
1827                 emac_recycle_rx_skb(dev, slot, 0);
1828                 goto next;
1829         }
1830
1831         if (received) {
1832                 DBG2(dev, "rx %d BDs" NL, received);
1833                 dev->rx_slot = slot;
1834         }
1835
1836         if (unlikely(budget && test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags))) {
1837                 mb();
1838                 if (!(dev->rx_desc[slot].ctrl & MAL_RX_CTRL_EMPTY)) {
1839                         DBG2(dev, "rx restart" NL);
1840                         received = 0;
1841                         goto again;
1842                 }
1843
1844                 if (dev->rx_sg_skb) {
1845                         DBG2(dev, "dropping partial rx packet" NL);
1846                         ++dev->estats.rx_dropped_error;
1847                         dev_kfree_skb(dev->rx_sg_skb);
1848                         dev->rx_sg_skb = NULL;
1849                 }
1850
1851                 clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1852                 mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1853                 emac_rx_enable(dev);
1854                 dev->rx_slot = 0;
1855         }
1856         return received;
1857 }
1858
1859 /* NAPI poll context */
1860 static int emac_peek_rx(void *param)
1861 {
1862         struct emac_instance *dev = param;
1863
1864         return !(dev->rx_desc[dev->rx_slot].ctrl & MAL_RX_CTRL_EMPTY);
1865 }
1866
1867 /* NAPI poll context */
1868 static int emac_peek_rx_sg(void *param)
1869 {
1870         struct emac_instance *dev = param;
1871
1872         int slot = dev->rx_slot;
1873         while (1) {
1874                 u16 ctrl = dev->rx_desc[slot].ctrl;
1875                 if (ctrl & MAL_RX_CTRL_EMPTY)
1876                         return 0;
1877                 else if (ctrl & MAL_RX_CTRL_LAST)
1878                         return 1;
1879
1880                 slot = (slot + 1) % NUM_RX_BUFF;
1881
1882                 /* I'm just being paranoid here :) */
1883                 if (unlikely(slot == dev->rx_slot))
1884                         return 0;
1885         }
1886 }
1887
1888 /* Hard IRQ */
1889 static void emac_rxde(void *param)
1890 {
1891         struct emac_instance *dev = param;
1892
1893         ++dev->estats.rx_stopped;
1894         emac_rx_disable_async(dev);
1895 }
1896
1897 /* Hard IRQ */
1898 static irqreturn_t emac_irq(int irq, void *dev_instance)
1899 {
1900         struct emac_instance *dev = dev_instance;
1901         struct emac_regs __iomem *p = dev->emacp;
1902         struct emac_error_stats *st = &dev->estats;
1903         u32 isr;
1904
1905         spin_lock(&dev->lock);
1906
1907         isr = in_be32(&p->isr);
1908         out_be32(&p->isr, isr);
1909
1910         DBG(dev, "isr = %08x" NL, isr);
1911
1912         if (isr & EMAC4_ISR_TXPE)
1913                 ++st->tx_parity;
1914         if (isr & EMAC4_ISR_RXPE)
1915                 ++st->rx_parity;
1916         if (isr & EMAC4_ISR_TXUE)
1917                 ++st->tx_underrun;
1918         if (isr & EMAC4_ISR_RXOE)
1919                 ++st->rx_fifo_overrun;
1920         if (isr & EMAC_ISR_OVR)
1921                 ++st->rx_overrun;
1922         if (isr & EMAC_ISR_BP)
1923                 ++st->rx_bad_packet;
1924         if (isr & EMAC_ISR_RP)
1925                 ++st->rx_runt_packet;
1926         if (isr & EMAC_ISR_SE)
1927                 ++st->rx_short_event;
1928         if (isr & EMAC_ISR_ALE)
1929                 ++st->rx_alignment_error;
1930         if (isr & EMAC_ISR_BFCS)
1931                 ++st->rx_bad_fcs;
1932         if (isr & EMAC_ISR_PTLE)
1933                 ++st->rx_packet_too_long;
1934         if (isr & EMAC_ISR_ORE)
1935                 ++st->rx_out_of_range;
1936         if (isr & EMAC_ISR_IRE)
1937                 ++st->rx_in_range;
1938         if (isr & EMAC_ISR_SQE)
1939                 ++st->tx_sqe;
1940         if (isr & EMAC_ISR_TE)
1941                 ++st->tx_errors;
1942
1943         spin_unlock(&dev->lock);
1944
1945         return IRQ_HANDLED;
1946 }
1947
1948 static struct net_device_stats *emac_stats(struct net_device *ndev)
1949 {
1950         struct emac_instance *dev = netdev_priv(ndev);
1951         struct emac_stats *st = &dev->stats;
1952         struct emac_error_stats *est = &dev->estats;
1953         struct net_device_stats *nst = &ndev->stats;
1954         unsigned long flags;
1955
1956         DBG2(dev, "stats" NL);
1957
1958         /* Compute "legacy" statistics */
1959         spin_lock_irqsave(&dev->lock, flags);
1960         nst->rx_packets = (unsigned long)st->rx_packets;
1961         nst->rx_bytes = (unsigned long)st->rx_bytes;
1962         nst->tx_packets = (unsigned long)st->tx_packets;
1963         nst->tx_bytes = (unsigned long)st->tx_bytes;
1964         nst->rx_dropped = (unsigned long)(est->rx_dropped_oom +
1965                                           est->rx_dropped_error +
1966                                           est->rx_dropped_resize +
1967                                           est->rx_dropped_mtu);
1968         nst->tx_dropped = (unsigned long)est->tx_dropped;
1969
1970         nst->rx_errors = (unsigned long)est->rx_bd_errors;
1971         nst->rx_fifo_errors = (unsigned long)(est->rx_bd_overrun +
1972                                               est->rx_fifo_overrun +
1973                                               est->rx_overrun);
1974         nst->rx_frame_errors = (unsigned long)(est->rx_bd_alignment_error +
1975                                                est->rx_alignment_error);
1976         nst->rx_crc_errors = (unsigned long)(est->rx_bd_bad_fcs +
1977                                              est->rx_bad_fcs);
1978         nst->rx_length_errors = (unsigned long)(est->rx_bd_runt_packet +
1979                                                 est->rx_bd_short_event +
1980                                                 est->rx_bd_packet_too_long +
1981                                                 est->rx_bd_out_of_range +
1982                                                 est->rx_bd_in_range +
1983                                                 est->rx_runt_packet +
1984                                                 est->rx_short_event +
1985                                                 est->rx_packet_too_long +
1986                                                 est->rx_out_of_range +
1987                                                 est->rx_in_range);
1988
1989         nst->tx_errors = (unsigned long)(est->tx_bd_errors + est->tx_errors);
1990         nst->tx_fifo_errors = (unsigned long)(est->tx_bd_underrun +
1991                                               est->tx_underrun);
1992         nst->tx_carrier_errors = (unsigned long)est->tx_bd_carrier_loss;
1993         nst->collisions = (unsigned long)(est->tx_bd_excessive_deferral +
1994                                           est->tx_bd_excessive_collisions +
1995                                           est->tx_bd_late_collision +
1996                                           est->tx_bd_multple_collisions);
1997         spin_unlock_irqrestore(&dev->lock, flags);
1998         return nst;
1999 }
2000
2001 static struct mal_commac_ops emac_commac_ops = {
2002         .poll_tx = &emac_poll_tx,
2003         .poll_rx = &emac_poll_rx,
2004         .peek_rx = &emac_peek_rx,
2005         .rxde = &emac_rxde,
2006 };
2007
2008 static struct mal_commac_ops emac_commac_sg_ops = {
2009         .poll_tx = &emac_poll_tx,
2010         .poll_rx = &emac_poll_rx,
2011         .peek_rx = &emac_peek_rx_sg,
2012         .rxde = &emac_rxde,
2013 };
2014
2015 /* Ethtool support */
2016 static int emac_ethtool_get_link_ksettings(struct net_device *ndev,
2017                                            struct ethtool_link_ksettings *cmd)
2018 {
2019         struct emac_instance *dev = netdev_priv(ndev);
2020         u32 supported, advertising;
2021
2022         supported = dev->phy.features;
2023         cmd->base.port = PORT_MII;
2024         cmd->base.phy_address = dev->phy.address;
2025
2026         mutex_lock(&dev->link_lock);
2027         advertising = dev->phy.advertising;
2028         cmd->base.autoneg = dev->phy.autoneg;
2029         cmd->base.speed = dev->phy.speed;
2030         cmd->base.duplex = dev->phy.duplex;
2031         mutex_unlock(&dev->link_lock);
2032
2033         ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
2034                                                 supported);
2035         ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
2036                                                 advertising);
2037
2038         return 0;
2039 }
2040
2041 static int
2042 emac_ethtool_set_link_ksettings(struct net_device *ndev,
2043                                 const struct ethtool_link_ksettings *cmd)
2044 {
2045         struct emac_instance *dev = netdev_priv(ndev);
2046         u32 f = dev->phy.features;
2047         u32 advertising;
2048
2049         ethtool_convert_link_mode_to_legacy_u32(&advertising,
2050                                                 cmd->link_modes.advertising);
2051
2052         DBG(dev, "set_settings(%d, %d, %d, 0x%08x)" NL,
2053             cmd->base.autoneg, cmd->base.speed, cmd->base.duplex, advertising);
2054
2055         /* Basic sanity checks */
2056         if (dev->phy.address < 0)
2057                 return -EOPNOTSUPP;
2058         if (cmd->base.autoneg != AUTONEG_ENABLE &&
2059             cmd->base.autoneg != AUTONEG_DISABLE)
2060                 return -EINVAL;
2061         if (cmd->base.autoneg == AUTONEG_ENABLE && advertising == 0)
2062                 return -EINVAL;
2063         if (cmd->base.duplex != DUPLEX_HALF && cmd->base.duplex != DUPLEX_FULL)
2064                 return -EINVAL;
2065
2066         if (cmd->base.autoneg == AUTONEG_DISABLE) {
2067                 switch (cmd->base.speed) {
2068                 case SPEED_10:
2069                         if (cmd->base.duplex == DUPLEX_HALF &&
2070                             !(f & SUPPORTED_10baseT_Half))
2071                                 return -EINVAL;
2072                         if (cmd->base.duplex == DUPLEX_FULL &&
2073                             !(f & SUPPORTED_10baseT_Full))
2074                                 return -EINVAL;
2075                         break;
2076                 case SPEED_100:
2077                         if (cmd->base.duplex == DUPLEX_HALF &&
2078                             !(f & SUPPORTED_100baseT_Half))
2079                                 return -EINVAL;
2080                         if (cmd->base.duplex == DUPLEX_FULL &&
2081                             !(f & SUPPORTED_100baseT_Full))
2082                                 return -EINVAL;
2083                         break;
2084                 case SPEED_1000:
2085                         if (cmd->base.duplex == DUPLEX_HALF &&
2086                             !(f & SUPPORTED_1000baseT_Half))
2087                                 return -EINVAL;
2088                         if (cmd->base.duplex == DUPLEX_FULL &&
2089                             !(f & SUPPORTED_1000baseT_Full))
2090                                 return -EINVAL;
2091                         break;
2092                 default:
2093                         return -EINVAL;
2094                 }
2095
2096                 mutex_lock(&dev->link_lock);
2097                 dev->phy.def->ops->setup_forced(&dev->phy, cmd->base.speed,
2098                                                 cmd->base.duplex);
2099                 mutex_unlock(&dev->link_lock);
2100
2101         } else {
2102                 if (!(f & SUPPORTED_Autoneg))
2103                         return -EINVAL;
2104
2105                 mutex_lock(&dev->link_lock);
2106                 dev->phy.def->ops->setup_aneg(&dev->phy,
2107                                               (advertising & f) |
2108                                               (dev->phy.advertising &
2109                                                (ADVERTISED_Pause |
2110                                                 ADVERTISED_Asym_Pause)));
2111                 mutex_unlock(&dev->link_lock);
2112         }
2113         emac_force_link_update(dev);
2114
2115         return 0;
2116 }
2117
2118 static void
2119 emac_ethtool_get_ringparam(struct net_device *ndev,
2120                            struct ethtool_ringparam *rp,
2121                            struct kernel_ethtool_ringparam *kernel_rp,
2122                            struct netlink_ext_ack *extack)
2123 {
2124         rp->rx_max_pending = rp->rx_pending = NUM_RX_BUFF;
2125         rp->tx_max_pending = rp->tx_pending = NUM_TX_BUFF;
2126 }
2127
2128 static void emac_ethtool_get_pauseparam(struct net_device *ndev,
2129                                         struct ethtool_pauseparam *pp)
2130 {
2131         struct emac_instance *dev = netdev_priv(ndev);
2132
2133         mutex_lock(&dev->link_lock);
2134         if ((dev->phy.features & SUPPORTED_Autoneg) &&
2135             (dev->phy.advertising & (ADVERTISED_Pause | ADVERTISED_Asym_Pause)))
2136                 pp->autoneg = 1;
2137
2138         if (dev->phy.duplex == DUPLEX_FULL) {
2139                 if (dev->phy.pause)
2140                         pp->rx_pause = pp->tx_pause = 1;
2141                 else if (dev->phy.asym_pause)
2142                         pp->tx_pause = 1;
2143         }
2144         mutex_unlock(&dev->link_lock);
2145 }
2146
2147 static int emac_get_regs_len(struct emac_instance *dev)
2148 {
2149                 return sizeof(struct emac_ethtool_regs_subhdr) +
2150                         sizeof(struct emac_regs);
2151 }
2152
2153 static int emac_ethtool_get_regs_len(struct net_device *ndev)
2154 {
2155         struct emac_instance *dev = netdev_priv(ndev);
2156         int size;
2157
2158         size = sizeof(struct emac_ethtool_regs_hdr) +
2159                 emac_get_regs_len(dev) + mal_get_regs_len(dev->mal);
2160         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2161                 size += zmii_get_regs_len(dev->zmii_dev);
2162         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2163                 size += rgmii_get_regs_len(dev->rgmii_dev);
2164         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2165                 size += tah_get_regs_len(dev->tah_dev);
2166
2167         return size;
2168 }
2169
2170 static void *emac_dump_regs(struct emac_instance *dev, void *buf)
2171 {
2172         struct emac_ethtool_regs_subhdr *hdr = buf;
2173
2174         hdr->index = dev->cell_index;
2175         if (emac_has_feature(dev, EMAC_FTR_EMAC4SYNC)) {
2176                 hdr->version = EMAC4SYNC_ETHTOOL_REGS_VER;
2177         } else if (emac_has_feature(dev, EMAC_FTR_EMAC4)) {
2178                 hdr->version = EMAC4_ETHTOOL_REGS_VER;
2179         } else {
2180                 hdr->version = EMAC_ETHTOOL_REGS_VER;
2181         }
2182         memcpy_fromio(hdr + 1, dev->emacp, sizeof(struct emac_regs));
2183         return (void *)(hdr + 1) + sizeof(struct emac_regs);
2184 }
2185
2186 static void emac_ethtool_get_regs(struct net_device *ndev,
2187                                   struct ethtool_regs *regs, void *buf)
2188 {
2189         struct emac_instance *dev = netdev_priv(ndev);
2190         struct emac_ethtool_regs_hdr *hdr = buf;
2191
2192         hdr->components = 0;
2193         buf = hdr + 1;
2194
2195         buf = mal_dump_regs(dev->mal, buf);
2196         buf = emac_dump_regs(dev, buf);
2197         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII)) {
2198                 hdr->components |= EMAC_ETHTOOL_REGS_ZMII;
2199                 buf = zmii_dump_regs(dev->zmii_dev, buf);
2200         }
2201         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII)) {
2202                 hdr->components |= EMAC_ETHTOOL_REGS_RGMII;
2203                 buf = rgmii_dump_regs(dev->rgmii_dev, buf);
2204         }
2205         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH)) {
2206                 hdr->components |= EMAC_ETHTOOL_REGS_TAH;
2207                 buf = tah_dump_regs(dev->tah_dev, buf);
2208         }
2209 }
2210
2211 static int emac_ethtool_nway_reset(struct net_device *ndev)
2212 {
2213         struct emac_instance *dev = netdev_priv(ndev);
2214         int res = 0;
2215
2216         DBG(dev, "nway_reset" NL);
2217
2218         if (dev->phy.address < 0)
2219                 return -EOPNOTSUPP;
2220
2221         mutex_lock(&dev->link_lock);
2222         if (!dev->phy.autoneg) {
2223                 res = -EINVAL;
2224                 goto out;
2225         }
2226
2227         dev->phy.def->ops->setup_aneg(&dev->phy, dev->phy.advertising);
2228  out:
2229         mutex_unlock(&dev->link_lock);
2230         emac_force_link_update(dev);
2231         return res;
2232 }
2233
2234 static int emac_ethtool_get_sset_count(struct net_device *ndev, int stringset)
2235 {
2236         if (stringset == ETH_SS_STATS)
2237                 return EMAC_ETHTOOL_STATS_COUNT;
2238         else
2239                 return -EINVAL;
2240 }
2241
2242 static void emac_ethtool_get_strings(struct net_device *ndev, u32 stringset,
2243                                      u8 * buf)
2244 {
2245         if (stringset == ETH_SS_STATS)
2246                 memcpy(buf, &emac_stats_keys, sizeof(emac_stats_keys));
2247 }
2248
2249 static void emac_ethtool_get_ethtool_stats(struct net_device *ndev,
2250                                            struct ethtool_stats *estats,
2251                                            u64 * tmp_stats)
2252 {
2253         struct emac_instance *dev = netdev_priv(ndev);
2254
2255         memcpy(tmp_stats, &dev->stats, sizeof(dev->stats));
2256         tmp_stats += sizeof(dev->stats) / sizeof(u64);
2257         memcpy(tmp_stats, &dev->estats, sizeof(dev->estats));
2258 }
2259
2260 static void emac_ethtool_get_drvinfo(struct net_device *ndev,
2261                                      struct ethtool_drvinfo *info)
2262 {
2263         struct emac_instance *dev = netdev_priv(ndev);
2264
2265         strscpy(info->driver, "ibm_emac", sizeof(info->driver));
2266         strscpy(info->version, DRV_VERSION, sizeof(info->version));
2267         snprintf(info->bus_info, sizeof(info->bus_info), "PPC 4xx EMAC-%d %pOF",
2268                  dev->cell_index, dev->ofdev->dev.of_node);
2269 }
2270
2271 static const struct ethtool_ops emac_ethtool_ops = {
2272         .get_drvinfo = emac_ethtool_get_drvinfo,
2273
2274         .get_regs_len = emac_ethtool_get_regs_len,
2275         .get_regs = emac_ethtool_get_regs,
2276
2277         .nway_reset = emac_ethtool_nway_reset,
2278
2279         .get_ringparam = emac_ethtool_get_ringparam,
2280         .get_pauseparam = emac_ethtool_get_pauseparam,
2281
2282         .get_strings = emac_ethtool_get_strings,
2283         .get_sset_count = emac_ethtool_get_sset_count,
2284         .get_ethtool_stats = emac_ethtool_get_ethtool_stats,
2285
2286         .get_link = ethtool_op_get_link,
2287         .get_link_ksettings = emac_ethtool_get_link_ksettings,
2288         .set_link_ksettings = emac_ethtool_set_link_ksettings,
2289 };
2290
2291 static int emac_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2292 {
2293         struct emac_instance *dev = netdev_priv(ndev);
2294         struct mii_ioctl_data *data = if_mii(rq);
2295
2296         DBG(dev, "ioctl %08x" NL, cmd);
2297
2298         if (dev->phy.address < 0)
2299                 return -EOPNOTSUPP;
2300
2301         switch (cmd) {
2302         case SIOCGMIIPHY:
2303                 data->phy_id = dev->phy.address;
2304                 fallthrough;
2305         case SIOCGMIIREG:
2306                 data->val_out = emac_mdio_read(ndev, dev->phy.address,
2307                                                data->reg_num);
2308                 return 0;
2309
2310         case SIOCSMIIREG:
2311                 emac_mdio_write(ndev, dev->phy.address, data->reg_num,
2312                                 data->val_in);
2313                 return 0;
2314         default:
2315                 return -EOPNOTSUPP;
2316         }
2317 }
2318
2319 struct emac_depentry {
2320         u32                     phandle;
2321         struct device_node      *node;
2322         struct platform_device  *ofdev;
2323         void                    *drvdata;
2324 };
2325
2326 #define EMAC_DEP_MAL_IDX        0
2327 #define EMAC_DEP_ZMII_IDX       1
2328 #define EMAC_DEP_RGMII_IDX      2
2329 #define EMAC_DEP_TAH_IDX        3
2330 #define EMAC_DEP_MDIO_IDX       4
2331 #define EMAC_DEP_PREV_IDX       5
2332 #define EMAC_DEP_COUNT          6
2333
2334 static int emac_check_deps(struct emac_instance *dev,
2335                            struct emac_depentry *deps)
2336 {
2337         int i, there = 0;
2338         struct device_node *np;
2339
2340         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2341                 /* no dependency on that item, allright */
2342                 if (deps[i].phandle == 0) {
2343                         there++;
2344                         continue;
2345                 }
2346                 /* special case for blist as the dependency might go away */
2347                 if (i == EMAC_DEP_PREV_IDX) {
2348                         np = *(dev->blist - 1);
2349                         if (np == NULL) {
2350                                 deps[i].phandle = 0;
2351                                 there++;
2352                                 continue;
2353                         }
2354                         if (deps[i].node == NULL)
2355                                 deps[i].node = of_node_get(np);
2356                 }
2357                 if (deps[i].node == NULL)
2358                         deps[i].node = of_find_node_by_phandle(deps[i].phandle);
2359                 if (deps[i].node == NULL)
2360                         continue;
2361                 if (deps[i].ofdev == NULL)
2362                         deps[i].ofdev = of_find_device_by_node(deps[i].node);
2363                 if (deps[i].ofdev == NULL)
2364                         continue;
2365                 if (deps[i].drvdata == NULL)
2366                         deps[i].drvdata = platform_get_drvdata(deps[i].ofdev);
2367                 if (deps[i].drvdata != NULL)
2368                         there++;
2369         }
2370         if (there != EMAC_DEP_COUNT)
2371                 return -EPROBE_DEFER;
2372         return 0;
2373 }
2374
2375 static void emac_put_deps(struct emac_instance *dev)
2376 {
2377         platform_device_put(dev->mal_dev);
2378         platform_device_put(dev->zmii_dev);
2379         platform_device_put(dev->rgmii_dev);
2380         platform_device_put(dev->mdio_dev);
2381         platform_device_put(dev->tah_dev);
2382 }
2383
2384 static int emac_wait_deps(struct emac_instance *dev)
2385 {
2386         struct emac_depentry deps[EMAC_DEP_COUNT];
2387         int i, err;
2388
2389         memset(&deps, 0, sizeof(deps));
2390
2391         deps[EMAC_DEP_MAL_IDX].phandle = dev->mal_ph;
2392         deps[EMAC_DEP_ZMII_IDX].phandle = dev->zmii_ph;
2393         deps[EMAC_DEP_RGMII_IDX].phandle = dev->rgmii_ph;
2394         if (dev->tah_ph)
2395                 deps[EMAC_DEP_TAH_IDX].phandle = dev->tah_ph;
2396         if (dev->mdio_ph)
2397                 deps[EMAC_DEP_MDIO_IDX].phandle = dev->mdio_ph;
2398         if (dev->blist && dev->blist > emac_boot_list)
2399                 deps[EMAC_DEP_PREV_IDX].phandle = 0xffffffffu;
2400         err = emac_check_deps(dev, deps);
2401         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2402                 of_node_put(deps[i].node);
2403                 if (err)
2404                         platform_device_put(deps[i].ofdev);
2405         }
2406         if (!err) {
2407                 dev->mal_dev = deps[EMAC_DEP_MAL_IDX].ofdev;
2408                 dev->zmii_dev = deps[EMAC_DEP_ZMII_IDX].ofdev;
2409                 dev->rgmii_dev = deps[EMAC_DEP_RGMII_IDX].ofdev;
2410                 dev->tah_dev = deps[EMAC_DEP_TAH_IDX].ofdev;
2411                 dev->mdio_dev = deps[EMAC_DEP_MDIO_IDX].ofdev;
2412         }
2413         platform_device_put(deps[EMAC_DEP_PREV_IDX].ofdev);
2414         return err;
2415 }
2416
2417 static int emac_read_uint_prop(struct device_node *np, const char *name,
2418                                u32 *val, int fatal)
2419 {
2420         int err;
2421
2422         err = of_property_read_u32(np, name, val);
2423         if (err) {
2424                 if (fatal)
2425                         pr_err("%pOF: missing %s property", np, name);
2426                 return err;
2427         }
2428         return 0;
2429 }
2430
2431 static void emac_adjust_link(struct net_device *ndev)
2432 {
2433         struct emac_instance *dev = netdev_priv(ndev);
2434         struct phy_device *phy = ndev->phydev;
2435
2436         dev->phy.autoneg = phy->autoneg;
2437         dev->phy.speed = phy->speed;
2438         dev->phy.duplex = phy->duplex;
2439         dev->phy.pause = phy->pause;
2440         dev->phy.asym_pause = phy->asym_pause;
2441         ethtool_convert_link_mode_to_legacy_u32(&dev->phy.advertising,
2442                                                 phy->advertising);
2443 }
2444
2445 static int emac_mii_bus_read(struct mii_bus *bus, int addr, int regnum)
2446 {
2447         int ret = emac_mdio_read(bus->priv, addr, regnum);
2448         /* This is a workaround for powered down ports/phys.
2449          * In the wild, this was seen on the Cisco Meraki MX60(W).
2450          * This hardware disables ports as part of the handoff
2451          * procedure. Accessing the ports will lead to errors
2452          * (-ETIMEDOUT, -EREMOTEIO) that do more harm than good.
2453          */
2454         return ret < 0 ? 0xffff : ret;
2455 }
2456
2457 static int emac_mii_bus_write(struct mii_bus *bus, int addr,
2458                               int regnum, u16 val)
2459 {
2460         emac_mdio_write(bus->priv, addr, regnum, val);
2461         return 0;
2462 }
2463
2464 static int emac_mii_bus_reset(struct mii_bus *bus)
2465 {
2466         struct emac_instance *dev = netdev_priv(bus->priv);
2467
2468         return emac_reset(dev);
2469 }
2470
2471 static int emac_mdio_phy_start_aneg(struct mii_phy *phy,
2472                                     struct phy_device *phy_dev)
2473 {
2474         phy_dev->autoneg = phy->autoneg;
2475         phy_dev->speed = phy->speed;
2476         phy_dev->duplex = phy->duplex;
2477         ethtool_convert_legacy_u32_to_link_mode(phy_dev->advertising,
2478                                                 phy->advertising);
2479         return phy_start_aneg(phy_dev);
2480 }
2481
2482 static int emac_mdio_setup_aneg(struct mii_phy *phy, u32 advertise)
2483 {
2484         struct net_device *ndev = phy->dev;
2485
2486         phy->autoneg = AUTONEG_ENABLE;
2487         phy->advertising = advertise;
2488         return emac_mdio_phy_start_aneg(phy, ndev->phydev);
2489 }
2490
2491 static int emac_mdio_setup_forced(struct mii_phy *phy, int speed, int fd)
2492 {
2493         struct net_device *ndev = phy->dev;
2494
2495         phy->autoneg = AUTONEG_DISABLE;
2496         phy->speed = speed;
2497         phy->duplex = fd;
2498         return emac_mdio_phy_start_aneg(phy, ndev->phydev);
2499 }
2500
2501 static int emac_mdio_poll_link(struct mii_phy *phy)
2502 {
2503         struct net_device *ndev = phy->dev;
2504         struct emac_instance *dev = netdev_priv(ndev);
2505         int res;
2506
2507         res = phy_read_status(ndev->phydev);
2508         if (res) {
2509                 dev_err(&dev->ofdev->dev, "link update failed (%d).", res);
2510                 return ethtool_op_get_link(ndev);
2511         }
2512
2513         return ndev->phydev->link;
2514 }
2515
2516 static int emac_mdio_read_link(struct mii_phy *phy)
2517 {
2518         struct net_device *ndev = phy->dev;
2519         struct phy_device *phy_dev = ndev->phydev;
2520         int res;
2521
2522         res = phy_read_status(phy_dev);
2523         if (res)
2524                 return res;
2525
2526         phy->speed = phy_dev->speed;
2527         phy->duplex = phy_dev->duplex;
2528         phy->pause = phy_dev->pause;
2529         phy->asym_pause = phy_dev->asym_pause;
2530         return 0;
2531 }
2532
2533 static int emac_mdio_init_phy(struct mii_phy *phy)
2534 {
2535         struct net_device *ndev = phy->dev;
2536
2537         phy_start(ndev->phydev);
2538         return phy_init_hw(ndev->phydev);
2539 }
2540
2541 static const struct mii_phy_ops emac_dt_mdio_phy_ops = {
2542         .init           = emac_mdio_init_phy,
2543         .setup_aneg     = emac_mdio_setup_aneg,
2544         .setup_forced   = emac_mdio_setup_forced,
2545         .poll_link      = emac_mdio_poll_link,
2546         .read_link      = emac_mdio_read_link,
2547 };
2548
2549 static int emac_dt_mdio_probe(struct emac_instance *dev)
2550 {
2551         struct device_node *mii_np;
2552         struct mii_bus *bus;
2553         int res;
2554
2555         mii_np = of_get_child_by_name(dev->ofdev->dev.of_node, "mdio");
2556         if (!mii_np) {
2557                 dev_err(&dev->ofdev->dev, "no mdio definition found.");
2558                 return -ENODEV;
2559         }
2560
2561         if (!of_device_is_available(mii_np)) {
2562                 res = -ENODEV;
2563                 goto put_node;
2564         }
2565
2566         bus = devm_mdiobus_alloc(&dev->ofdev->dev);
2567         if (!bus) {
2568                 res = -ENOMEM;
2569                 goto put_node;
2570         }
2571
2572         bus->priv = dev->ndev;
2573         bus->parent = dev->ndev->dev.parent;
2574         bus->name = "emac_mdio";
2575         bus->read = &emac_mii_bus_read;
2576         bus->write = &emac_mii_bus_write;
2577         bus->reset = &emac_mii_bus_reset;
2578         snprintf(bus->id, MII_BUS_ID_SIZE, "%s", dev->ofdev->name);
2579         res = devm_of_mdiobus_register(&dev->ofdev->dev, bus, mii_np);
2580         if (res) {
2581                 dev_err(&dev->ofdev->dev, "cannot register MDIO bus %s (%d)",
2582                         bus->name, res);
2583         }
2584
2585  put_node:
2586         of_node_put(mii_np);
2587         return res;
2588 }
2589
2590 static int emac_dt_phy_connect(struct emac_instance *dev,
2591                                struct device_node *phy_handle)
2592 {
2593         struct phy_device *phy_dev;
2594
2595         dev->phy.def = devm_kzalloc(&dev->ofdev->dev, sizeof(*dev->phy.def),
2596                                     GFP_KERNEL);
2597         if (!dev->phy.def)
2598                 return -ENOMEM;
2599
2600         phy_dev = of_phy_connect(dev->ndev, phy_handle, &emac_adjust_link, 0,
2601                                  dev->phy_mode);
2602         if (!phy_dev) {
2603                 dev_err(&dev->ofdev->dev, "failed to connect to PHY.\n");
2604                 return -ENODEV;
2605         }
2606
2607         dev->phy.def->phy_id = phy_dev->drv->phy_id;
2608         dev->phy.def->phy_id_mask = phy_dev->drv->phy_id_mask;
2609         dev->phy.def->name = phy_dev->drv->name;
2610         dev->phy.def->ops = &emac_dt_mdio_phy_ops;
2611         ethtool_convert_link_mode_to_legacy_u32(&dev->phy.features,
2612                                                 phy_dev->supported);
2613         dev->phy.address = phy_dev->mdio.addr;
2614         dev->phy.mode = phy_dev->interface;
2615         return 0;
2616 }
2617
2618 static int emac_dt_phy_probe(struct emac_instance *dev)
2619 {
2620         struct device_node *np = dev->ofdev->dev.of_node;
2621         struct device_node *phy_handle;
2622         int res = 1;
2623
2624         phy_handle = of_parse_phandle(np, "phy-handle", 0);
2625
2626         if (phy_handle) {
2627                 res = emac_dt_mdio_probe(dev);
2628                 if (!res) {
2629                         res = emac_dt_phy_connect(dev, phy_handle);
2630                 }
2631         }
2632
2633         of_node_put(phy_handle);
2634         return res;
2635 }
2636
2637 static int emac_init_phy(struct emac_instance *dev)
2638 {
2639         struct device_node *np = dev->ofdev->dev.of_node;
2640         struct net_device *ndev = dev->ndev;
2641         u32 phy_map, adv;
2642         int i;
2643
2644         dev->phy.dev = ndev;
2645         dev->phy.mode = dev->phy_mode;
2646
2647         /* PHY-less configuration. */
2648         if ((dev->phy_address == 0xffffffff && dev->phy_map == 0xffffffff) ||
2649             of_phy_is_fixed_link(np)) {
2650                 emac_reset(dev);
2651
2652                 /* PHY-less configuration. */
2653                 dev->phy.address = -1;
2654                 dev->phy.features = SUPPORTED_MII;
2655                 if (emac_phy_supports_gige(dev->phy_mode))
2656                         dev->phy.features |= SUPPORTED_1000baseT_Full;
2657                 else
2658                         dev->phy.features |= SUPPORTED_100baseT_Full;
2659                 dev->phy.pause = 1;
2660
2661                 if (of_phy_is_fixed_link(np)) {
2662                         int res = emac_dt_mdio_probe(dev);
2663
2664                         if (res)
2665                                 return res;
2666
2667                         res = of_phy_register_fixed_link(np);
2668                         ndev->phydev = of_phy_find_device(np);
2669                         if (res || !ndev->phydev)
2670                                 return res ? res : -EINVAL;
2671                         emac_adjust_link(dev->ndev);
2672                         put_device(&ndev->phydev->mdio.dev);
2673                 }
2674                 return 0;
2675         }
2676
2677         mutex_lock(&emac_phy_map_lock);
2678         phy_map = dev->phy_map | busy_phy_map;
2679
2680         DBG(dev, "PHY maps %08x %08x" NL, dev->phy_map, busy_phy_map);
2681
2682         dev->phy.mdio_read = emac_mdio_read;
2683         dev->phy.mdio_write = emac_mdio_write;
2684
2685         /* Enable internal clock source */
2686 #ifdef CONFIG_PPC_DCR_NATIVE
2687         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2688                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2689 #endif
2690         /* PHY clock workaround */
2691         emac_rx_clk_tx(dev);
2692
2693         /* Enable internal clock source on 440GX*/
2694 #ifdef CONFIG_PPC_DCR_NATIVE
2695         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2696                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2697 #endif
2698         /* Configure EMAC with defaults so we can at least use MDIO
2699          * This is needed mostly for 440GX
2700          */
2701         if (emac_phy_gpcs(dev->phy.mode)) {
2702                 /* XXX
2703                  * Make GPCS PHY address equal to EMAC index.
2704                  * We probably should take into account busy_phy_map
2705                  * and/or phy_map here.
2706                  *
2707                  * Note that the busy_phy_map is currently global
2708                  * while it should probably be per-ASIC...
2709                  */
2710                 dev->phy.gpcs_address = dev->gpcs_address;
2711                 if (dev->phy.gpcs_address == 0xffffffff)
2712                         dev->phy.address = dev->cell_index;
2713         }
2714
2715         emac_configure(dev);
2716
2717         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII)) {
2718                 int res = emac_dt_phy_probe(dev);
2719
2720                 switch (res) {
2721                 case 1:
2722                         /* No phy-handle property configured.
2723                          * Continue with the existing phy probe
2724                          * and setup code.
2725                          */
2726                         break;
2727
2728                 case 0:
2729                         mutex_unlock(&emac_phy_map_lock);
2730                         goto init_phy;
2731
2732                 default:
2733                         mutex_unlock(&emac_phy_map_lock);
2734                         dev_err(&dev->ofdev->dev, "failed to attach dt phy (%d).\n",
2735                                 res);
2736                         return res;
2737                 }
2738         }
2739
2740         if (dev->phy_address != 0xffffffff)
2741                 phy_map = ~(1 << dev->phy_address);
2742
2743         for (i = 0; i < 0x20; phy_map >>= 1, ++i)
2744                 if (!(phy_map & 1)) {
2745                         int r;
2746                         busy_phy_map |= 1 << i;
2747
2748                         /* Quick check if there is a PHY at the address */
2749                         r = emac_mdio_read(dev->ndev, i, MII_BMCR);
2750                         if (r == 0xffff || r < 0)
2751                                 continue;
2752                         if (!emac_mii_phy_probe(&dev->phy, i))
2753                                 break;
2754                 }
2755
2756         /* Enable external clock source */
2757 #ifdef CONFIG_PPC_DCR_NATIVE
2758         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2759                 dcri_clrset(SDR0, SDR0_MFR, SDR0_MFR_ECS, 0);
2760 #endif
2761         mutex_unlock(&emac_phy_map_lock);
2762         if (i == 0x20) {
2763                 printk(KERN_WARNING "%pOF: can't find PHY!\n", np);
2764                 return -ENXIO;
2765         }
2766
2767  init_phy:
2768         /* Init PHY */
2769         if (dev->phy.def->ops->init)
2770                 dev->phy.def->ops->init(&dev->phy);
2771
2772         /* Disable any PHY features not supported by the platform */
2773         dev->phy.def->features &= ~dev->phy_feat_exc;
2774         dev->phy.features &= ~dev->phy_feat_exc;
2775
2776         /* Setup initial link parameters */
2777         if (dev->phy.features & SUPPORTED_Autoneg) {
2778                 adv = dev->phy.features;
2779                 if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x))
2780                         adv |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
2781                 /* Restart autonegotiation */
2782                 dev->phy.def->ops->setup_aneg(&dev->phy, adv);
2783         } else {
2784                 u32 f = dev->phy.def->features;
2785                 int speed = SPEED_10, fd = DUPLEX_HALF;
2786
2787                 /* Select highest supported speed/duplex */
2788                 if (f & SUPPORTED_1000baseT_Full) {
2789                         speed = SPEED_1000;
2790                         fd = DUPLEX_FULL;
2791                 } else if (f & SUPPORTED_1000baseT_Half)
2792                         speed = SPEED_1000;
2793                 else if (f & SUPPORTED_100baseT_Full) {
2794                         speed = SPEED_100;
2795                         fd = DUPLEX_FULL;
2796                 } else if (f & SUPPORTED_100baseT_Half)
2797                         speed = SPEED_100;
2798                 else if (f & SUPPORTED_10baseT_Full)
2799                         fd = DUPLEX_FULL;
2800
2801                 /* Force link parameters */
2802                 dev->phy.def->ops->setup_forced(&dev->phy, speed, fd);
2803         }
2804         return 0;
2805 }
2806
2807 static int emac_init_config(struct emac_instance *dev)
2808 {
2809         struct device_node *np = dev->ofdev->dev.of_node;
2810         int err;
2811
2812         /* Read config from device-tree */
2813         if (emac_read_uint_prop(np, "mal-device", &dev->mal_ph, 1))
2814                 return -ENXIO;
2815         if (emac_read_uint_prop(np, "mal-tx-channel", &dev->mal_tx_chan, 1))
2816                 return -ENXIO;
2817         if (emac_read_uint_prop(np, "mal-rx-channel", &dev->mal_rx_chan, 1))
2818                 return -ENXIO;
2819         if (emac_read_uint_prop(np, "cell-index", &dev->cell_index, 1))
2820                 return -ENXIO;
2821         if (emac_read_uint_prop(np, "max-frame-size", &dev->max_mtu, 0))
2822                 dev->max_mtu = ETH_DATA_LEN;
2823         if (emac_read_uint_prop(np, "rx-fifo-size", &dev->rx_fifo_size, 0))
2824                 dev->rx_fifo_size = 2048;
2825         if (emac_read_uint_prop(np, "tx-fifo-size", &dev->tx_fifo_size, 0))
2826                 dev->tx_fifo_size = 2048;
2827         if (emac_read_uint_prop(np, "rx-fifo-size-gige", &dev->rx_fifo_size_gige, 0))
2828                 dev->rx_fifo_size_gige = dev->rx_fifo_size;
2829         if (emac_read_uint_prop(np, "tx-fifo-size-gige", &dev->tx_fifo_size_gige, 0))
2830                 dev->tx_fifo_size_gige = dev->tx_fifo_size;
2831         if (emac_read_uint_prop(np, "phy-address", &dev->phy_address, 0))
2832                 dev->phy_address = 0xffffffff;
2833         if (emac_read_uint_prop(np, "phy-map", &dev->phy_map, 0))
2834                 dev->phy_map = 0xffffffff;
2835         if (emac_read_uint_prop(np, "gpcs-address", &dev->gpcs_address, 0))
2836                 dev->gpcs_address = 0xffffffff;
2837         if (emac_read_uint_prop(np->parent, "clock-frequency", &dev->opb_bus_freq, 1))
2838                 return -ENXIO;
2839         if (emac_read_uint_prop(np, "tah-device", &dev->tah_ph, 0))
2840                 dev->tah_ph = 0;
2841         if (emac_read_uint_prop(np, "tah-channel", &dev->tah_port, 0))
2842                 dev->tah_port = 0;
2843         if (emac_read_uint_prop(np, "mdio-device", &dev->mdio_ph, 0))
2844                 dev->mdio_ph = 0;
2845         if (emac_read_uint_prop(np, "zmii-device", &dev->zmii_ph, 0))
2846                 dev->zmii_ph = 0;
2847         if (emac_read_uint_prop(np, "zmii-channel", &dev->zmii_port, 0))
2848                 dev->zmii_port = 0xffffffff;
2849         if (emac_read_uint_prop(np, "rgmii-device", &dev->rgmii_ph, 0))
2850                 dev->rgmii_ph = 0;
2851         if (emac_read_uint_prop(np, "rgmii-channel", &dev->rgmii_port, 0))
2852                 dev->rgmii_port = 0xffffffff;
2853         if (emac_read_uint_prop(np, "fifo-entry-size", &dev->fifo_entry_size, 0))
2854                 dev->fifo_entry_size = 16;
2855         if (emac_read_uint_prop(np, "mal-burst-size", &dev->mal_burst_size, 0))
2856                 dev->mal_burst_size = 256;
2857
2858         /* PHY mode needs some decoding */
2859         err = of_get_phy_mode(np, &dev->phy_mode);
2860         if (err)
2861                 dev->phy_mode = PHY_INTERFACE_MODE_NA;
2862
2863         /* Check EMAC version */
2864         if (of_device_is_compatible(np, "ibm,emac4sync")) {
2865                 dev->features |= (EMAC_FTR_EMAC4 | EMAC_FTR_EMAC4SYNC);
2866                 if (of_device_is_compatible(np, "ibm,emac-460ex") ||
2867                     of_device_is_compatible(np, "ibm,emac-460gt"))
2868                         dev->features |= EMAC_FTR_460EX_PHY_CLK_FIX;
2869                 if (of_device_is_compatible(np, "ibm,emac-405ex") ||
2870                     of_device_is_compatible(np, "ibm,emac-405exr"))
2871                         dev->features |= EMAC_FTR_440EP_PHY_CLK_FIX;
2872                 if (of_device_is_compatible(np, "ibm,emac-apm821xx")) {
2873                         dev->features |= (EMAC_APM821XX_REQ_JUMBO_FRAME_SIZE |
2874                                           EMAC_FTR_APM821XX_NO_HALF_DUPLEX |
2875                                           EMAC_FTR_460EX_PHY_CLK_FIX);
2876                 }
2877         } else if (of_device_is_compatible(np, "ibm,emac4")) {
2878                 dev->features |= EMAC_FTR_EMAC4;
2879                 if (of_device_is_compatible(np, "ibm,emac-440gx"))
2880                         dev->features |= EMAC_FTR_440GX_PHY_CLK_FIX;
2881         } else {
2882                 if (of_device_is_compatible(np, "ibm,emac-440ep") ||
2883                     of_device_is_compatible(np, "ibm,emac-440gr"))
2884                         dev->features |= EMAC_FTR_440EP_PHY_CLK_FIX;
2885                 if (of_device_is_compatible(np, "ibm,emac-405ez")) {
2886 #ifdef CONFIG_IBM_EMAC_NO_FLOW_CTRL
2887                         dev->features |= EMAC_FTR_NO_FLOW_CONTROL_40x;
2888 #else
2889                         printk(KERN_ERR "%pOF: Flow control not disabled!\n",
2890                                         np);
2891                         return -ENXIO;
2892 #endif
2893                 }
2894
2895         }
2896
2897         /* Fixup some feature bits based on the device tree */
2898         if (of_property_read_bool(np, "has-inverted-stacr-oc"))
2899                 dev->features |= EMAC_FTR_STACR_OC_INVERT;
2900         if (of_property_read_bool(np, "has-new-stacr-staopc"))
2901                 dev->features |= EMAC_FTR_HAS_NEW_STACR;
2902
2903         /* CAB lacks the appropriate properties */
2904         if (of_device_is_compatible(np, "ibm,emac-axon"))
2905                 dev->features |= EMAC_FTR_HAS_NEW_STACR |
2906                         EMAC_FTR_STACR_OC_INVERT;
2907
2908         /* Enable TAH/ZMII/RGMII features as found */
2909         if (dev->tah_ph != 0) {
2910 #ifdef CONFIG_IBM_EMAC_TAH
2911                 dev->features |= EMAC_FTR_HAS_TAH;
2912 #else
2913                 printk(KERN_ERR "%pOF: TAH support not enabled !\n", np);
2914                 return -ENXIO;
2915 #endif
2916         }
2917
2918         if (dev->zmii_ph != 0) {
2919 #ifdef CONFIG_IBM_EMAC_ZMII
2920                 dev->features |= EMAC_FTR_HAS_ZMII;
2921 #else
2922                 printk(KERN_ERR "%pOF: ZMII support not enabled !\n", np);
2923                 return -ENXIO;
2924 #endif
2925         }
2926
2927         if (dev->rgmii_ph != 0) {
2928 #ifdef CONFIG_IBM_EMAC_RGMII
2929                 dev->features |= EMAC_FTR_HAS_RGMII;
2930 #else
2931                 printk(KERN_ERR "%pOF: RGMII support not enabled !\n", np);
2932                 return -ENXIO;
2933 #endif
2934         }
2935
2936         /* Read MAC-address */
2937         err = of_get_ethdev_address(np, dev->ndev);
2938         if (err)
2939                 return dev_err_probe(&dev->ofdev->dev, err,
2940                                      "Can't get valid [local-]mac-address from OF !\n");
2941
2942         /* IAHT and GAHT filter parameterization */
2943         if (emac_has_feature(dev, EMAC_FTR_EMAC4SYNC)) {
2944                 dev->xaht_slots_shift = EMAC4SYNC_XAHT_SLOTS_SHIFT;
2945                 dev->xaht_width_shift = EMAC4SYNC_XAHT_WIDTH_SHIFT;
2946         } else {
2947                 dev->xaht_slots_shift = EMAC4_XAHT_SLOTS_SHIFT;
2948                 dev->xaht_width_shift = EMAC4_XAHT_WIDTH_SHIFT;
2949         }
2950
2951         /* This should never happen */
2952         if (WARN_ON(EMAC_XAHT_REGS(dev) > EMAC_XAHT_MAX_REGS))
2953                 return -ENXIO;
2954
2955         DBG(dev, "features     : 0x%08x / 0x%08x\n", dev->features, EMAC_FTRS_POSSIBLE);
2956         DBG(dev, "tx_fifo_size : %d (%d gige)\n", dev->tx_fifo_size, dev->tx_fifo_size_gige);
2957         DBG(dev, "rx_fifo_size : %d (%d gige)\n", dev->rx_fifo_size, dev->rx_fifo_size_gige);
2958         DBG(dev, "max_mtu      : %d\n", dev->max_mtu);
2959         DBG(dev, "OPB freq     : %d\n", dev->opb_bus_freq);
2960
2961         return 0;
2962 }
2963
2964 static const struct net_device_ops emac_netdev_ops = {
2965         .ndo_open               = emac_open,
2966         .ndo_stop               = emac_close,
2967         .ndo_get_stats          = emac_stats,
2968         .ndo_set_rx_mode        = emac_set_multicast_list,
2969         .ndo_eth_ioctl          = emac_ioctl,
2970         .ndo_tx_timeout         = emac_tx_timeout,
2971         .ndo_validate_addr      = eth_validate_addr,
2972         .ndo_set_mac_address    = emac_set_mac_address,
2973         .ndo_start_xmit         = emac_start_xmit,
2974 };
2975
2976 static const struct net_device_ops emac_gige_netdev_ops = {
2977         .ndo_open               = emac_open,
2978         .ndo_stop               = emac_close,
2979         .ndo_get_stats          = emac_stats,
2980         .ndo_set_rx_mode        = emac_set_multicast_list,
2981         .ndo_eth_ioctl          = emac_ioctl,
2982         .ndo_tx_timeout         = emac_tx_timeout,
2983         .ndo_validate_addr      = eth_validate_addr,
2984         .ndo_set_mac_address    = emac_set_mac_address,
2985         .ndo_start_xmit         = emac_start_xmit_sg,
2986         .ndo_change_mtu         = emac_change_mtu,
2987 };
2988
2989 static int emac_probe(struct platform_device *ofdev)
2990 {
2991         struct net_device *ndev;
2992         struct emac_instance *dev;
2993         struct device_node *np = ofdev->dev.of_node;
2994         struct device_node **blist = NULL;
2995         int err, i;
2996
2997         /* Skip unused/unwired EMACS.  We leave the check for an unused
2998          * property here for now, but new flat device trees should set a
2999          * status property to "disabled" instead.
3000          */
3001         if (of_property_read_bool(np, "unused") || !of_device_is_available(np))
3002                 return -ENODEV;
3003
3004         /* Find ourselves in the bootlist if we are there */
3005         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
3006                 if (emac_boot_list[i] == np)
3007                         blist = &emac_boot_list[i];
3008
3009         /* Allocate our net_device structure */
3010         err = -ENOMEM;
3011         ndev = devm_alloc_etherdev(&ofdev->dev, sizeof(struct emac_instance));
3012         if (!ndev)
3013                 goto err_gone;
3014
3015         dev = netdev_priv(ndev);
3016         dev->ndev = ndev;
3017         dev->ofdev = ofdev;
3018         dev->blist = blist;
3019         SET_NETDEV_DEV(ndev, &ofdev->dev);
3020
3021         /* Initialize some embedded data structures */
3022         mutex_init(&dev->mdio_lock);
3023         mutex_init(&dev->link_lock);
3024         spin_lock_init(&dev->lock);
3025         INIT_WORK(&dev->reset_work, emac_reset_work);
3026
3027         /* Init various config data based on device-tree */
3028         err = emac_init_config(dev);
3029         if (err)
3030                 goto err_gone;
3031
3032         /* Get interrupts. EMAC irq is mandatory */
3033         dev->emac_irq = irq_of_parse_and_map(np, 0);
3034         if (!dev->emac_irq) {
3035                 printk(KERN_ERR "%pOF: Can't map main interrupt\n", np);
3036                 err = -ENODEV;
3037                 goto err_gone;
3038         }
3039
3040         /* Setup error IRQ handler */
3041         err = devm_request_irq(&ofdev->dev, dev->emac_irq, emac_irq, 0, "EMAC",
3042                                dev);
3043         if (err) {
3044                 dev_err_probe(&ofdev->dev, err, "failed to request IRQ %d",
3045                               dev->emac_irq);
3046                 goto err_gone;
3047         }
3048
3049         ndev->irq = dev->emac_irq;
3050
3051         /* Map EMAC regs */
3052         // TODO : platform_get_resource() and devm_ioremap_resource()
3053         dev->emacp = devm_of_iomap(&ofdev->dev, np, 0, NULL);
3054         if (!dev->emacp) {
3055                 dev_err(&ofdev->dev, "can't map device registers");
3056                 err = -ENOMEM;
3057                 goto err_gone;
3058         }
3059
3060         /* Wait for dependent devices */
3061         err = emac_wait_deps(dev);
3062         if (err)
3063                 goto err_gone;
3064         dev->mal = platform_get_drvdata(dev->mal_dev);
3065         if (dev->mdio_dev != NULL)
3066                 dev->mdio_instance = platform_get_drvdata(dev->mdio_dev);
3067
3068         /* Register with MAL */
3069         dev->commac.ops = &emac_commac_ops;
3070         dev->commac.dev = dev;
3071         dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan);
3072         dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan);
3073         err = mal_register_commac(dev->mal, &dev->commac);
3074         if (err) {
3075                 printk(KERN_ERR "%pOF: failed to register with mal %pOF!\n",
3076                        np, dev->mal_dev->dev.of_node);
3077                 goto err_rel_deps;
3078         }
3079         dev->rx_skb_size = emac_rx_skb_size(ndev->mtu);
3080         dev->rx_sync_size = emac_rx_sync_size(ndev->mtu);
3081
3082         /* Get pointers to BD rings */
3083         dev->tx_desc =
3084             dev->mal->bd_virt + mal_tx_bd_offset(dev->mal, dev->mal_tx_chan);
3085         dev->rx_desc =
3086             dev->mal->bd_virt + mal_rx_bd_offset(dev->mal, dev->mal_rx_chan);
3087
3088         DBG(dev, "tx_desc %p" NL, dev->tx_desc);
3089         DBG(dev, "rx_desc %p" NL, dev->rx_desc);
3090
3091         /* Clean rings */
3092         memset(dev->tx_desc, 0, NUM_TX_BUFF * sizeof(struct mal_descriptor));
3093         memset(dev->rx_desc, 0, NUM_RX_BUFF * sizeof(struct mal_descriptor));
3094         memset(dev->tx_skb, 0, NUM_TX_BUFF * sizeof(struct sk_buff *));
3095         memset(dev->rx_skb, 0, NUM_RX_BUFF * sizeof(struct sk_buff *));
3096
3097         /* Attach to ZMII, if needed */
3098         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII) &&
3099             (err = zmii_attach(dev->zmii_dev, dev->zmii_port, &dev->phy_mode)) != 0)
3100                 goto err_unreg_commac;
3101
3102         /* Attach to RGMII, if needed */
3103         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII) &&
3104             (err = rgmii_attach(dev->rgmii_dev, dev->rgmii_port, dev->phy_mode)) != 0)
3105                 goto err_detach_zmii;
3106
3107         /* Attach to TAH, if needed */
3108         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
3109             (err = tah_attach(dev->tah_dev, dev->tah_port)) != 0)
3110                 goto err_detach_rgmii;
3111
3112         /* Set some link defaults before we can find out real parameters */
3113         dev->phy.speed = SPEED_100;
3114         dev->phy.duplex = DUPLEX_FULL;
3115         dev->phy.autoneg = AUTONEG_DISABLE;
3116         dev->phy.pause = dev->phy.asym_pause = 0;
3117         dev->stop_timeout = STOP_TIMEOUT_100;
3118         INIT_DELAYED_WORK(&dev->link_work, emac_link_timer);
3119
3120         /* Some SoCs like APM821xx does not support Half Duplex mode. */
3121         if (emac_has_feature(dev, EMAC_FTR_APM821XX_NO_HALF_DUPLEX)) {
3122                 dev->phy_feat_exc = (SUPPORTED_1000baseT_Half |
3123                                      SUPPORTED_100baseT_Half |
3124                                      SUPPORTED_10baseT_Half);
3125         }
3126
3127         /* Find PHY if any */
3128         err = emac_init_phy(dev);
3129         if (err != 0)
3130                 goto err_detach_tah;
3131
3132         if (dev->tah_dev) {
3133                 ndev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG;
3134                 ndev->features |= ndev->hw_features | NETIF_F_RXCSUM;
3135         }
3136         ndev->watchdog_timeo = 5 * HZ;
3137         if (emac_phy_supports_gige(dev->phy_mode)) {
3138                 ndev->netdev_ops = &emac_gige_netdev_ops;
3139                 dev->commac.ops = &emac_commac_sg_ops;
3140         } else
3141                 ndev->netdev_ops = &emac_netdev_ops;
3142         ndev->ethtool_ops = &emac_ethtool_ops;
3143
3144         /* MTU range: 46 - 1500 or whatever is in OF */
3145         ndev->min_mtu = EMAC_MIN_MTU;
3146         ndev->max_mtu = dev->max_mtu;
3147
3148         netif_carrier_off(ndev);
3149
3150         err = devm_register_netdev(&ofdev->dev, ndev);
3151         if (err) {
3152                 printk(KERN_ERR "%pOF: failed to register net device (%d)!\n",
3153                        np, err);
3154                 goto err_detach_tah;
3155         }
3156
3157         /* Set our drvdata last as we don't want them visible until we are
3158          * fully initialized
3159          */
3160         wmb();
3161         platform_set_drvdata(ofdev, dev);
3162
3163         printk(KERN_INFO "%s: EMAC-%d %pOF, MAC %pM\n",
3164                ndev->name, dev->cell_index, np, ndev->dev_addr);
3165
3166         if (dev->phy_mode == PHY_INTERFACE_MODE_SGMII)
3167                 printk(KERN_NOTICE "%s: in SGMII mode\n", ndev->name);
3168
3169         if (dev->phy.address >= 0)
3170                 printk("%s: found %s PHY (0x%02x)\n", ndev->name,
3171                        dev->phy.def->name, dev->phy.address);
3172
3173         /* Life is good */
3174         return 0;
3175
3176         /* I have a bad feeling about this ... */
3177
3178  err_detach_tah:
3179         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
3180                 tah_detach(dev->tah_dev, dev->tah_port);
3181  err_detach_rgmii:
3182         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
3183                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
3184  err_detach_zmii:
3185         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
3186                 zmii_detach(dev->zmii_dev, dev->zmii_port);
3187  err_unreg_commac:
3188         mal_unregister_commac(dev->mal, &dev->commac);
3189  err_rel_deps:
3190         emac_put_deps(dev);
3191  err_gone:
3192         if (blist)
3193                 *blist = NULL;
3194         return err;
3195 }
3196
3197 static void emac_remove(struct platform_device *ofdev)
3198 {
3199         struct emac_instance *dev = platform_get_drvdata(ofdev);
3200
3201         DBG(dev, "remove" NL);
3202
3203         cancel_work_sync(&dev->reset_work);
3204
3205         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
3206                 tah_detach(dev->tah_dev, dev->tah_port);
3207         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
3208                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
3209         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
3210                 zmii_detach(dev->zmii_dev, dev->zmii_port);
3211
3212         busy_phy_map &= ~(1 << dev->phy.address);
3213         DBG(dev, "busy_phy_map now %#x" NL, busy_phy_map);
3214
3215         mal_unregister_commac(dev->mal, &dev->commac);
3216         emac_put_deps(dev);
3217 }
3218
3219 /* XXX Features in here should be replaced by properties... */
3220 static const struct of_device_id emac_match[] =
3221 {
3222         {
3223                 .type           = "network",
3224                 .compatible     = "ibm,emac",
3225         },
3226         {
3227                 .type           = "network",
3228                 .compatible     = "ibm,emac4",
3229         },
3230         {
3231                 .type           = "network",
3232                 .compatible     = "ibm,emac4sync",
3233         },
3234         {},
3235 };
3236 MODULE_DEVICE_TABLE(of, emac_match);
3237
3238 static struct platform_driver emac_driver = {
3239         .driver = {
3240                 .name = "emac",
3241                 .of_match_table = emac_match,
3242         },
3243         .probe = emac_probe,
3244         .remove_new = emac_remove,
3245 };
3246
3247 static void __init emac_make_bootlist(void)
3248 {
3249         struct device_node *np = NULL;
3250         int j, max, i = 0;
3251         int cell_indices[EMAC_BOOT_LIST_SIZE];
3252
3253         /* Collect EMACs */
3254         while((np = of_find_all_nodes(np)) != NULL) {
3255                 u32 idx;
3256
3257                 if (of_match_node(emac_match, np) == NULL)
3258                         continue;
3259                 if (of_property_read_bool(np, "unused"))
3260                         continue;
3261                 if (of_property_read_u32(np, "cell-index", &idx))
3262                         continue;
3263                 cell_indices[i] = idx;
3264                 emac_boot_list[i++] = of_node_get(np);
3265                 if (i >= EMAC_BOOT_LIST_SIZE) {
3266                         of_node_put(np);
3267                         break;
3268                 }
3269         }
3270         max = i;
3271
3272         /* Bubble sort them (doh, what a creative algorithm :-) */
3273         for (i = 0; max > 1 && (i < (max - 1)); i++)
3274                 for (j = i; j < max; j++) {
3275                         if (cell_indices[i] > cell_indices[j]) {
3276                                 swap(emac_boot_list[i], emac_boot_list[j]);
3277                                 swap(cell_indices[i], cell_indices[j]);
3278                         }
3279                 }
3280 }
3281
3282 static int __init emac_init(void)
3283 {
3284         int rc;
3285
3286         printk(KERN_INFO DRV_DESC ", version " DRV_VERSION "\n");
3287
3288         /* Build EMAC boot list */
3289         emac_make_bootlist();
3290
3291         /* Init submodules */
3292         rc = mal_init();
3293         if (rc)
3294                 goto err;
3295         rc = zmii_init();
3296         if (rc)
3297                 goto err_mal;
3298         rc = rgmii_init();
3299         if (rc)
3300                 goto err_zmii;
3301         rc = tah_init();
3302         if (rc)
3303                 goto err_rgmii;
3304         rc = platform_driver_register(&emac_driver);
3305         if (rc)
3306                 goto err_tah;
3307
3308         return 0;
3309
3310  err_tah:
3311         tah_exit();
3312  err_rgmii:
3313         rgmii_exit();
3314  err_zmii:
3315         zmii_exit();
3316  err_mal:
3317         mal_exit();
3318  err:
3319         return rc;
3320 }
3321
3322 static void __exit emac_exit(void)
3323 {
3324         int i;
3325
3326         platform_driver_unregister(&emac_driver);
3327
3328         tah_exit();
3329         rgmii_exit();
3330         zmii_exit();
3331         mal_exit();
3332
3333         /* Destroy EMAC boot list */
3334         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
3335                 of_node_put(emac_boot_list[i]);
3336 }
3337
3338 module_init(emac_init);
3339 module_exit(emac_exit);