drivers/net/ethernet/intel/e1000/e1000_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 1999 - 2006 Intel Corporation. */
   3
   4 #include "e1000.h"
   5 #include <net/ip6_checksum.h>
   6 #include <linux/io.h>
   7 #include <linux/prefetch.h>
   8 #include <linux/bitops.h>
   9 #include <linux/if_vlan.h>
  10
  11 char e1000_driver_name[] = "e1000";
  12 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  13 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
  14
  15 /* e1000_pci_tbl - PCI Device ID Table
  16  *
  17  * Last entry must be all 0s
  18  *
  19  * Macro expands to...
  20  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  21  */
  22 static const struct pci_device_id e1000_pci_tbl[] = {
  23         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  24         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  25         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  26         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  27         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  28         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  29         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  30         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  31         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  32         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  33         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  34         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  35         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  36         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  37         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  38         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  39         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  40         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  41         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  42         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  43         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  44         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  45         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  46         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  47         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  48         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  49         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  50         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  51         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  52         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  53         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  54         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  55         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  56         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  57         INTEL_E1000_ETHERNET_DEVICE(0x1099),
  58         INTEL_E1000_ETHERNET_DEVICE(0x10B5),
  59         INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
  60         /* required last entry */
  61         {0,}
  62 };
  63
  64 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
  65
  66 int e1000_up(struct e1000_adapter *adapter);
  67 void e1000_down(struct e1000_adapter *adapter);
  68 void e1000_reinit_locked(struct e1000_adapter *adapter);
  69 void e1000_reset(struct e1000_adapter *adapter);
  70 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
  71 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
  72 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
  73 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
  74 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
  75                                     struct e1000_tx_ring *txdr);
  76 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
  77                                     struct e1000_rx_ring *rxdr);
  78 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
  79                                     struct e1000_tx_ring *tx_ring);
  80 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
  81                                     struct e1000_rx_ring *rx_ring);
  82 void e1000_update_stats(struct e1000_adapter *adapter);
  83
  84 static int e1000_init_module(void);
  85 static void e1000_exit_module(void);
  86 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  87 static void e1000_remove(struct pci_dev *pdev);
  88 static int e1000_alloc_queues(struct e1000_adapter *adapter);
  89 static int e1000_sw_init(struct e1000_adapter *adapter);
  90 int e1000_open(struct net_device *netdev);
  91 int e1000_close(struct net_device *netdev);
  92 static void e1000_configure_tx(struct e1000_adapter *adapter);
  93 static void e1000_configure_rx(struct e1000_adapter *adapter);
  94 static void e1000_setup_rctl(struct e1000_adapter *adapter);
  95 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
  96 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
  97 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
  98                                 struct e1000_tx_ring *tx_ring);
  99 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 100                                 struct e1000_rx_ring *rx_ring);
 101 static void e1000_set_rx_mode(struct net_device *netdev);
 102 static void e1000_update_phy_info_task(struct work_struct *work);
 103 static void e1000_watchdog(struct work_struct *work);
 104 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
 105 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
 106                                     struct net_device *netdev);
 107 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 108 static int e1000_set_mac(struct net_device *netdev, void *p);
 109 static irqreturn_t e1000_intr(int irq, void *data);
 110 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
 111                                struct e1000_tx_ring *tx_ring);
 112 static int e1000_clean(struct napi_struct *napi, int budget);
 113 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 114                                struct e1000_rx_ring *rx_ring,
 115                                int *work_done, int work_to_do);
 116 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
 117                                      struct e1000_rx_ring *rx_ring,
 118                                      int *work_done, int work_to_do);
 119 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
 120                                          struct e1000_rx_ring *rx_ring,
 121                                          int cleaned_count)
 122 {
 123 }
 124 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 125                                    struct e1000_rx_ring *rx_ring,
 126                                    int cleaned_count);
 127 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 128                                          struct e1000_rx_ring *rx_ring,
 129                                          int cleaned_count);
 130 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 131 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 132                            int cmd);
 133 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 134 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 135 static void e1000_tx_timeout(struct net_device *dev, unsigned int txqueue);
 136 static void e1000_reset_task(struct work_struct *work);
 137 static void e1000_smartspeed(struct e1000_adapter *adapter);
 138 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 139                                        struct sk_buff *skb);
 140
 141 static bool e1000_vlan_used(struct e1000_adapter *adapter);
 142 static void e1000_vlan_mode(struct net_device *netdev,
 143                             netdev_features_t features);
 144 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
 145                                      bool filter_on);
 146 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
 147                                  __be16 proto, u16 vid);
 148 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 149                                   __be16 proto, u16 vid);
 150 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 151
 152 static int e1000_suspend(struct device *dev);
 153 static int e1000_resume(struct device *dev);
 154 static void e1000_shutdown(struct pci_dev *pdev);
 155
 156 #ifdef CONFIG_NET_POLL_CONTROLLER
 157 /* for netdump / net console */
 158 static void e1000_netpoll (struct net_device *netdev);
 159 #endif
 160
 161 #define COPYBREAK_DEFAULT 256
 162 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
 163 module_param(copybreak, uint, 0644);
 164 MODULE_PARM_DESC(copybreak,
 165         "Maximum size of packet that is copied to a new buffer on receive");
 166
 167 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
 168                                                 pci_channel_state_t state);
 169 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
 170 static void e1000_io_resume(struct pci_dev *pdev);
 171
 172 static const struct pci_error_handlers e1000_err_handler = {
 173         .error_detected = e1000_io_error_detected,
 174         .slot_reset = e1000_io_slot_reset,
 175         .resume = e1000_io_resume,
 176 };
 177
 178 static DEFINE_SIMPLE_DEV_PM_OPS(e1000_pm_ops, e1000_suspend, e1000_resume);
 179
 180 static struct pci_driver e1000_driver = {
 181         .name     = e1000_driver_name,
 182         .id_table = e1000_pci_tbl,
 183         .probe    = e1000_probe,
 184         .remove   = e1000_remove,
 185         .driver.pm = pm_sleep_ptr(&e1000_pm_ops),
 186         .shutdown = e1000_shutdown,
 187         .err_handler = &e1000_err_handler
 188 };
 189
 190 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 191 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 192 MODULE_LICENSE("GPL v2");
 193
 194 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 195 static int debug = -1;
 196 module_param(debug, int, 0);
 197 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 198
 199 /**
 200  * e1000_get_hw_dev - helper function for getting netdev
 201  * @hw: pointer to HW struct
 202  *
 203  * return device used by hardware layer to print debugging information
 204  *
 205  **/
 206 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
 207 {
 208         struct e1000_adapter *adapter = hw->back;
 209         return adapter->netdev;
 210 }
 211
 212 /**
 213  * e1000_init_module - Driver Registration Routine
 214  *
 215  * e1000_init_module is the first routine called when the driver is
 216  * loaded. All it does is register with the PCI subsystem.
 217  **/
 218 static int __init e1000_init_module(void)
 219 {
 220         int ret;
 221         pr_info("%s\n", e1000_driver_string);
 222
 223         pr_info("%s\n", e1000_copyright);
 224
 225         ret = pci_register_driver(&e1000_driver);
 226         if (copybreak != COPYBREAK_DEFAULT) {
 227                 if (copybreak == 0)
 228                         pr_info("copybreak disabled\n");
 229                 else
 230                         pr_info("copybreak enabled for "
 231                                    "packets <= %u bytes\n", copybreak);
 232         }
 233         return ret;
 234 }
 235
 236 module_init(e1000_init_module);
 237
 238 /**
 239  * e1000_exit_module - Driver Exit Cleanup Routine
 240  *
 241  * e1000_exit_module is called just before the driver is removed
 242  * from memory.
 243  **/
 244 static void __exit e1000_exit_module(void)
 245 {
 246         pci_unregister_driver(&e1000_driver);
 247 }
 248
 249 module_exit(e1000_exit_module);
 250
 251 static int e1000_request_irq(struct e1000_adapter *adapter)
 252 {
 253         struct net_device *netdev = adapter->netdev;
 254         irq_handler_t handler = e1000_intr;
 255         int irq_flags = IRQF_SHARED;
 256         int err;
 257
 258         err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
 259                           netdev);
 260         if (err) {
 261                 e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
 262         }
 263
 264         return err;
 265 }
 266
 267 static void e1000_free_irq(struct e1000_adapter *adapter)
 268 {
 269         struct net_device *netdev = adapter->netdev;
 270
 271         free_irq(adapter->pdev->irq, netdev);
 272 }
 273
 274 /**
 275  * e1000_irq_disable - Mask off interrupt generation on the NIC
 276  * @adapter: board private structure
 277  **/
 278 static void e1000_irq_disable(struct e1000_adapter *adapter)
 279 {
 280         struct e1000_hw *hw = &adapter->hw;
 281
 282         ew32(IMC, ~0);
 283         E1000_WRITE_FLUSH();
 284         synchronize_irq(adapter->pdev->irq);
 285 }
 286
 287 /**
 288  * e1000_irq_enable - Enable default interrupt generation settings
 289  * @adapter: board private structure
 290  **/
 291 static void e1000_irq_enable(struct e1000_adapter *adapter)
 292 {
 293         struct e1000_hw *hw = &adapter->hw;
 294
 295         ew32(IMS, IMS_ENABLE_MASK);
 296         E1000_WRITE_FLUSH();
 297 }
 298
 299 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
 300 {
 301         struct e1000_hw *hw = &adapter->hw;
 302         struct net_device *netdev = adapter->netdev;
 303         u16 vid = hw->mng_cookie.vlan_id;
 304         u16 old_vid = adapter->mng_vlan_id;
 305
 306         if (!e1000_vlan_used(adapter))
 307                 return;
 308
 309         if (!test_bit(vid, adapter->active_vlans)) {
 310                 if (hw->mng_cookie.status &
 311                     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 312                         e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
 313                         adapter->mng_vlan_id = vid;
 314                 } else {
 315                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 316                 }
 317                 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
 318                     (vid != old_vid) &&
 319                     !test_bit(old_vid, adapter->active_vlans))
 320                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
 321                                                old_vid);
 322         } else {
 323                 adapter->mng_vlan_id = vid;
 324         }
 325 }
 326
 327 static void e1000_init_manageability(struct e1000_adapter *adapter)
 328 {
 329         struct e1000_hw *hw = &adapter->hw;
 330
 331         if (adapter->en_mng_pt) {
 332                 u32 manc = er32(MANC);
 333
 334                 /* disable hardware interception of ARP */
 335                 manc &= ~(E1000_MANC_ARP_EN);
 336
 337                 ew32(MANC, manc);
 338         }
 339 }
 340
 341 static void e1000_release_manageability(struct e1000_adapter *adapter)
 342 {
 343         struct e1000_hw *hw = &adapter->hw;
 344
 345         if (adapter->en_mng_pt) {
 346                 u32 manc = er32(MANC);
 347
 348                 /* re-enable hardware interception of ARP */
 349                 manc |= E1000_MANC_ARP_EN;
 350
 351                 ew32(MANC, manc);
 352         }
 353 }
 354
 355 /**
 356  * e1000_configure - configure the hardware for RX and TX
 357  * @adapter: private board structure
 358  **/
 359 static void e1000_configure(struct e1000_adapter *adapter)
 360 {
 361         struct net_device *netdev = adapter->netdev;
 362         int i;
 363
 364         e1000_set_rx_mode(netdev);
 365
 366         e1000_restore_vlan(adapter);
 367         e1000_init_manageability(adapter);
 368
 369         e1000_configure_tx(adapter);
 370         e1000_setup_rctl(adapter);
 371         e1000_configure_rx(adapter);
 372         /* call E1000_DESC_UNUSED which always leaves
 373          * at least 1 descriptor unused to make sure
 374          * next_to_use != next_to_clean
 375          */
 376         for (i = 0; i < adapter->num_rx_queues; i++) {
 377                 struct e1000_rx_ring *ring = &adapter->rx_ring[i];
 378                 adapter->alloc_rx_buf(adapter, ring,
 379                                       E1000_DESC_UNUSED(ring));
 380         }
 381 }
 382
 383 int e1000_up(struct e1000_adapter *adapter)
 384 {
 385         struct e1000_hw *hw = &adapter->hw;
 386
 387         /* hardware has been reset, we need to reload some things */
 388         e1000_configure(adapter);
 389
 390         clear_bit(__E1000_DOWN, &adapter->flags);
 391
 392         napi_enable(&adapter->napi);
 393
 394         e1000_irq_enable(adapter);
 395
 396         netif_wake_queue(adapter->netdev);
 397
 398         /* fire a link change interrupt to start the watchdog */
 399         ew32(ICS, E1000_ICS_LSC);
 400         return 0;
 401 }
 402
 403 /**
 404  * e1000_power_up_phy - restore link in case the phy was powered down
 405  * @adapter: address of board private structure
 406  *
 407  * The phy may be powered down to save power and turn off link when the
 408  * driver is unloaded and wake on lan is not enabled (among others)
 409  * *** this routine MUST be followed by a call to e1000_reset ***
 410  **/
 411 void e1000_power_up_phy(struct e1000_adapter *adapter)
 412 {
 413         struct e1000_hw *hw = &adapter->hw;
 414         u16 mii_reg = 0;
 415
 416         /* Just clear the power down bit to wake the phy back up */
 417         if (hw->media_type == e1000_media_type_copper) {
 418                 /* according to the manual, the phy will retain its
 419                  * settings across a power-down/up cycle
 420                  */
 421                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 422                 mii_reg &= ~MII_CR_POWER_DOWN;
 423                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 424         }
 425 }
 426
 427 static void e1000_power_down_phy(struct e1000_adapter *adapter)
 428 {
 429         struct e1000_hw *hw = &adapter->hw;
 430
 431         /* Power down the PHY so no link is implied when interface is down *
 432          * The PHY cannot be powered down if any of the following is true *
 433          * (a) WoL is enabled
 434          * (b) AMT is active
 435          * (c) SoL/IDER session is active
 436          */
 437         if (!adapter->wol && hw->mac_type >= e1000_82540 &&
 438            hw->media_type == e1000_media_type_copper) {
 439                 u16 mii_reg = 0;
 440
 441                 switch (hw->mac_type) {
 442                 case e1000_82540:
 443                 case e1000_82545:
 444                 case e1000_82545_rev_3:
 445                 case e1000_82546:
 446                 case e1000_ce4100:
 447                 case e1000_82546_rev_3:
 448                 case e1000_82541:
 449                 case e1000_82541_rev_2:
 450                 case e1000_82547:
 451                 case e1000_82547_rev_2:
 452                         if (er32(MANC) & E1000_MANC_SMBUS_EN)
 453                                 goto out;
 454                         break;
 455                 default:
 456                         goto out;
 457                 }
 458                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 459                 mii_reg |= MII_CR_POWER_DOWN;
 460                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 461                 msleep(1);
 462         }
 463 out:
 464         return;
 465 }
 466
 467 static void e1000_down_and_stop(struct e1000_adapter *adapter)
 468 {
 469         set_bit(__E1000_DOWN, &adapter->flags);
 470
 471         cancel_delayed_work_sync(&adapter->watchdog_task);
 472
 473         /*
 474          * Since the watchdog task can reschedule other tasks, we should cancel
 475          * it first, otherwise we can run into the situation when a work is
 476          * still running after the adapter has been turned down.
 477          */
 478
 479         cancel_delayed_work_sync(&adapter->phy_info_task);
 480         cancel_delayed_work_sync(&adapter->fifo_stall_task);
 481
 482         /* Only kill reset task if adapter is not resetting */
 483         if (!test_bit(__E1000_RESETTING, &adapter->flags))
 484                 cancel_work_sync(&adapter->reset_task);
 485 }
 486
 487 void e1000_down(struct e1000_adapter *adapter)
 488 {
 489         struct e1000_hw *hw = &adapter->hw;
 490         struct net_device *netdev = adapter->netdev;
 491         u32 rctl, tctl;
 492
 493         /* disable receives in the hardware */
 494         rctl = er32(RCTL);
 495         ew32(RCTL, rctl & ~E1000_RCTL_EN);
 496         /* flush and sleep below */
 497
 498         netif_tx_disable(netdev);
 499
 500         /* disable transmits in the hardware */
 501         tctl = er32(TCTL);
 502         tctl &= ~E1000_TCTL_EN;
 503         ew32(TCTL, tctl);
 504         /* flush both disables and wait for them to finish */
 505         E1000_WRITE_FLUSH();
 506         msleep(10);
 507
 508         /* Set the carrier off after transmits have been disabled in the
 509          * hardware, to avoid race conditions with e1000_watchdog() (which
 510          * may be running concurrently to us, checking for the carrier
 511          * bit to decide whether it should enable transmits again). Such
 512          * a race condition would result into transmission being disabled
 513          * in the hardware until the next IFF_DOWN+IFF_UP cycle.
 514          */
 515         netif_carrier_off(netdev);
 516
 517         napi_disable(&adapter->napi);
 518
 519         e1000_irq_disable(adapter);
 520
 521         /* Setting DOWN must be after irq_disable to prevent
 522          * a screaming interrupt.  Setting DOWN also prevents
 523          * tasks from rescheduling.
 524          */
 525         e1000_down_and_stop(adapter);
 526
 527         adapter->link_speed = 0;
 528         adapter->link_duplex = 0;
 529
 530         e1000_reset(adapter);
 531         e1000_clean_all_tx_rings(adapter);
 532         e1000_clean_all_rx_rings(adapter);
 533 }
 534
 535 void e1000_reinit_locked(struct e1000_adapter *adapter)
 536 {
 537         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
 538                 msleep(1);
 539
 540         /* only run the task if not already down */
 541         if (!test_bit(__E1000_DOWN, &adapter->flags)) {
 542                 e1000_down(adapter);
 543                 e1000_up(adapter);
 544         }
 545
 546         clear_bit(__E1000_RESETTING, &adapter->flags);
 547 }
 548
 549 void e1000_reset(struct e1000_adapter *adapter)
 550 {
 551         struct e1000_hw *hw = &adapter->hw;
 552         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
 553         bool legacy_pba_adjust = false;
 554         u16 hwm;
 555
 556         /* Repartition Pba for greater than 9k mtu
 557          * To take effect CTRL.RST is required.
 558          */
 559
 560         switch (hw->mac_type) {
 561         case e1000_82542_rev2_0:
 562         case e1000_82542_rev2_1:
 563         case e1000_82543:
 564         case e1000_82544:
 565         case e1000_82540:
 566         case e1000_82541:
 567         case e1000_82541_rev_2:
 568                 legacy_pba_adjust = true;
 569                 pba = E1000_PBA_48K;
 570                 break;
 571         case e1000_82545:
 572         case e1000_82545_rev_3:
 573         case e1000_82546:
 574         case e1000_ce4100:
 575         case e1000_82546_rev_3:
 576                 pba = E1000_PBA_48K;
 577                 break;
 578         case e1000_82547:
 579         case e1000_82547_rev_2:
 580                 legacy_pba_adjust = true;
 581                 pba = E1000_PBA_30K;
 582                 break;
 583         case e1000_undefined:
 584         case e1000_num_macs:
 585                 break;
 586         }
 587
 588         if (legacy_pba_adjust) {
 589                 if (hw->max_frame_size > E1000_RXBUFFER_8192)
 590                         pba -= 8; /* allocate more FIFO for Tx */
 591
 592                 if (hw->mac_type == e1000_82547) {
 593                         adapter->tx_fifo_head = 0;
 594                         adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 595                         adapter->tx_fifo_size =
 596                                 (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 597                         atomic_set(&adapter->tx_fifo_stall, 0);
 598                 }
 599         } else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
 600                 /* adjust PBA for jumbo frames */
 601                 ew32(PBA, pba);
 602
 603                 /* To maintain wire speed transmits, the Tx FIFO should be
 604                  * large enough to accommodate two full transmit packets,
 605                  * rounded up to the next 1KB and expressed in KB.  Likewise,
 606                  * the Rx FIFO should be large enough to accommodate at least
 607                  * one full receive packet and is similarly rounded up and
 608                  * expressed in KB.
 609                  */
 610                 pba = er32(PBA);
 611                 /* upper 16 bits has Tx packet buffer allocation size in KB */
 612                 tx_space = pba >> 16;
 613                 /* lower 16 bits has Rx packet buffer allocation size in KB */
 614                 pba &= 0xffff;
 615                 /* the Tx fifo also stores 16 bytes of information about the Tx
 616                  * but don't include ethernet FCS because hardware appends it
 617                  */
 618                 min_tx_space = (hw->max_frame_size +
 619                                 sizeof(struct e1000_tx_desc) -
 620                                 ETH_FCS_LEN) * 2;
 621                 min_tx_space = ALIGN(min_tx_space, 1024);
 622                 min_tx_space >>= 10;
 623                 /* software strips receive CRC, so leave room for it */
 624                 min_rx_space = hw->max_frame_size;
 625                 min_rx_space = ALIGN(min_rx_space, 1024);
 626                 min_rx_space >>= 10;
 627
 628                 /* If current Tx allocation is less than the min Tx FIFO size,
 629                  * and the min Tx FIFO size is less than the current Rx FIFO
 630                  * allocation, take space away from current Rx allocation
 631                  */
 632                 if (tx_space < min_tx_space &&
 633                     ((min_tx_space - tx_space) < pba)) {
 634                         pba = pba - (min_tx_space - tx_space);
 635
 636                         /* PCI/PCIx hardware has PBA alignment constraints */
 637                         switch (hw->mac_type) {
 638                         case e1000_82545 ... e1000_82546_rev_3:
 639                                 pba &= ~(E1000_PBA_8K - 1);
 640                                 break;
 641                         default:
 642                                 break;
 643                         }
 644
 645                         /* if short on Rx space, Rx wins and must trump Tx
 646                          * adjustment or use Early Receive if available
 647                          */
 648                         if (pba < min_rx_space)
 649                                 pba = min_rx_space;
 650                 }
 651         }
 652
 653         ew32(PBA, pba);
 654
 655         /* flow control settings:
 656          * The high water mark must be low enough to fit one full frame
 657          * (or the size used for early receive) above it in the Rx FIFO.
 658          * Set it to the lower of:
 659          * - 90% of the Rx FIFO size, and
 660          * - the full Rx FIFO size minus the early receive size (for parts
 661          *   with ERT support assuming ERT set to E1000_ERT_2048), or
 662          * - the full Rx FIFO size minus one full frame
 663          */
 664         hwm = min(((pba << 10) * 9 / 10),
 665                   ((pba << 10) - hw->max_frame_size));
 666
 667         hw->fc_high_water = hwm & 0xFFF8;       /* 8-byte granularity */
 668         hw->fc_low_water = hw->fc_high_water - 8;
 669         hw->fc_pause_time = E1000_FC_PAUSE_TIME;
 670         hw->fc_send_xon = 1;
 671         hw->fc = hw->original_fc;
 672
 673         /* Allow time for pending master requests to run */
 674         e1000_reset_hw(hw);
 675         if (hw->mac_type >= e1000_82544)
 676                 ew32(WUC, 0);
 677
 678         if (e1000_init_hw(hw))
 679                 e_dev_err("Hardware Error\n");
 680         e1000_update_mng_vlan(adapter);
 681
 682         /* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
 683         if (hw->mac_type >= e1000_82544 &&
 684             hw->autoneg == 1 &&
 685             hw->autoneg_advertised == ADVERTISE_1000_FULL) {
 686                 u32 ctrl = er32(CTRL);
 687                 /* clear phy power management bit if we are in gig only mode,
 688                  * which if enabled will attempt negotiation to 100Mb, which
 689                  * can cause a loss of link at power off or driver unload
 690                  */
 691                 ctrl &= ~E1000_CTRL_SWDPIN3;
 692                 ew32(CTRL, ctrl);
 693         }
 694
 695         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 696         ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
 697
 698         e1000_reset_adaptive(hw);
 699         e1000_phy_get_info(hw, &adapter->phy_info);
 700
 701         e1000_release_manageability(adapter);
 702 }
 703
 704 /* Dump the eeprom for users having checksum issues */
 705 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
 706 {
 707         struct net_device *netdev = adapter->netdev;
 708         struct ethtool_eeprom eeprom;
 709         const struct ethtool_ops *ops = netdev->ethtool_ops;
 710         u8 *data;
 711         int i;
 712         u16 csum_old, csum_new = 0;
 713
 714         eeprom.len = ops->get_eeprom_len(netdev);
 715         eeprom.offset = 0;
 716
 717         data = kmalloc(eeprom.len, GFP_KERNEL);
 718         if (!data)
 719                 return;
 720
 721         ops->get_eeprom(netdev, &eeprom, data);
 722
 723         csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
 724                    (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
 725         for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
 726                 csum_new += data[i] + (data[i + 1] << 8);
 727         csum_new = EEPROM_SUM - csum_new;
 728
 729         pr_err("/*********************/\n");
 730         pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
 731         pr_err("Calculated              : 0x%04x\n", csum_new);
 732
 733         pr_err("Offset    Values\n");
 734         pr_err("========  ======\n");
 735         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
 736
 737         pr_err("Include this output when contacting your support provider.\n");
 738         pr_err("This is not a software error! Something bad happened to\n");
 739         pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
 740         pr_err("result in further problems, possibly loss of data,\n");
 741         pr_err("corruption or system hangs!\n");
 742         pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
 743         pr_err("which is invalid and requires you to set the proper MAC\n");
 744         pr_err("address manually before continuing to enable this network\n");
 745         pr_err("device. Please inspect the EEPROM dump and report the\n");
 746         pr_err("issue to your hardware vendor or Intel Customer Support.\n");
 747         pr_err("/*********************/\n");
 748
 749         kfree(data);
 750 }
 751
 752 /**
 753  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 754  * @pdev: PCI device information struct
 755  *
 756  * Return true if an adapter needs ioport resources
 757  **/
 758 static int e1000_is_need_ioport(struct pci_dev *pdev)
 759 {
 760         switch (pdev->device) {
 761         case E1000_DEV_ID_82540EM:
 762         case E1000_DEV_ID_82540EM_LOM:
 763         case E1000_DEV_ID_82540EP:
 764         case E1000_DEV_ID_82540EP_LOM:
 765         case E1000_DEV_ID_82540EP_LP:
 766         case E1000_DEV_ID_82541EI:
 767         case E1000_DEV_ID_82541EI_MOBILE:
 768         case E1000_DEV_ID_82541ER:
 769         case E1000_DEV_ID_82541ER_LOM:
 770         case E1000_DEV_ID_82541GI:
 771         case E1000_DEV_ID_82541GI_LF:
 772         case E1000_DEV_ID_82541GI_MOBILE:
 773         case E1000_DEV_ID_82544EI_COPPER:
 774         case E1000_DEV_ID_82544EI_FIBER:
 775         case E1000_DEV_ID_82544GC_COPPER:
 776         case E1000_DEV_ID_82544GC_LOM:
 777         case E1000_DEV_ID_82545EM_COPPER:
 778         case E1000_DEV_ID_82545EM_FIBER:
 779         case E1000_DEV_ID_82546EB_COPPER:
 780         case E1000_DEV_ID_82546EB_FIBER:
 781         case E1000_DEV_ID_82546EB_QUAD_COPPER:
 782                 return true;
 783         default:
 784                 return false;
 785         }
 786 }
 787
 788 static netdev_features_t e1000_fix_features(struct net_device *netdev,
 789         netdev_features_t features)
 790 {
 791         /* Since there is no support for separate Rx/Tx vlan accel
 792          * enable/disable make sure Tx flag is always in same state as Rx.
 793          */
 794         if (features & NETIF_F_HW_VLAN_CTAG_RX)
 795                 features |= NETIF_F_HW_VLAN_CTAG_TX;
 796         else
 797                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
 798
 799         return features;
 800 }
 801
 802 static int e1000_set_features(struct net_device *netdev,
 803         netdev_features_t features)
 804 {
 805         struct e1000_adapter *adapter = netdev_priv(netdev);
 806         netdev_features_t changed = features ^ netdev->features;
 807
 808         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
 809                 e1000_vlan_mode(netdev, features);
 810
 811         if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
 812                 return 0;
 813
 814         netdev->features = features;
 815         adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
 816
 817         if (netif_running(netdev))
 818                 e1000_reinit_locked(adapter);
 819         else
 820                 e1000_reset(adapter);
 821
 822         return 1;
 823 }
 824
 825 static const struct net_device_ops e1000_netdev_ops = {
 826         .ndo_open               = e1000_open,
 827         .ndo_stop               = e1000_close,
 828         .ndo_start_xmit         = e1000_xmit_frame,
 829         .ndo_set_rx_mode        = e1000_set_rx_mode,
 830         .ndo_set_mac_address    = e1000_set_mac,
 831         .ndo_tx_timeout         = e1000_tx_timeout,
 832         .ndo_change_mtu         = e1000_change_mtu,
 833         .ndo_eth_ioctl          = e1000_ioctl,
 834         .ndo_validate_addr      = eth_validate_addr,
 835         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
 836         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
 837 #ifdef CONFIG_NET_POLL_CONTROLLER
 838         .ndo_poll_controller    = e1000_netpoll,
 839 #endif
 840         .ndo_fix_features       = e1000_fix_features,
 841         .ndo_set_features       = e1000_set_features,
 842 };
 843
 844 /**
 845  * e1000_init_hw_struct - initialize members of hw struct
 846  * @adapter: board private struct
 847  * @hw: structure used by e1000_hw.c
 848  *
 849  * Factors out initialization of the e1000_hw struct to its own function
 850  * that can be called very early at init (just after struct allocation).
 851  * Fields are initialized based on PCI device information and
 852  * OS network device settings (MTU size).
 853  * Returns negative error codes if MAC type setup fails.
 854  */
 855 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
 856                                 struct e1000_hw *hw)
 857 {
 858         struct pci_dev *pdev = adapter->pdev;
 859
 860         /* PCI config space info */
 861         hw->vendor_id = pdev->vendor;
 862         hw->device_id = pdev->device;
 863         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 864         hw->subsystem_id = pdev->subsystem_device;
 865         hw->revision_id = pdev->revision;
 866
 867         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 868
 869         hw->max_frame_size = adapter->netdev->mtu +
 870                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 871         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 872
 873         /* identify the MAC */
 874         if (e1000_set_mac_type(hw)) {
 875                 e_err(probe, "Unknown MAC Type\n");
 876                 return -EIO;
 877         }
 878
 879         switch (hw->mac_type) {
 880         default:
 881                 break;
 882         case e1000_82541:
 883         case e1000_82547:
 884         case e1000_82541_rev_2:
 885         case e1000_82547_rev_2:
 886                 hw->phy_init_script = 1;
 887                 break;
 888         }
 889
 890         e1000_set_media_type(hw);
 891         e1000_get_bus_info(hw);
 892
 893         hw->wait_autoneg_complete = false;
 894         hw->tbi_compatibility_en = true;
 895         hw->adaptive_ifs = true;
 896
 897         /* Copper options */
 898
 899         if (hw->media_type == e1000_media_type_copper) {
 900                 hw->mdix = AUTO_ALL_MODES;
 901                 hw->disable_polarity_correction = false;
 902                 hw->master_slave = E1000_MASTER_SLAVE;
 903         }
 904
 905         return 0;
 906 }
 907
 908 /**
 909  * e1000_probe - Device Initialization Routine
 910  * @pdev: PCI device information struct
 911  * @ent: entry in e1000_pci_tbl
 912  *
 913  * Returns 0 on success, negative on failure
 914  *
 915  * e1000_probe initializes an adapter identified by a pci_dev structure.
 916  * The OS initialization, configuring of the adapter private structure,
 917  * and a hardware reset occur.
 918  **/
 919 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 920 {
 921         struct net_device *netdev;
 922         struct e1000_adapter *adapter = NULL;
 923         struct e1000_hw *hw;
 924
 925         static int cards_found;
 926         static int global_quad_port_a; /* global ksp3 port a indication */
 927         int i, err, pci_using_dac;
 928         u16 eeprom_data = 0;
 929         u16 tmp = 0;
 930         u16 eeprom_apme_mask = E1000_EEPROM_APME;
 931         int bars, need_ioport;
 932         bool disable_dev = false;
 933
 934         /* do not allocate ioport bars when not needed */
 935         need_ioport = e1000_is_need_ioport(pdev);
 936         if (need_ioport) {
 937                 bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
 938                 err = pci_enable_device(pdev);
 939         } else {
 940                 bars = pci_select_bars(pdev, IORESOURCE_MEM);
 941                 err = pci_enable_device_mem(pdev);
 942         }
 943         if (err)
 944                 return err;
 945
 946         err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
 947         if (err)
 948                 goto err_pci_reg;
 949
 950         pci_set_master(pdev);
 951         err = pci_save_state(pdev);
 952         if (err)
 953                 goto err_alloc_etherdev;
 954
 955         err = -ENOMEM;
 956         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 957         if (!netdev)
 958                 goto err_alloc_etherdev;
 959
 960         SET_NETDEV_DEV(netdev, &pdev->dev);
 961
 962         pci_set_drvdata(pdev, netdev);
 963         adapter = netdev_priv(netdev);
 964         adapter->netdev = netdev;
 965         adapter->pdev = pdev;
 966         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 967         adapter->bars = bars;
 968         adapter->need_ioport = need_ioport;
 969
 970         hw = &adapter->hw;
 971         hw->back = adapter;
 972
 973         err = -EIO;
 974         hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
 975         if (!hw->hw_addr)
 976                 goto err_ioremap;
 977
 978         if (adapter->need_ioport) {
 979                 for (i = BAR_1; i < PCI_STD_NUM_BARS; i++) {
 980                         if (pci_resource_len(pdev, i) == 0)
 981                                 continue;
 982                         if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
 983                                 hw->io_base = pci_resource_start(pdev, i);
 984                                 break;
 985                         }
 986                 }
 987         }
 988
 989         /* make ready for any if (hw->...) below */
 990         err = e1000_init_hw_struct(adapter, hw);
 991         if (err)
 992                 goto err_sw_init;
 993
 994         /* there is a workaround being applied below that limits
 995          * 64-bit DMA addresses to 64-bit hardware.  There are some
 996          * 32-bit adapters that Tx hang when given 64-bit DMA addresses
 997          */
 998         pci_using_dac = 0;
 999         if ((hw->bus_type == e1000_bus_type_pcix) &&
1000             !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1001                 pci_using_dac = 1;
1002         } else {
1003                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1004                 if (err) {
1005                         pr_err("No usable DMA config, aborting\n");
1006                         goto err_dma;
1007                 }
1008         }
1009
1010         netdev->netdev_ops = &e1000_netdev_ops;
1011         e1000_set_ethtool_ops(netdev);
1012         netdev->watchdog_timeo = 5 * HZ;
1013         netif_napi_add(netdev, &adapter->napi, e1000_clean);
1014
1015         strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
1016
1017         adapter->bd_number = cards_found;
1018
1019         /* setup the private structure */
1020
1021         err = e1000_sw_init(adapter);
1022         if (err)
1023                 goto err_sw_init;
1024
1025         err = -EIO;
1026         if (hw->mac_type == e1000_ce4100) {
1027                 hw->ce4100_gbe_mdio_base_virt =
1028                                         ioremap(pci_resource_start(pdev, BAR_1),
1029                                                 pci_resource_len(pdev, BAR_1));
1030
1031                 if (!hw->ce4100_gbe_mdio_base_virt)
1032                         goto err_mdio_ioremap;
1033         }
1034
1035         if (hw->mac_type >= e1000_82543) {
1036                 netdev->hw_features = NETIF_F_SG |
1037                                    NETIF_F_HW_CSUM |
1038                                    NETIF_F_HW_VLAN_CTAG_RX;
1039                 netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1040                                    NETIF_F_HW_VLAN_CTAG_FILTER;
1041         }
1042
1043         if ((hw->mac_type >= e1000_82544) &&
1044            (hw->mac_type != e1000_82547))
1045                 netdev->hw_features |= NETIF_F_TSO;
1046
1047         netdev->priv_flags |= IFF_SUPP_NOFCS;
1048
1049         netdev->features |= netdev->hw_features;
1050         netdev->hw_features |= (NETIF_F_RXCSUM |
1051                                 NETIF_F_RXALL |
1052                                 NETIF_F_RXFCS);
1053
1054         if (pci_using_dac) {
1055                 netdev->features |= NETIF_F_HIGHDMA;
1056                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1057         }
1058
1059         netdev->vlan_features |= (NETIF_F_TSO |
1060                                   NETIF_F_HW_CSUM |
1061                                   NETIF_F_SG);
1062
1063         /* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1064         if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1065             hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1066                 netdev->priv_flags |= IFF_UNICAST_FLT;
1067
1068         /* MTU range: 46 - 16110 */
1069         netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1070         netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1071
1072         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1073
1074         /* initialize eeprom parameters */
1075         if (e1000_init_eeprom_params(hw)) {
1076                 e_err(probe, "EEPROM initialization failed\n");
1077                 goto err_eeprom;
1078         }
1079
1080         /* before reading the EEPROM, reset the controller to
1081          * put the device in a known good starting state
1082          */
1083
1084         e1000_reset_hw(hw);
1085
1086         /* make sure the EEPROM is good */
1087         if (e1000_validate_eeprom_checksum(hw) < 0) {
1088                 e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1089                 e1000_dump_eeprom(adapter);
1090                 /* set MAC address to all zeroes to invalidate and temporary
1091                  * disable this device for the user. This blocks regular
1092                  * traffic while still permitting ethtool ioctls from reaching
1093                  * the hardware as well as allowing the user to run the
1094                  * interface after manually setting a hw addr using
1095                  * `ip set address`
1096                  */
1097                 memset(hw->mac_addr, 0, netdev->addr_len);
1098         } else {
1099                 /* copy the MAC address out of the EEPROM */
1100                 if (e1000_read_mac_addr(hw))
1101                         e_err(probe, "EEPROM Read Error\n");
1102         }
1103         /* don't block initialization here due to bad MAC address */
1104         eth_hw_addr_set(netdev, hw->mac_addr);
1105
1106         if (!is_valid_ether_addr(netdev->dev_addr))
1107                 e_err(probe, "Invalid MAC Address\n");
1108
1109
1110         INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1111         INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1112                           e1000_82547_tx_fifo_stall_task);
1113         INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1114         INIT_WORK(&adapter->reset_task, e1000_reset_task);
1115
1116         e1000_check_options(adapter);
1117
1118         /* Initial Wake on LAN setting
1119          * If APM wake is enabled in the EEPROM,
1120          * enable the ACPI Magic Packet filter
1121          */
1122
1123         switch (hw->mac_type) {
1124         case e1000_82542_rev2_0:
1125         case e1000_82542_rev2_1:
1126         case e1000_82543:
1127                 break;
1128         case e1000_82544:
1129                 e1000_read_eeprom(hw,
1130                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1131                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
1132                 break;
1133         case e1000_82546:
1134         case e1000_82546_rev_3:
1135                 if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1136                         e1000_read_eeprom(hw,
1137                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1138                         break;
1139                 }
1140                 fallthrough;
1141         default:
1142                 e1000_read_eeprom(hw,
1143                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1144                 break;
1145         }
1146         if (eeprom_data & eeprom_apme_mask)
1147                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1148
1149         /* now that we have the eeprom settings, apply the special cases
1150          * where the eeprom may be wrong or the board simply won't support
1151          * wake on lan on a particular port
1152          */
1153         switch (pdev->device) {
1154         case E1000_DEV_ID_82546GB_PCIE:
1155                 adapter->eeprom_wol = 0;
1156                 break;
1157         case E1000_DEV_ID_82546EB_FIBER:
1158         case E1000_DEV_ID_82546GB_FIBER:
1159                 /* Wake events only supported on port A for dual fiber
1160                  * regardless of eeprom setting
1161                  */
1162                 if (er32(STATUS) & E1000_STATUS_FUNC_1)
1163                         adapter->eeprom_wol = 0;
1164                 break;
1165         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1166                 /* if quad port adapter, disable WoL on all but port A */
1167                 if (global_quad_port_a != 0)
1168                         adapter->eeprom_wol = 0;
1169                 else
1170                         adapter->quad_port_a = true;
1171                 /* Reset for multiple quad port adapters */
1172                 if (++global_quad_port_a == 4)
1173                         global_quad_port_a = 0;
1174                 break;
1175         }
1176
1177         /* initialize the wol settings based on the eeprom settings */
1178         adapter->wol = adapter->eeprom_wol;
1179         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1180
1181         /* Auto detect PHY address */
1182         if (hw->mac_type == e1000_ce4100) {
1183                 for (i = 0; i < 32; i++) {
1184                         hw->phy_addr = i;
1185                         e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1186
1187                         if (tmp != 0 && tmp != 0xFF)
1188                                 break;
1189                 }
1190
1191                 if (i >= 32)
1192                         goto err_eeprom;
1193         }
1194
1195         /* reset the hardware with the new settings */
1196         e1000_reset(adapter);
1197
1198         strcpy(netdev->name, "eth%d");
1199         err = register_netdev(netdev);
1200         if (err)
1201                 goto err_register;
1202
1203         e1000_vlan_filter_on_off(adapter, false);
1204
1205         /* print bus type/speed/width info */
1206         e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1207                ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1208                ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1209                 (hw->bus_speed == e1000_bus_speed_120) ? 120 :
1210                 (hw->bus_speed == e1000_bus_speed_100) ? 100 :
1211                 (hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1212                ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1213                netdev->dev_addr);
1214
1215         /* carrier off reporting is important to ethtool even BEFORE open */
1216         netif_carrier_off(netdev);
1217
1218         e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1219
1220         cards_found++;
1221         return 0;
1222
1223 err_register:
1224 err_eeprom:
1225         e1000_phy_hw_reset(hw);
1226
1227         if (hw->flash_address)
1228                 iounmap(hw->flash_address);
1229         kfree(adapter->tx_ring);
1230         kfree(adapter->rx_ring);
1231 err_dma:
1232 err_sw_init:
1233 err_mdio_ioremap:
1234         iounmap(hw->ce4100_gbe_mdio_base_virt);
1235         iounmap(hw->hw_addr);
1236 err_ioremap:
1237         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1238         free_netdev(netdev);
1239 err_alloc_etherdev:
1240         pci_release_selected_regions(pdev, bars);
1241 err_pci_reg:
1242         if (!adapter || disable_dev)
1243                 pci_disable_device(pdev);
1244         return err;
1245 }
1246
1247 /**
1248  * e1000_remove - Device Removal Routine
1249  * @pdev: PCI device information struct
1250  *
1251  * e1000_remove is called by the PCI subsystem to alert the driver
1252  * that it should release a PCI device. That could be caused by a
1253  * Hot-Plug event, or because the driver is going to be removed from
1254  * memory.
1255  **/
1256 static void e1000_remove(struct pci_dev *pdev)
1257 {
1258         struct net_device *netdev = pci_get_drvdata(pdev);
1259         struct e1000_adapter *adapter = netdev_priv(netdev);
1260         struct e1000_hw *hw = &adapter->hw;
1261         bool disable_dev;
1262
1263         e1000_down_and_stop(adapter);
1264         e1000_release_manageability(adapter);
1265
1266         unregister_netdev(netdev);
1267
1268         e1000_phy_hw_reset(hw);
1269
1270         kfree(adapter->tx_ring);
1271         kfree(adapter->rx_ring);
1272
1273         if (hw->mac_type == e1000_ce4100)
1274                 iounmap(hw->ce4100_gbe_mdio_base_virt);
1275         iounmap(hw->hw_addr);
1276         if (hw->flash_address)
1277                 iounmap(hw->flash_address);
1278         pci_release_selected_regions(pdev, adapter->bars);
1279
1280         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1281         free_netdev(netdev);
1282
1283         if (disable_dev)
1284                 pci_disable_device(pdev);
1285 }
1286
1287 /**
1288  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1289  * @adapter: board private structure to initialize
1290  *
1291  * e1000_sw_init initializes the Adapter private data structure.
1292  * e1000_init_hw_struct MUST be called before this function
1293  **/
1294 static int e1000_sw_init(struct e1000_adapter *adapter)
1295 {
1296         adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1297
1298         adapter->num_tx_queues = 1;
1299         adapter->num_rx_queues = 1;
1300
1301         if (e1000_alloc_queues(adapter)) {
1302                 e_err(probe, "Unable to allocate memory for queues\n");
1303                 return -ENOMEM;
1304         }
1305
1306         /* Explicitly disable IRQ since the NIC can be in any state. */
1307         e1000_irq_disable(adapter);
1308
1309         spin_lock_init(&adapter->stats_lock);
1310
1311         set_bit(__E1000_DOWN, &adapter->flags);
1312
1313         return 0;
1314 }
1315
1316 /**
1317  * e1000_alloc_queues - Allocate memory for all rings
1318  * @adapter: board private structure to initialize
1319  *
1320  * We allocate one ring per queue at run-time since we don't know the
1321  * number of queues at compile-time.
1322  **/
1323 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1324 {
1325         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1326                                    sizeof(struct e1000_tx_ring), GFP_KERNEL);
1327         if (!adapter->tx_ring)
1328                 return -ENOMEM;
1329
1330         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1331                                    sizeof(struct e1000_rx_ring), GFP_KERNEL);
1332         if (!adapter->rx_ring) {
1333                 kfree(adapter->tx_ring);
1334                 return -ENOMEM;
1335         }
1336
1337         return E1000_SUCCESS;
1338 }
1339
1340 /**
1341  * e1000_open - Called when a network interface is made active
1342  * @netdev: network interface device structure
1343  *
1344  * Returns 0 on success, negative value on failure
1345  *
1346  * The open entry point is called when a network interface is made
1347  * active by the system (IFF_UP).  At this point all resources needed
1348  * for transmit and receive operations are allocated, the interrupt
1349  * handler is registered with the OS, the watchdog task is started,
1350  * and the stack is notified that the interface is ready.
1351  **/
1352 int e1000_open(struct net_device *netdev)
1353 {
1354         struct e1000_adapter *adapter = netdev_priv(netdev);
1355         struct e1000_hw *hw = &adapter->hw;
1356         int err;
1357
1358         /* disallow open during test */
1359         if (test_bit(__E1000_TESTING, &adapter->flags))
1360                 return -EBUSY;
1361
1362         netif_carrier_off(netdev);
1363
1364         /* allocate transmit descriptors */
1365         err = e1000_setup_all_tx_resources(adapter);
1366         if (err)
1367                 goto err_setup_tx;
1368
1369         /* allocate receive descriptors */
1370         err = e1000_setup_all_rx_resources(adapter);
1371         if (err)
1372                 goto err_setup_rx;
1373
1374         e1000_power_up_phy(adapter);
1375
1376         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1377         if ((hw->mng_cookie.status &
1378                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1379                 e1000_update_mng_vlan(adapter);
1380         }
1381
1382         /* before we allocate an interrupt, we must be ready to handle it.
1383          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1384          * as soon as we call pci_request_irq, so we have to setup our
1385          * clean_rx handler before we do so.
1386          */
1387         e1000_configure(adapter);
1388
1389         err = e1000_request_irq(adapter);
1390         if (err)
1391                 goto err_req_irq;
1392
1393         /* From here on the code is the same as e1000_up() */
1394         clear_bit(__E1000_DOWN, &adapter->flags);
1395
1396         napi_enable(&adapter->napi);
1397
1398         e1000_irq_enable(adapter);
1399
1400         netif_start_queue(netdev);
1401
1402         /* fire a link status change interrupt to start the watchdog */
1403         ew32(ICS, E1000_ICS_LSC);
1404
1405         return E1000_SUCCESS;
1406
1407 err_req_irq:
1408         e1000_power_down_phy(adapter);
1409         e1000_free_all_rx_resources(adapter);
1410 err_setup_rx:
1411         e1000_free_all_tx_resources(adapter);
1412 err_setup_tx:
1413         e1000_reset(adapter);
1414
1415         return err;
1416 }
1417
1418 /**
1419  * e1000_close - Disables a network interface
1420  * @netdev: network interface device structure
1421  *
1422  * Returns 0, this is not allowed to fail
1423  *
1424  * The close entry point is called when an interface is de-activated
1425  * by the OS.  The hardware is still under the drivers control, but
1426  * needs to be disabled.  A global MAC reset is issued to stop the
1427  * hardware, and all transmit and receive resources are freed.
1428  **/
1429 int e1000_close(struct net_device *netdev)
1430 {
1431         struct e1000_adapter *adapter = netdev_priv(netdev);
1432         struct e1000_hw *hw = &adapter->hw;
1433         int count = E1000_CHECK_RESET_COUNT;
1434
1435         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags) && count--)
1436                 usleep_range(10000, 20000);
1437
1438         WARN_ON(count < 0);
1439
1440         /* signal that we're down so that the reset task will no longer run */
1441         set_bit(__E1000_DOWN, &adapter->flags);
1442         clear_bit(__E1000_RESETTING, &adapter->flags);
1443
1444         e1000_down(adapter);
1445         e1000_power_down_phy(adapter);
1446         e1000_free_irq(adapter);
1447
1448         e1000_free_all_tx_resources(adapter);
1449         e1000_free_all_rx_resources(adapter);
1450
1451         /* kill manageability vlan ID if supported, but not if a vlan with
1452          * the same ID is registered on the host OS (let 8021q kill it)
1453          */
1454         if ((hw->mng_cookie.status &
1455              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1456             !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1457                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1458                                        adapter->mng_vlan_id);
1459         }
1460
1461         return 0;
1462 }
1463
1464 /**
1465  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1466  * @adapter: address of board private structure
1467  * @start: address of beginning of memory
1468  * @len: length of memory
1469  **/
1470 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1471                                   unsigned long len)
1472 {
1473         struct e1000_hw *hw = &adapter->hw;
1474         unsigned long begin = (unsigned long)start;
1475         unsigned long end = begin + len;
1476
1477         /* First rev 82545 and 82546 need to not allow any memory
1478          * write location to cross 64k boundary due to errata 23
1479          */
1480         if (hw->mac_type == e1000_82545 ||
1481             hw->mac_type == e1000_ce4100 ||
1482             hw->mac_type == e1000_82546) {
1483                 return ((begin ^ (end - 1)) >> 16) == 0;
1484         }
1485
1486         return true;
1487 }
1488
1489 /**
1490  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1491  * @adapter: board private structure
1492  * @txdr:    tx descriptor ring (for a specific queue) to setup
1493  *
1494  * Return 0 on success, negative on failure
1495  **/
1496 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1497                                     struct e1000_tx_ring *txdr)
1498 {
1499         struct pci_dev *pdev = adapter->pdev;
1500         int size;
1501
1502         size = sizeof(struct e1000_tx_buffer) * txdr->count;
1503         txdr->buffer_info = vzalloc(size);
1504         if (!txdr->buffer_info)
1505                 return -ENOMEM;
1506
1507         /* round up to nearest 4K */
1508
1509         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1510         txdr->size = ALIGN(txdr->size, 4096);
1511
1512         txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1513                                         GFP_KERNEL);
1514         if (!txdr->desc) {
1515 setup_tx_desc_die:
1516                 vfree(txdr->buffer_info);
1517                 return -ENOMEM;
1518         }
1519
1520         /* Fix for errata 23, can't cross 64kB boundary */
1521         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1522                 void *olddesc = txdr->desc;
1523                 dma_addr_t olddma = txdr->dma;
1524                 e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1525                       txdr->size, txdr->desc);
1526                 /* Try again, without freeing the previous */
1527                 txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1528                                                 &txdr->dma, GFP_KERNEL);
1529                 /* Failed allocation, critical failure */
1530                 if (!txdr->desc) {
1531                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1532                                           olddma);
1533                         goto setup_tx_desc_die;
1534                 }
1535
1536                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1537                         /* give up */
1538                         dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1539                                           txdr->dma);
1540                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1541                                           olddma);
1542                         e_err(probe, "Unable to allocate aligned memory "
1543                               "for the transmit descriptor ring\n");
1544                         vfree(txdr->buffer_info);
1545                         return -ENOMEM;
1546                 } else {
1547                         /* Free old allocation, new allocation was successful */
1548                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1549                                           olddma);
1550                 }
1551         }
1552         memset(txdr->desc, 0, txdr->size);
1553
1554         txdr->next_to_use = 0;
1555         txdr->next_to_clean = 0;
1556
1557         return 0;
1558 }
1559
1560 /**
1561  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1562  *                                (Descriptors) for all queues
1563  * @adapter: board private structure
1564  *
1565  * Return 0 on success, negative on failure
1566  **/
1567 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1568 {
1569         int i, err = 0;
1570
1571         for (i = 0; i < adapter->num_tx_queues; i++) {
1572                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1573                 if (err) {
1574                         e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1575                         for (i-- ; i >= 0; i--)
1576                                 e1000_free_tx_resources(adapter,
1577                                                         &adapter->tx_ring[i]);
1578                         break;
1579                 }
1580         }
1581
1582         return err;
1583 }
1584
1585 /**
1586  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1587  * @adapter: board private structure
1588  *
1589  * Configure the Tx unit of the MAC after a reset.
1590  **/
1591 static void e1000_configure_tx(struct e1000_adapter *adapter)
1592 {
1593         u64 tdba;
1594         struct e1000_hw *hw = &adapter->hw;
1595         u32 tdlen, tctl, tipg;
1596         u32 ipgr1, ipgr2;
1597
1598         /* Setup the HW Tx Head and Tail descriptor pointers */
1599
1600         switch (adapter->num_tx_queues) {
1601         case 1:
1602         default:
1603                 tdba = adapter->tx_ring[0].dma;
1604                 tdlen = adapter->tx_ring[0].count *
1605                         sizeof(struct e1000_tx_desc);
1606                 ew32(TDLEN, tdlen);
1607                 ew32(TDBAH, (tdba >> 32));
1608                 ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1609                 ew32(TDT, 0);
1610                 ew32(TDH, 0);
1611                 adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1612                                            E1000_TDH : E1000_82542_TDH);
1613                 adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1614                                            E1000_TDT : E1000_82542_TDT);
1615                 break;
1616         }
1617
1618         /* Set the default values for the Tx Inter Packet Gap timer */
1619         if ((hw->media_type == e1000_media_type_fiber ||
1620              hw->media_type == e1000_media_type_internal_serdes))
1621                 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1622         else
1623                 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1624
1625         switch (hw->mac_type) {
1626         case e1000_82542_rev2_0:
1627         case e1000_82542_rev2_1:
1628                 tipg = DEFAULT_82542_TIPG_IPGT;
1629                 ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1630                 ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1631                 break;
1632         default:
1633                 ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1634                 ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1635                 break;
1636         }
1637         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1638         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1639         ew32(TIPG, tipg);
1640
1641         /* Set the Tx Interrupt Delay register */
1642
1643         ew32(TIDV, adapter->tx_int_delay);
1644         if (hw->mac_type >= e1000_82540)
1645                 ew32(TADV, adapter->tx_abs_int_delay);
1646
1647         /* Program the Transmit Control Register */
1648
1649         tctl = er32(TCTL);
1650         tctl &= ~E1000_TCTL_CT;
1651         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1652                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1653
1654         e1000_config_collision_dist(hw);
1655
1656         /* Setup Transmit Descriptor Settings for eop descriptor */
1657         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1658
1659         /* only set IDE if we are delaying interrupts using the timers */
1660         if (adapter->tx_int_delay)
1661                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1662
1663         if (hw->mac_type < e1000_82543)
1664                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1665         else
1666                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1667
1668         /* Cache if we're 82544 running in PCI-X because we'll
1669          * need this to apply a workaround later in the send path.
1670          */
1671         if (hw->mac_type == e1000_82544 &&
1672             hw->bus_type == e1000_bus_type_pcix)
1673                 adapter->pcix_82544 = true;
1674
1675         ew32(TCTL, tctl);
1676
1677 }
1678
1679 /**
1680  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1681  * @adapter: board private structure
1682  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1683  *
1684  * Returns 0 on success, negative on failure
1685  **/
1686 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1687                                     struct e1000_rx_ring *rxdr)
1688 {
1689         struct pci_dev *pdev = adapter->pdev;
1690         int size, desc_len;
1691
1692         size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1693         rxdr->buffer_info = vzalloc(size);
1694         if (!rxdr->buffer_info)
1695                 return -ENOMEM;
1696
1697         desc_len = sizeof(struct e1000_rx_desc);
1698
1699         /* Round up to nearest 4K */
1700
1701         rxdr->size = rxdr->count * desc_len;
1702         rxdr->size = ALIGN(rxdr->size, 4096);
1703
1704         rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1705                                         GFP_KERNEL);
1706         if (!rxdr->desc) {
1707 setup_rx_desc_die:
1708                 vfree(rxdr->buffer_info);
1709                 return -ENOMEM;
1710         }
1711
1712         /* Fix for errata 23, can't cross 64kB boundary */
1713         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1714                 void *olddesc = rxdr->desc;
1715                 dma_addr_t olddma = rxdr->dma;
1716                 e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1717                       rxdr->size, rxdr->desc);
1718                 /* Try again, without freeing the previous */
1719                 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1720                                                 &rxdr->dma, GFP_KERNEL);
1721                 /* Failed allocation, critical failure */
1722                 if (!rxdr->desc) {
1723                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1724                                           olddma);
1725                         goto setup_rx_desc_die;
1726                 }
1727
1728                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1729                         /* give up */
1730                         dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1731                                           rxdr->dma);
1732                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1733                                           olddma);
1734                         e_err(probe, "Unable to allocate aligned memory for "
1735                               "the Rx descriptor ring\n");
1736                         goto setup_rx_desc_die;
1737                 } else {
1738                         /* Free old allocation, new allocation was successful */
1739                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1740                                           olddma);
1741                 }
1742         }
1743         memset(rxdr->desc, 0, rxdr->size);
1744
1745         rxdr->next_to_clean = 0;
1746         rxdr->next_to_use = 0;
1747         rxdr->rx_skb_top = NULL;
1748
1749         return 0;
1750 }
1751
1752 /**
1753  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1754  *                                (Descriptors) for all queues
1755  * @adapter: board private structure
1756  *
1757  * Return 0 on success, negative on failure
1758  **/
1759 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1760 {
1761         int i, err = 0;
1762
1763         for (i = 0; i < adapter->num_rx_queues; i++) {
1764                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1765                 if (err) {
1766                         e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1767                         for (i-- ; i >= 0; i--)
1768                                 e1000_free_rx_resources(adapter,
1769                                                         &adapter->rx_ring[i]);
1770                         break;
1771                 }
1772         }
1773
1774         return err;
1775 }
1776
1777 /**
1778  * e1000_setup_rctl - configure the receive control registers
1779  * @adapter: Board private structure
1780  **/
1781 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1782 {
1783         struct e1000_hw *hw = &adapter->hw;
1784         u32 rctl;
1785
1786         rctl = er32(RCTL);
1787
1788         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1789
1790         rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1791                 E1000_RCTL_RDMTS_HALF |
1792                 (hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1793
1794         if (hw->tbi_compatibility_on == 1)
1795                 rctl |= E1000_RCTL_SBP;
1796         else
1797                 rctl &= ~E1000_RCTL_SBP;
1798
1799         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1800                 rctl &= ~E1000_RCTL_LPE;
1801         else
1802                 rctl |= E1000_RCTL_LPE;
1803
1804         /* Setup buffer sizes */
1805         rctl &= ~E1000_RCTL_SZ_4096;
1806         rctl |= E1000_RCTL_BSEX;
1807         switch (adapter->rx_buffer_len) {
1808         case E1000_RXBUFFER_2048:
1809         default:
1810                 rctl |= E1000_RCTL_SZ_2048;
1811                 rctl &= ~E1000_RCTL_BSEX;
1812                 break;
1813         case E1000_RXBUFFER_4096:
1814                 rctl |= E1000_RCTL_SZ_4096;
1815                 break;
1816         case E1000_RXBUFFER_8192:
1817                 rctl |= E1000_RCTL_SZ_8192;
1818                 break;
1819         case E1000_RXBUFFER_16384:
1820                 rctl |= E1000_RCTL_SZ_16384;
1821                 break;
1822         }
1823
1824         /* This is useful for sniffing bad packets. */
1825         if (adapter->netdev->features & NETIF_F_RXALL) {
1826                 /* UPE and MPE will be handled by normal PROMISC logic
1827                  * in e1000e_set_rx_mode
1828                  */
1829                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1830                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
1831                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1832
1833                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1834                           E1000_RCTL_DPF | /* Allow filtered pause */
1835                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1836                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1837                  * and that breaks VLANs.
1838                  */
1839         }
1840
1841         ew32(RCTL, rctl);
1842 }
1843
1844 /**
1845  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1846  * @adapter: board private structure
1847  *
1848  * Configure the Rx unit of the MAC after a reset.
1849  **/
1850 static void e1000_configure_rx(struct e1000_adapter *adapter)
1851 {
1852         u64 rdba;
1853         struct e1000_hw *hw = &adapter->hw;
1854         u32 rdlen, rctl, rxcsum;
1855
1856         if (adapter->netdev->mtu > ETH_DATA_LEN) {
1857                 rdlen = adapter->rx_ring[0].count *
1858                         sizeof(struct e1000_rx_desc);
1859                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1860                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1861         } else {
1862                 rdlen = adapter->rx_ring[0].count *
1863                         sizeof(struct e1000_rx_desc);
1864                 adapter->clean_rx = e1000_clean_rx_irq;
1865                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1866         }
1867
1868         /* disable receives while setting up the descriptors */
1869         rctl = er32(RCTL);
1870         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1871
1872         /* set the Receive Delay Timer Register */
1873         ew32(RDTR, adapter->rx_int_delay);
1874
1875         if (hw->mac_type >= e1000_82540) {
1876                 ew32(RADV, adapter->rx_abs_int_delay);
1877                 if (adapter->itr_setting != 0)
1878                         ew32(ITR, 1000000000 / (adapter->itr * 256));
1879         }
1880
1881         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1882          * the Base and Length of the Rx Descriptor Ring
1883          */
1884         switch (adapter->num_rx_queues) {
1885         case 1:
1886         default:
1887                 rdba = adapter->rx_ring[0].dma;
1888                 ew32(RDLEN, rdlen);
1889                 ew32(RDBAH, (rdba >> 32));
1890                 ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1891                 ew32(RDT, 0);
1892                 ew32(RDH, 0);
1893                 adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1894                                            E1000_RDH : E1000_82542_RDH);
1895                 adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1896                                            E1000_RDT : E1000_82542_RDT);
1897                 break;
1898         }
1899
1900         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1901         if (hw->mac_type >= e1000_82543) {
1902                 rxcsum = er32(RXCSUM);
1903                 if (adapter->rx_csum)
1904                         rxcsum |= E1000_RXCSUM_TUOFL;
1905                 else
1906                         /* don't need to clear IPPCSE as it defaults to 0 */
1907                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1908                 ew32(RXCSUM, rxcsum);
1909         }
1910
1911         /* Enable Receives */
1912         ew32(RCTL, rctl | E1000_RCTL_EN);
1913 }
1914
1915 /**
1916  * e1000_free_tx_resources - Free Tx Resources per Queue
1917  * @adapter: board private structure
1918  * @tx_ring: Tx descriptor ring for a specific queue
1919  *
1920  * Free all transmit software resources
1921  **/
1922 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1923                                     struct e1000_tx_ring *tx_ring)
1924 {
1925         struct pci_dev *pdev = adapter->pdev;
1926
1927         e1000_clean_tx_ring(adapter, tx_ring);
1928
1929         vfree(tx_ring->buffer_info);
1930         tx_ring->buffer_info = NULL;
1931
1932         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1933                           tx_ring->dma);
1934
1935         tx_ring->desc = NULL;
1936 }
1937
1938 /**
1939  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1940  * @adapter: board private structure
1941  *
1942  * Free all transmit software resources
1943  **/
1944 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1945 {
1946         int i;
1947
1948         for (i = 0; i < adapter->num_tx_queues; i++)
1949                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1950 }
1951
1952 static void
1953 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1954                                  struct e1000_tx_buffer *buffer_info,
1955                                  int budget)
1956 {
1957         if (buffer_info->dma) {
1958                 if (buffer_info->mapped_as_page)
1959                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1960                                        buffer_info->length, DMA_TO_DEVICE);
1961                 else
1962                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1963                                          buffer_info->length,
1964                                          DMA_TO_DEVICE);
1965                 buffer_info->dma = 0;
1966         }
1967         if (buffer_info->skb) {
1968                 napi_consume_skb(buffer_info->skb, budget);
1969                 buffer_info->skb = NULL;
1970         }
1971         buffer_info->time_stamp = 0;
1972         /* buffer_info must be completely set up in the transmit path */
1973 }
1974
1975 /**
1976  * e1000_clean_tx_ring - Free Tx Buffers
1977  * @adapter: board private structure
1978  * @tx_ring: ring to be cleaned
1979  **/
1980 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1981                                 struct e1000_tx_ring *tx_ring)
1982 {
1983         struct e1000_hw *hw = &adapter->hw;
1984         struct e1000_tx_buffer *buffer_info;
1985         unsigned long size;
1986         unsigned int i;
1987
1988         /* Free all the Tx ring sk_buffs */
1989
1990         for (i = 0; i < tx_ring->count; i++) {
1991                 buffer_info = &tx_ring->buffer_info[i];
1992                 e1000_unmap_and_free_tx_resource(adapter, buffer_info, 0);
1993         }
1994
1995         netdev_reset_queue(adapter->netdev);
1996         size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
1997         memset(tx_ring->buffer_info, 0, size);
1998
1999         /* Zero out the descriptor ring */
2000
2001         memset(tx_ring->desc, 0, tx_ring->size);
2002
2003         tx_ring->next_to_use = 0;
2004         tx_ring->next_to_clean = 0;
2005         tx_ring->last_tx_tso = false;
2006
2007         writel(0, hw->hw_addr + tx_ring->tdh);
2008         writel(0, hw->hw_addr + tx_ring->tdt);
2009 }
2010
2011 /**
2012  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2013  * @adapter: board private structure
2014  **/
2015 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2016 {
2017         int i;
2018
2019         for (i = 0; i < adapter->num_tx_queues; i++)
2020                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2021 }
2022
2023 /**
2024  * e1000_free_rx_resources - Free Rx Resources
2025  * @adapter: board private structure
2026  * @rx_ring: ring to clean the resources from
2027  *
2028  * Free all receive software resources
2029  **/
2030 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2031                                     struct e1000_rx_ring *rx_ring)
2032 {
2033         struct pci_dev *pdev = adapter->pdev;
2034
2035         e1000_clean_rx_ring(adapter, rx_ring);
2036
2037         vfree(rx_ring->buffer_info);
2038         rx_ring->buffer_info = NULL;
2039
2040         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2041                           rx_ring->dma);
2042
2043         rx_ring->desc = NULL;
2044 }
2045
2046 /**
2047  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2048  * @adapter: board private structure
2049  *
2050  * Free all receive software resources
2051  **/
2052 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2053 {
2054         int i;
2055
2056         for (i = 0; i < adapter->num_rx_queues; i++)
2057                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2058 }
2059
2060 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2061 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2062 {
2063         return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2064                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2065 }
2066
2067 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2068 {
2069         unsigned int len = e1000_frag_len(a);
2070         u8 *data = netdev_alloc_frag(len);
2071
2072         if (likely(data))
2073                 data += E1000_HEADROOM;
2074         return data;
2075 }
2076
2077 /**
2078  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2079  * @adapter: board private structure
2080  * @rx_ring: ring to free buffers from
2081  **/
2082 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2083                                 struct e1000_rx_ring *rx_ring)
2084 {
2085         struct e1000_hw *hw = &adapter->hw;
2086         struct e1000_rx_buffer *buffer_info;
2087         struct pci_dev *pdev = adapter->pdev;
2088         unsigned long size;
2089         unsigned int i;
2090
2091         /* Free all the Rx netfrags */
2092         for (i = 0; i < rx_ring->count; i++) {
2093                 buffer_info = &rx_ring->buffer_info[i];
2094                 if (adapter->clean_rx == e1000_clean_rx_irq) {
2095                         if (buffer_info->dma)
2096                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
2097                                                  adapter->rx_buffer_len,
2098                                                  DMA_FROM_DEVICE);
2099                         if (buffer_info->rxbuf.data) {
2100                                 skb_free_frag(buffer_info->rxbuf.data);
2101                                 buffer_info->rxbuf.data = NULL;
2102                         }
2103                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2104                         if (buffer_info->dma)
2105                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
2106                                                adapter->rx_buffer_len,
2107                                                DMA_FROM_DEVICE);
2108                         if (buffer_info->rxbuf.page) {
2109                                 put_page(buffer_info->rxbuf.page);
2110                                 buffer_info->rxbuf.page = NULL;
2111                         }
2112                 }
2113
2114                 buffer_info->dma = 0;
2115         }
2116
2117         /* there also may be some cached data from a chained receive */
2118         napi_free_frags(&adapter->napi);
2119         rx_ring->rx_skb_top = NULL;
2120
2121         size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2122         memset(rx_ring->buffer_info, 0, size);
2123
2124         /* Zero out the descriptor ring */
2125         memset(rx_ring->desc, 0, rx_ring->size);
2126
2127         rx_ring->next_to_clean = 0;
2128         rx_ring->next_to_use = 0;
2129
2130         writel(0, hw->hw_addr + rx_ring->rdh);
2131         writel(0, hw->hw_addr + rx_ring->rdt);
2132 }
2133
2134 /**
2135  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2136  * @adapter: board private structure
2137  **/
2138 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2139 {
2140         int i;
2141
2142         for (i = 0; i < adapter->num_rx_queues; i++)
2143                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2144 }
2145
2146 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2147  * and memory write and invalidate disabled for certain operations
2148  */
2149 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2150 {
2151         struct e1000_hw *hw = &adapter->hw;
2152         struct net_device *netdev = adapter->netdev;
2153         u32 rctl;
2154
2155         e1000_pci_clear_mwi(hw);
2156
2157         rctl = er32(RCTL);
2158         rctl |= E1000_RCTL_RST;
2159         ew32(RCTL, rctl);
2160         E1000_WRITE_FLUSH();
2161         mdelay(5);
2162
2163         if (netif_running(netdev))
2164                 e1000_clean_all_rx_rings(adapter);
2165 }
2166
2167 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2168 {
2169         struct e1000_hw *hw = &adapter->hw;
2170         struct net_device *netdev = adapter->netdev;
2171         u32 rctl;
2172
2173         rctl = er32(RCTL);
2174         rctl &= ~E1000_RCTL_RST;
2175         ew32(RCTL, rctl);
2176         E1000_WRITE_FLUSH();
2177         mdelay(5);
2178
2179         if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2180                 e1000_pci_set_mwi(hw);
2181
2182         if (netif_running(netdev)) {
2183                 /* No need to loop, because 82542 supports only 1 queue */
2184                 struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2185                 e1000_configure_rx(adapter);
2186                 adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2187         }
2188 }
2189
2190 /**
2191  * e1000_set_mac - Change the Ethernet Address of the NIC
2192  * @netdev: network interface device structure
2193  * @p: pointer to an address structure
2194  *
2195  * Returns 0 on success, negative on failure
2196  **/
2197 static int e1000_set_mac(struct net_device *netdev, void *p)
2198 {
2199         struct e1000_adapter *adapter = netdev_priv(netdev);
2200         struct e1000_hw *hw = &adapter->hw;
2201         struct sockaddr *addr = p;
2202
2203         if (!is_valid_ether_addr(addr->sa_data))
2204                 return -EADDRNOTAVAIL;
2205
2206         /* 82542 2.0 needs to be in reset to write receive address registers */
2207
2208         if (hw->mac_type == e1000_82542_rev2_0)
2209                 e1000_enter_82542_rst(adapter);
2210
2211         eth_hw_addr_set(netdev, addr->sa_data);
2212         memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2213
2214         e1000_rar_set(hw, hw->mac_addr, 0);
2215
2216         if (hw->mac_type == e1000_82542_rev2_0)
2217                 e1000_leave_82542_rst(adapter);
2218
2219         return 0;
2220 }
2221
2222 /**
2223  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2224  * @netdev: network interface device structure
2225  *
2226  * The set_rx_mode entry point is called whenever the unicast or multicast
2227  * address lists or the network interface flags are updated. This routine is
2228  * responsible for configuring the hardware for proper unicast, multicast,
2229  * promiscuous mode, and all-multi behavior.
2230  **/
2231 static void e1000_set_rx_mode(struct net_device *netdev)
2232 {
2233         struct e1000_adapter *adapter = netdev_priv(netdev);
2234         struct e1000_hw *hw = &adapter->hw;
2235         struct netdev_hw_addr *ha;
2236         bool use_uc = false;
2237         u32 rctl;
2238         u32 hash_value;
2239         int i, rar_entries = E1000_RAR_ENTRIES;
2240         int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2241         u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2242
2243         if (!mcarray)
2244                 return;
2245
2246         /* Check for Promiscuous and All Multicast modes */
2247
2248         rctl = er32(RCTL);
2249
2250         if (netdev->flags & IFF_PROMISC) {
2251                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2252                 rctl &= ~E1000_RCTL_VFE;
2253         } else {
2254                 if (netdev->flags & IFF_ALLMULTI)
2255                         rctl |= E1000_RCTL_MPE;
2256                 else
2257                         rctl &= ~E1000_RCTL_MPE;
2258                 /* Enable VLAN filter if there is a VLAN */
2259                 if (e1000_vlan_used(adapter))
2260                         rctl |= E1000_RCTL_VFE;
2261         }
2262
2263         if (netdev_uc_count(netdev) > rar_entries - 1) {
2264                 rctl |= E1000_RCTL_UPE;
2265         } else if (!(netdev->flags & IFF_PROMISC)) {
2266                 rctl &= ~E1000_RCTL_UPE;
2267                 use_uc = true;
2268         }
2269
2270         ew32(RCTL, rctl);
2271
2272         /* 82542 2.0 needs to be in reset to write receive address registers */
2273
2274         if (hw->mac_type == e1000_82542_rev2_0)
2275                 e1000_enter_82542_rst(adapter);
2276
2277         /* load the first 14 addresses into the exact filters 1-14. Unicast
2278          * addresses take precedence to avoid disabling unicast filtering
2279          * when possible.
2280          *
2281          * RAR 0 is used for the station MAC address
2282          * if there are not 14 addresses, go ahead and clear the filters
2283          */
2284         i = 1;
2285         if (use_uc)
2286                 netdev_for_each_uc_addr(ha, netdev) {
2287                         if (i == rar_entries)
2288                                 break;
2289                         e1000_rar_set(hw, ha->addr, i++);
2290                 }
2291
2292         netdev_for_each_mc_addr(ha, netdev) {
2293                 if (i == rar_entries) {
2294                         /* load any remaining addresses into the hash table */
2295                         u32 hash_reg, hash_bit, mta;
2296                         hash_value = e1000_hash_mc_addr(hw, ha->addr);
2297                         hash_reg = (hash_value >> 5) & 0x7F;
2298                         hash_bit = hash_value & 0x1F;
2299                         mta = (1 << hash_bit);
2300                         mcarray[hash_reg] |= mta;
2301                 } else {
2302                         e1000_rar_set(hw, ha->addr, i++);
2303                 }
2304         }
2305
2306         for (; i < rar_entries; i++) {
2307                 E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2308                 E1000_WRITE_FLUSH();
2309                 E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2310                 E1000_WRITE_FLUSH();
2311         }
2312
2313         /* write the hash table completely, write from bottom to avoid
2314          * both stupid write combining chipsets, and flushing each write
2315          */
2316         for (i = mta_reg_count - 1; i >= 0 ; i--) {
2317                 /* If we are on an 82544 has an errata where writing odd
2318                  * offsets overwrites the previous even offset, but writing
2319                  * backwards over the range solves the issue by always
2320                  * writing the odd offset first
2321                  */
2322                 E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2323         }
2324         E1000_WRITE_FLUSH();
2325
2326         if (hw->mac_type == e1000_82542_rev2_0)
2327                 e1000_leave_82542_rst(adapter);
2328
2329         kfree(mcarray);
2330 }
2331
2332 /**
2333  * e1000_update_phy_info_task - get phy info
2334  * @work: work struct contained inside adapter struct
2335  *
2336  * Need to wait a few seconds after link up to get diagnostic information from
2337  * the phy
2338  */
2339 static void e1000_update_phy_info_task(struct work_struct *work)
2340 {
2341         struct e1000_adapter *adapter = container_of(work,
2342                                                      struct e1000_adapter,
2343                                                      phy_info_task.work);
2344
2345         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2346 }
2347
2348 /**
2349  * e1000_82547_tx_fifo_stall_task - task to complete work
2350  * @work: work struct contained inside adapter struct
2351  **/
2352 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2353 {
2354         struct e1000_adapter *adapter = container_of(work,
2355                                                      struct e1000_adapter,
2356                                                      fifo_stall_task.work);
2357         struct e1000_hw *hw = &adapter->hw;
2358         struct net_device *netdev = adapter->netdev;
2359         u32 tctl;
2360
2361         if (atomic_read(&adapter->tx_fifo_stall)) {
2362                 if ((er32(TDT) == er32(TDH)) &&
2363                    (er32(TDFT) == er32(TDFH)) &&
2364                    (er32(TDFTS) == er32(TDFHS))) {
2365                         tctl = er32(TCTL);
2366                         ew32(TCTL, tctl & ~E1000_TCTL_EN);
2367                         ew32(TDFT, adapter->tx_head_addr);
2368                         ew32(TDFH, adapter->tx_head_addr);
2369                         ew32(TDFTS, adapter->tx_head_addr);
2370                         ew32(TDFHS, adapter->tx_head_addr);
2371                         ew32(TCTL, tctl);
2372                         E1000_WRITE_FLUSH();
2373
2374                         adapter->tx_fifo_head = 0;
2375                         atomic_set(&adapter->tx_fifo_stall, 0);
2376                         netif_wake_queue(netdev);
2377                 } else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2378                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
2379                 }
2380         }
2381 }
2382
2383 bool e1000_has_link(struct e1000_adapter *adapter)
2384 {
2385         struct e1000_hw *hw = &adapter->hw;
2386         bool link_active = false;
2387
2388         /* get_link_status is set on LSC (link status) interrupt or rx
2389          * sequence error interrupt (except on intel ce4100).
2390          * get_link_status will stay false until the
2391          * e1000_check_for_link establishes link for copper adapters
2392          * ONLY
2393          */
2394         switch (hw->media_type) {
2395         case e1000_media_type_copper:
2396                 if (hw->mac_type == e1000_ce4100)
2397                         hw->get_link_status = 1;
2398                 if (hw->get_link_status) {
2399                         e1000_check_for_link(hw);
2400                         link_active = !hw->get_link_status;
2401                 } else {
2402                         link_active = true;
2403                 }
2404                 break;
2405         case e1000_media_type_fiber:
2406                 e1000_check_for_link(hw);
2407                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2408                 break;
2409         case e1000_media_type_internal_serdes:
2410                 e1000_check_for_link(hw);
2411                 link_active = hw->serdes_has_link;
2412                 break;
2413         default:
2414                 break;
2415         }
2416
2417         return link_active;
2418 }
2419
2420 /**
2421  * e1000_watchdog - work function
2422  * @work: work struct contained inside adapter struct
2423  **/
2424 static void e1000_watchdog(struct work_struct *work)
2425 {
2426         struct e1000_adapter *adapter = container_of(work,
2427                                                      struct e1000_adapter,
2428                                                      watchdog_task.work);
2429         struct e1000_hw *hw = &adapter->hw;
2430         struct net_device *netdev = adapter->netdev;
2431         struct e1000_tx_ring *txdr = adapter->tx_ring;
2432         u32 link, tctl;
2433
2434         link = e1000_has_link(adapter);
2435         if ((netif_carrier_ok(netdev)) && link)
2436                 goto link_up;
2437
2438         if (link) {
2439                 if (!netif_carrier_ok(netdev)) {
2440                         u32 ctrl;
2441                         /* update snapshot of PHY registers on LSC */
2442                         e1000_get_speed_and_duplex(hw,
2443                                                    &adapter->link_speed,
2444                                                    &adapter->link_duplex);
2445
2446                         ctrl = er32(CTRL);
2447                         pr_info("%s NIC Link is Up %d Mbps %s, "
2448                                 "Flow Control: %s\n",
2449                                 netdev->name,
2450                                 adapter->link_speed,
2451                                 adapter->link_duplex == FULL_DUPLEX ?
2452                                 "Full Duplex" : "Half Duplex",
2453                                 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2454                                 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2455                                 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2456                                 E1000_CTRL_TFCE) ? "TX" : "None")));
2457
2458                         /* adjust timeout factor according to speed/duplex */
2459                         adapter->tx_timeout_factor = 1;
2460                         switch (adapter->link_speed) {
2461                         case SPEED_10:
2462                                 adapter->tx_timeout_factor = 16;
2463                                 break;
2464                         case SPEED_100:
2465                                 /* maybe add some timeout factor ? */
2466                                 break;
2467                         }
2468
2469                         /* enable transmits in the hardware */
2470                         tctl = er32(TCTL);
2471                         tctl |= E1000_TCTL_EN;
2472                         ew32(TCTL, tctl);
2473
2474                         netif_carrier_on(netdev);
2475                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2476                                 schedule_delayed_work(&adapter->phy_info_task,
2477                                                       2 * HZ);
2478                         adapter->smartspeed = 0;
2479                 }
2480         } else {
2481                 if (netif_carrier_ok(netdev)) {
2482                         adapter->link_speed = 0;
2483                         adapter->link_duplex = 0;
2484                         pr_info("%s NIC Link is Down\n",
2485                                 netdev->name);
2486                         netif_carrier_off(netdev);
2487
2488                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2489                                 schedule_delayed_work(&adapter->phy_info_task,
2490                                                       2 * HZ);
2491                 }
2492
2493                 e1000_smartspeed(adapter);
2494         }
2495
2496 link_up:
2497         e1000_update_stats(adapter);
2498
2499         hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2500         adapter->tpt_old = adapter->stats.tpt;
2501         hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2502         adapter->colc_old = adapter->stats.colc;
2503
2504         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2505         adapter->gorcl_old = adapter->stats.gorcl;
2506         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2507         adapter->gotcl_old = adapter->stats.gotcl;
2508
2509         e1000_update_adaptive(hw);
2510
2511         if (!netif_carrier_ok(netdev)) {
2512                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2513                         /* We've lost link, so the controller stops DMA,
2514                          * but we've got queued Tx work that's never going
2515                          * to get done, so reset controller to flush Tx.
2516                          * (Do the reset outside of interrupt context).
2517                          */
2518                         adapter->tx_timeout_count++;
2519                         schedule_work(&adapter->reset_task);
2520                         /* exit immediately since reset is imminent */
2521                         return;
2522                 }
2523         }
2524
2525         /* Simple mode for Interrupt Throttle Rate (ITR) */
2526         if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2527                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
2528                  * Total asymmetrical Tx or Rx gets ITR=8000;
2529                  * everyone else is between 2000-8000.
2530                  */
2531                 u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2532                 u32 dif = (adapter->gotcl > adapter->gorcl ?
2533                             adapter->gotcl - adapter->gorcl :
2534                             adapter->gorcl - adapter->gotcl) / 10000;
2535                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2536
2537                 ew32(ITR, 1000000000 / (itr * 256));
2538         }
2539
2540         /* Cause software interrupt to ensure rx ring is cleaned */
2541         ew32(ICS, E1000_ICS_RXDMT0);
2542
2543         /* Force detection of hung controller every watchdog period */
2544         adapter->detect_tx_hung = true;
2545
2546         /* Reschedule the task */
2547         if (!test_bit(__E1000_DOWN, &adapter->flags))
2548                 schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2549 }
2550
2551 enum latency_range {
2552         lowest_latency = 0,
2553         low_latency = 1,
2554         bulk_latency = 2,
2555         latency_invalid = 255
2556 };
2557
2558 /**
2559  * e1000_update_itr - update the dynamic ITR value based on statistics
2560  * @adapter: pointer to adapter
2561  * @itr_setting: current adapter->itr
2562  * @packets: the number of packets during this measurement interval
2563  * @bytes: the number of bytes during this measurement interval
2564  *
2565  *      Stores a new ITR value based on packets and byte
2566  *      counts during the last interrupt.  The advantage of per interrupt
2567  *      computation is faster updates and more accurate ITR for the current
2568  *      traffic pattern.  Constants in this function were computed
2569  *      based on theoretical maximum wire speed and thresholds were set based
2570  *      on testing data as well as attempting to minimize response time
2571  *      while increasing bulk throughput.
2572  *      this functionality is controlled by the InterruptThrottleRate module
2573  *      parameter (see e1000_param.c)
2574  **/
2575 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2576                                      u16 itr_setting, int packets, int bytes)
2577 {
2578         unsigned int retval = itr_setting;
2579         struct e1000_hw *hw = &adapter->hw;
2580
2581         if (unlikely(hw->mac_type < e1000_82540))
2582                 goto update_itr_done;
2583
2584         if (packets == 0)
2585                 goto update_itr_done;
2586
2587         switch (itr_setting) {
2588         case lowest_latency:
2589                 /* jumbo frames get bulk treatment*/
2590                 if (bytes/packets > 8000)
2591                         retval = bulk_latency;
2592                 else if ((packets < 5) && (bytes > 512))
2593                         retval = low_latency;
2594                 break;
2595         case low_latency:  /* 50 usec aka 20000 ints/s */
2596                 if (bytes > 10000) {
2597                         /* jumbo frames need bulk latency setting */
2598                         if (bytes/packets > 8000)
2599                                 retval = bulk_latency;
2600                         else if ((packets < 10) || ((bytes/packets) > 1200))
2601                                 retval = bulk_latency;
2602                         else if ((packets > 35))
2603                                 retval = lowest_latency;
2604                 } else if (bytes/packets > 2000)
2605                         retval = bulk_latency;
2606                 else if (packets <= 2 && bytes < 512)
2607                         retval = lowest_latency;
2608                 break;
2609         case bulk_latency: /* 250 usec aka 4000 ints/s */
2610                 if (bytes > 25000) {
2611                         if (packets > 35)
2612                                 retval = low_latency;
2613                 } else if (bytes < 6000) {
2614                         retval = low_latency;
2615                 }
2616                 break;
2617         }
2618
2619 update_itr_done:
2620         return retval;
2621 }
2622
2623 static void e1000_set_itr(struct e1000_adapter *adapter)
2624 {
2625         struct e1000_hw *hw = &adapter->hw;
2626         u16 current_itr;
2627         u32 new_itr = adapter->itr;
2628
2629         if (unlikely(hw->mac_type < e1000_82540))
2630                 return;
2631
2632         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2633         if (unlikely(adapter->link_speed != SPEED_1000)) {
2634                 new_itr = 4000;
2635                 goto set_itr_now;
2636         }
2637
2638         adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2639                                            adapter->total_tx_packets,
2640                                            adapter->total_tx_bytes);
2641         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2642         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2643                 adapter->tx_itr = low_latency;
2644
2645         adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2646                                            adapter->total_rx_packets,
2647                                            adapter->total_rx_bytes);
2648         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2649         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2650                 adapter->rx_itr = low_latency;
2651
2652         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2653
2654         switch (current_itr) {
2655         /* counts and packets in update_itr are dependent on these numbers */
2656         case lowest_latency:
2657                 new_itr = 70000;
2658                 break;
2659         case low_latency:
2660                 new_itr = 20000; /* aka hwitr = ~200 */
2661                 break;
2662         case bulk_latency:
2663                 new_itr = 4000;
2664                 break;
2665         default:
2666                 break;
2667         }
2668
2669 set_itr_now:
2670         if (new_itr != adapter->itr) {
2671                 /* this attempts to bias the interrupt rate towards Bulk
2672                  * by adding intermediate steps when interrupt rate is
2673                  * increasing
2674                  */
2675                 new_itr = new_itr > adapter->itr ?
2676                           min(adapter->itr + (new_itr >> 2), new_itr) :
2677                           new_itr;
2678                 adapter->itr = new_itr;
2679                 ew32(ITR, 1000000000 / (new_itr * 256));
2680         }
2681 }
2682
2683 #define E1000_TX_FLAGS_CSUM             0x00000001
2684 #define E1000_TX_FLAGS_VLAN             0x00000002
2685 #define E1000_TX_FLAGS_TSO              0x00000004
2686 #define E1000_TX_FLAGS_IPV4             0x00000008
2687 #define E1000_TX_FLAGS_NO_FCS           0x00000010
2688 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2689 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2690
2691 static int e1000_tso(struct e1000_adapter *adapter,
2692                      struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2693                      __be16 protocol)
2694 {
2695         struct e1000_context_desc *context_desc;
2696         struct e1000_tx_buffer *buffer_info;
2697         unsigned int i;
2698         u32 cmd_length = 0;
2699         u16 ipcse = 0, tucse, mss;
2700         u8 ipcss, ipcso, tucss, tucso, hdr_len;
2701
2702         if (skb_is_gso(skb)) {
2703                 int err;
2704
2705                 err = skb_cow_head(skb, 0);
2706                 if (err < 0)
2707                         return err;
2708
2709                 hdr_len = skb_tcp_all_headers(skb);
2710                 mss = skb_shinfo(skb)->gso_size;
2711                 if (protocol == htons(ETH_P_IP)) {
2712                         struct iphdr *iph = ip_hdr(skb);
2713                         iph->tot_len = 0;
2714                         iph->check = 0;
2715                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2716                                                                  iph->daddr, 0,
2717                                                                  IPPROTO_TCP,
2718                                                                  0);
2719                         cmd_length = E1000_TXD_CMD_IP;
2720                         ipcse = skb_transport_offset(skb) - 1;
2721                 } else if (skb_is_gso_v6(skb)) {
2722                         tcp_v6_gso_csum_prep(skb);
2723                         ipcse = 0;
2724                 }
2725                 ipcss = skb_network_offset(skb);
2726                 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2727                 tucss = skb_transport_offset(skb);
2728                 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2729                 tucse = 0;
2730
2731                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2732                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2733
2734                 i = tx_ring->next_to_use;
2735                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2736                 buffer_info = &tx_ring->buffer_info[i];
2737
2738                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2739                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2740                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2741                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2742                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2743                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2744                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2745                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2746                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2747
2748                 buffer_info->time_stamp = jiffies;
2749                 buffer_info->next_to_watch = i;
2750
2751                 if (++i == tx_ring->count)
2752                         i = 0;
2753
2754                 tx_ring->next_to_use = i;
2755
2756                 return true;
2757         }
2758         return false;
2759 }
2760
2761 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2762                           struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2763                           __be16 protocol)
2764 {
2765         struct e1000_context_desc *context_desc;
2766         struct e1000_tx_buffer *buffer_info;
2767         unsigned int i;
2768         u8 css;
2769         u32 cmd_len = E1000_TXD_CMD_DEXT;
2770
2771         if (skb->ip_summed != CHECKSUM_PARTIAL)
2772                 return false;
2773
2774         switch (protocol) {
2775         case cpu_to_be16(ETH_P_IP):
2776                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2777                         cmd_len |= E1000_TXD_CMD_TCP;
2778                 break;
2779         case cpu_to_be16(ETH_P_IPV6):
2780                 /* XXX not handling all IPV6 headers */
2781                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2782                         cmd_len |= E1000_TXD_CMD_TCP;
2783                 break;
2784         default:
2785                 if (unlikely(net_ratelimit()))
2786                         e_warn(drv, "checksum_partial proto=%x!\n",
2787                                skb->protocol);
2788                 break;
2789         }
2790
2791         css = skb_checksum_start_offset(skb);
2792
2793         i = tx_ring->next_to_use;
2794         buffer_info = &tx_ring->buffer_info[i];
2795         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2796
2797         context_desc->lower_setup.ip_config = 0;
2798         context_desc->upper_setup.tcp_fields.tucss = css;
2799         context_desc->upper_setup.tcp_fields.tucso =
2800                 css + skb->csum_offset;
2801         context_desc->upper_setup.tcp_fields.tucse = 0;
2802         context_desc->tcp_seg_setup.data = 0;
2803         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2804
2805         buffer_info->time_stamp = jiffies;
2806         buffer_info->next_to_watch = i;
2807
2808         if (unlikely(++i == tx_ring->count))
2809                 i = 0;
2810
2811         tx_ring->next_to_use = i;
2812
2813         return true;
2814 }
2815
2816 #define E1000_MAX_TXD_PWR       12
2817 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2818
2819 static int e1000_tx_map(struct e1000_adapter *adapter,
2820                         struct e1000_tx_ring *tx_ring,
2821                         struct sk_buff *skb, unsigned int first,
2822                         unsigned int max_per_txd, unsigned int nr_frags,
2823                         unsigned int mss)
2824 {
2825         struct e1000_hw *hw = &adapter->hw;
2826         struct pci_dev *pdev = adapter->pdev;
2827         struct e1000_tx_buffer *buffer_info;
2828         unsigned int len = skb_headlen(skb);
2829         unsigned int offset = 0, size, count = 0, i;
2830         unsigned int f, bytecount, segs;
2831
2832         i = tx_ring->next_to_use;
2833
2834         while (len) {
2835                 buffer_info = &tx_ring->buffer_info[i];
2836                 size = min(len, max_per_txd);
2837                 /* Workaround for Controller erratum --
2838                  * descriptor for non-tso packet in a linear SKB that follows a
2839                  * tso gets written back prematurely before the data is fully
2840                  * DMA'd to the controller
2841                  */
2842                 if (!skb->data_len && tx_ring->last_tx_tso &&
2843                     !skb_is_gso(skb)) {
2844                         tx_ring->last_tx_tso = false;
2845                         size -= 4;
2846                 }
2847
2848                 /* Workaround for premature desc write-backs
2849                  * in TSO mode.  Append 4-byte sentinel desc
2850                  */
2851                 if (unlikely(mss && !nr_frags && size == len && size > 8))
2852                         size -= 4;
2853                 /* work-around for errata 10 and it applies
2854                  * to all controllers in PCI-X mode
2855                  * The fix is to make sure that the first descriptor of a
2856                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2857                  */
2858                 if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2859                              (size > 2015) && count == 0))
2860                         size = 2015;
2861
2862                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2863                  * terminating buffers within evenly-aligned dwords.
2864                  */
2865                 if (unlikely(adapter->pcix_82544 &&
2866                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2867                    size > 4))
2868                         size -= 4;
2869
2870                 buffer_info->length = size;
2871                 /* set time_stamp *before* dma to help avoid a possible race */
2872                 buffer_info->time_stamp = jiffies;
2873                 buffer_info->mapped_as_page = false;
2874                 buffer_info->dma = dma_map_single(&pdev->dev,
2875                                                   skb->data + offset,
2876                                                   size, DMA_TO_DEVICE);
2877                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2878                         goto dma_error;
2879                 buffer_info->next_to_watch = i;
2880
2881                 len -= size;
2882                 offset += size;
2883                 count++;
2884                 if (len) {
2885                         i++;
2886                         if (unlikely(i == tx_ring->count))
2887                                 i = 0;
2888                 }
2889         }
2890
2891         for (f = 0; f < nr_frags; f++) {
2892                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2893
2894                 len = skb_frag_size(frag);
2895                 offset = 0;
2896
2897                 while (len) {
2898                         unsigned long bufend;
2899                         i++;
2900                         if (unlikely(i == tx_ring->count))
2901                                 i = 0;
2902
2903                         buffer_info = &tx_ring->buffer_info[i];
2904                         size = min(len, max_per_txd);
2905                         /* Workaround for premature desc write-backs
2906                          * in TSO mode.  Append 4-byte sentinel desc
2907                          */
2908                         if (unlikely(mss && f == (nr_frags-1) &&
2909                             size == len && size > 8))
2910                                 size -= 4;
2911                         /* Workaround for potential 82544 hang in PCI-X.
2912                          * Avoid terminating buffers within evenly-aligned
2913                          * dwords.
2914                          */
2915                         bufend = (unsigned long)
2916                                 page_to_phys(skb_frag_page(frag));
2917                         bufend += offset + size - 1;
2918                         if (unlikely(adapter->pcix_82544 &&
2919                                      !(bufend & 4) &&
2920                                      size > 4))
2921                                 size -= 4;
2922
2923                         buffer_info->length = size;
2924                         buffer_info->time_stamp = jiffies;
2925                         buffer_info->mapped_as_page = true;
2926                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2927                                                 offset, size, DMA_TO_DEVICE);
2928                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2929                                 goto dma_error;
2930                         buffer_info->next_to_watch = i;
2931
2932                         len -= size;
2933                         offset += size;
2934                         count++;
2935                 }
2936         }
2937
2938         segs = skb_shinfo(skb)->gso_segs ?: 1;
2939         /* multiply data chunks by size of headers */
2940         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2941
2942         tx_ring->buffer_info[i].skb = skb;
2943         tx_ring->buffer_info[i].segs = segs;
2944         tx_ring->buffer_info[i].bytecount = bytecount;
2945         tx_ring->buffer_info[first].next_to_watch = i;
2946
2947         return count;
2948
2949 dma_error:
2950         dev_err(&pdev->dev, "TX DMA map failed\n");
2951         buffer_info->dma = 0;
2952         if (count)
2953                 count--;
2954
2955         while (count--) {
2956                 if (i == 0)
2957                         i += tx_ring->count;
2958                 i--;
2959                 buffer_info = &tx_ring->buffer_info[i];
2960                 e1000_unmap_and_free_tx_resource(adapter, buffer_info, 0);
2961         }
2962
2963         return 0;
2964 }
2965
2966 static void e1000_tx_queue(struct e1000_adapter *adapter,
2967                            struct e1000_tx_ring *tx_ring, int tx_flags,
2968                            int count)
2969 {
2970         struct e1000_tx_desc *tx_desc = NULL;
2971         struct e1000_tx_buffer *buffer_info;
2972         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2973         unsigned int i;
2974
2975         if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2976                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2977                              E1000_TXD_CMD_TSE;
2978                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2979
2980                 if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2981                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2982         }
2983
2984         if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2985                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2986                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2987         }
2988
2989         if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2990                 txd_lower |= E1000_TXD_CMD_VLE;
2991                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
2992         }
2993
2994         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
2995                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
2996
2997         i = tx_ring->next_to_use;
2998
2999         while (count--) {
3000                 buffer_info = &tx_ring->buffer_info[i];
3001                 tx_desc = E1000_TX_DESC(*tx_ring, i);
3002                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3003                 tx_desc->lower.data =
3004                         cpu_to_le32(txd_lower | buffer_info->length);
3005                 tx_desc->upper.data = cpu_to_le32(txd_upper);
3006                 if (unlikely(++i == tx_ring->count))
3007                         i = 0;
3008         }
3009
3010         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3011
3012         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3013         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3014                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3015
3016         /* Force memory writes to complete before letting h/w
3017          * know there are new descriptors to fetch.  (Only
3018          * applicable for weak-ordered memory model archs,
3019          * such as IA-64).
3020          */
3021         dma_wmb();
3022
3023         tx_ring->next_to_use = i;
3024 }
3025
3026 /* 82547 workaround to avoid controller hang in half-duplex environment.
3027  * The workaround is to avoid queuing a large packet that would span
3028  * the internal Tx FIFO ring boundary by notifying the stack to resend
3029  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3030  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3031  * to the beginning of the Tx FIFO.
3032  */
3033
3034 #define E1000_FIFO_HDR                  0x10
3035 #define E1000_82547_PAD_LEN             0x3E0
3036
3037 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3038                                        struct sk_buff *skb)
3039 {
3040         u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3041         u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3042
3043         skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3044
3045         if (adapter->link_duplex != HALF_DUPLEX)
3046                 goto no_fifo_stall_required;
3047
3048         if (atomic_read(&adapter->tx_fifo_stall))
3049                 return 1;
3050
3051         if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3052                 atomic_set(&adapter->tx_fifo_stall, 1);
3053                 return 1;
3054         }
3055
3056 no_fifo_stall_required:
3057         adapter->tx_fifo_head += skb_fifo_len;
3058         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3059                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
3060         return 0;
3061 }
3062
3063 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3064 {
3065         struct e1000_adapter *adapter = netdev_priv(netdev);
3066         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3067
3068         netif_stop_queue(netdev);
3069         /* Herbert's original patch had:
3070          *  smp_mb__after_netif_stop_queue();
3071          * but since that doesn't exist yet, just open code it.
3072          */
3073         smp_mb();
3074
3075         /* We need to check again in a case another CPU has just
3076          * made room available.
3077          */
3078         if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3079                 return -EBUSY;
3080
3081         /* A reprieve! */
3082         netif_start_queue(netdev);
3083         ++adapter->restart_queue;
3084         return 0;
3085 }
3086
3087 static int e1000_maybe_stop_tx(struct net_device *netdev,
3088                                struct e1000_tx_ring *tx_ring, int size)
3089 {
3090         if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3091                 return 0;
3092         return __e1000_maybe_stop_tx(netdev, size);
3093 }
3094
3095 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3096 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3097                                     struct net_device *netdev)
3098 {
3099         struct e1000_adapter *adapter = netdev_priv(netdev);
3100         struct e1000_hw *hw = &adapter->hw;
3101         struct e1000_tx_ring *tx_ring;
3102         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3103         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3104         unsigned int tx_flags = 0;
3105         unsigned int len = skb_headlen(skb);
3106         unsigned int nr_frags;
3107         unsigned int mss;
3108         int count = 0;
3109         int tso;
3110         unsigned int f;
3111         __be16 protocol = vlan_get_protocol(skb);
3112
3113         /* This goes back to the question of how to logically map a Tx queue
3114          * to a flow.  Right now, performance is impacted slightly negatively
3115          * if using multiple Tx queues.  If the stack breaks away from a
3116          * single qdisc implementation, we can look at this again.
3117          */
3118         tx_ring = adapter->tx_ring;
3119
3120         /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3121          * packets may get corrupted during padding by HW.
3122          * To WA this issue, pad all small packets manually.
3123          */
3124         if (eth_skb_pad(skb))
3125                 return NETDEV_TX_OK;
3126
3127         mss = skb_shinfo(skb)->gso_size;
3128         /* The controller does a simple calculation to
3129          * make sure there is enough room in the FIFO before
3130          * initiating the DMA for each buffer.  The calc is:
3131          * 4 = ceil(buffer len/mss).  To make sure we don't
3132          * overrun the FIFO, adjust the max buffer len if mss
3133          * drops.
3134          */
3135         if (mss) {
3136                 u8 hdr_len;
3137                 max_per_txd = min(mss << 2, max_per_txd);
3138                 max_txd_pwr = fls(max_per_txd) - 1;
3139
3140                 hdr_len = skb_tcp_all_headers(skb);
3141                 if (skb->data_len && hdr_len == len) {
3142                         switch (hw->mac_type) {
3143                         case e1000_82544: {
3144                                 unsigned int pull_size;
3145
3146                                 /* Make sure we have room to chop off 4 bytes,
3147                                  * and that the end alignment will work out to
3148                                  * this hardware's requirements
3149                                  * NOTE: this is a TSO only workaround
3150                                  * if end byte alignment not correct move us
3151                                  * into the next dword
3152                                  */
3153                                 if ((unsigned long)(skb_tail_pointer(skb) - 1)
3154                                     & 4)
3155                                         break;
3156                                 pull_size = min((unsigned int)4, skb->data_len);
3157                                 if (!__pskb_pull_tail(skb, pull_size)) {
3158                                         e_err(drv, "__pskb_pull_tail "
3159                                               "failed.\n");
3160                                         dev_kfree_skb_any(skb);
3161                                         return NETDEV_TX_OK;
3162                                 }
3163                                 len = skb_headlen(skb);
3164                                 break;
3165                         }
3166                         default:
3167                                 /* do nothing */
3168                                 break;
3169                         }
3170                 }
3171         }
3172
3173         /* reserve a descriptor for the offload context */
3174         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3175                 count++;
3176         count++;
3177
3178         /* Controller Erratum workaround */
3179         if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3180                 count++;
3181
3182         count += TXD_USE_COUNT(len, max_txd_pwr);
3183
3184         if (adapter->pcix_82544)
3185                 count++;
3186
3187         /* work-around for errata 10 and it applies to all controllers
3188          * in PCI-X mode, so add one more descriptor to the count
3189          */
3190         if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3191                         (len > 2015)))
3192                 count++;
3193
3194         nr_frags = skb_shinfo(skb)->nr_frags;
3195         for (f = 0; f < nr_frags; f++)
3196                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3197                                        max_txd_pwr);
3198         if (adapter->pcix_82544)
3199                 count += nr_frags;
3200
3201         /* need: count + 2 desc gap to keep tail from touching
3202          * head, otherwise try next time
3203          */
3204         if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3205                 return NETDEV_TX_BUSY;
3206
3207         if (unlikely((hw->mac_type == e1000_82547) &&
3208                      (e1000_82547_fifo_workaround(adapter, skb)))) {
3209                 netif_stop_queue(netdev);
3210                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3211                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
3212                 return NETDEV_TX_BUSY;
3213         }
3214
3215         if (skb_vlan_tag_present(skb)) {
3216                 tx_flags |= E1000_TX_FLAGS_VLAN;
3217                 tx_flags |= (skb_vlan_tag_get(skb) <<
3218                              E1000_TX_FLAGS_VLAN_SHIFT);
3219         }
3220
3221         first = tx_ring->next_to_use;
3222
3223         tso = e1000_tso(adapter, tx_ring, skb, protocol);
3224         if (tso < 0) {
3225                 dev_kfree_skb_any(skb);
3226                 return NETDEV_TX_OK;
3227         }
3228
3229         if (likely(tso)) {
3230                 if (likely(hw->mac_type != e1000_82544))
3231                         tx_ring->last_tx_tso = true;
3232                 tx_flags |= E1000_TX_FLAGS_TSO;
3233         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3234                 tx_flags |= E1000_TX_FLAGS_CSUM;
3235
3236         if (protocol == htons(ETH_P_IP))
3237                 tx_flags |= E1000_TX_FLAGS_IPV4;
3238
3239         if (unlikely(skb->no_fcs))
3240                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
3241
3242         count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3243                              nr_frags, mss);
3244
3245         if (count) {
3246                 /* The descriptors needed is higher than other Intel drivers
3247                  * due to a number of workarounds.  The breakdown is below:
3248                  * Data descriptors: MAX_SKB_FRAGS + 1
3249                  * Context Descriptor: 1
3250                  * Keep head from touching tail: 2
3251                  * Workarounds: 3
3252                  */
3253                 int desc_needed = MAX_SKB_FRAGS + 7;
3254
3255                 netdev_sent_queue(netdev, skb->len);
3256                 skb_tx_timestamp(skb);
3257
3258                 e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3259
3260                 /* 82544 potentially requires twice as many data descriptors
3261                  * in order to guarantee buffers don't end on evenly-aligned
3262                  * dwords
3263                  */
3264                 if (adapter->pcix_82544)
3265                         desc_needed += MAX_SKB_FRAGS + 1;
3266
3267                 /* Make sure there is space in the ring for the next send. */
3268                 e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3269
3270                 if (!netdev_xmit_more() ||
3271                     netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3272                         writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3273                 }
3274         } else {
3275                 dev_kfree_skb_any(skb);
3276                 tx_ring->buffer_info[first].time_stamp = 0;
3277                 tx_ring->next_to_use = first;
3278         }
3279
3280         return NETDEV_TX_OK;
3281 }
3282
3283 #define NUM_REGS 38 /* 1 based count */
3284 static void e1000_regdump(struct e1000_adapter *adapter)
3285 {
3286         struct e1000_hw *hw = &adapter->hw;
3287         u32 regs[NUM_REGS];
3288         u32 *regs_buff = regs;
3289         int i = 0;
3290
3291         static const char * const reg_name[] = {
3292                 "CTRL",  "STATUS",
3293                 "RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3294                 "TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3295                 "TIDV", "TXDCTL", "TADV", "TARC0",
3296                 "TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3297                 "TXDCTL1", "TARC1",
3298                 "CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3299                 "TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3300                 "RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3301         };
3302
3303         regs_buff[0]  = er32(CTRL);
3304         regs_buff[1]  = er32(STATUS);
3305
3306         regs_buff[2]  = er32(RCTL);
3307         regs_buff[3]  = er32(RDLEN);
3308         regs_buff[4]  = er32(RDH);
3309         regs_buff[5]  = er32(RDT);
3310         regs_buff[6]  = er32(RDTR);
3311
3312         regs_buff[7]  = er32(TCTL);
3313         regs_buff[8]  = er32(TDBAL);
3314         regs_buff[9]  = er32(TDBAH);
3315         regs_buff[10] = er32(TDLEN);
3316         regs_buff[11] = er32(TDH);
3317         regs_buff[12] = er32(TDT);
3318         regs_buff[13] = er32(TIDV);
3319         regs_buff[14] = er32(TXDCTL);
3320         regs_buff[15] = er32(TADV);
3321         regs_buff[16] = er32(TARC0);
3322
3323         regs_buff[17] = er32(TDBAL1);
3324         regs_buff[18] = er32(TDBAH1);
3325         regs_buff[19] = er32(TDLEN1);
3326         regs_buff[20] = er32(TDH1);
3327         regs_buff[21] = er32(TDT1);
3328         regs_buff[22] = er32(TXDCTL1);
3329         regs_buff[23] = er32(TARC1);
3330         regs_buff[24] = er32(CTRL_EXT);
3331         regs_buff[25] = er32(ERT);
3332         regs_buff[26] = er32(RDBAL0);
3333         regs_buff[27] = er32(RDBAH0);
3334         regs_buff[28] = er32(TDFH);
3335         regs_buff[29] = er32(TDFT);
3336         regs_buff[30] = er32(TDFHS);
3337         regs_buff[31] = er32(TDFTS);
3338         regs_buff[32] = er32(TDFPC);
3339         regs_buff[33] = er32(RDFH);
3340         regs_buff[34] = er32(RDFT);
3341         regs_buff[35] = er32(RDFHS);
3342         regs_buff[36] = er32(RDFTS);
3343         regs_buff[37] = er32(RDFPC);
3344
3345         pr_info("Register dump\n");
3346         for (i = 0; i < NUM_REGS; i++)
3347                 pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3348 }
3349
3350 /*
3351  * e1000_dump: Print registers, tx ring and rx ring
3352  */
3353 static void e1000_dump(struct e1000_adapter *adapter)
3354 {
3355         /* this code doesn't handle multiple rings */
3356         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3357         struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3358         int i;
3359
3360         if (!netif_msg_hw(adapter))
3361                 return;
3362
3363         /* Print Registers */
3364         e1000_regdump(adapter);
3365
3366         /* transmit dump */
3367         pr_info("TX Desc ring0 dump\n");
3368
3369         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3370          *
3371          * Legacy Transmit Descriptor
3372          *   +--------------------------------------------------------------+
3373          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3374          *   +--------------------------------------------------------------+
3375          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3376          *   +--------------------------------------------------------------+
3377          *   63       48 47        36 35    32 31     24 23    16 15        0
3378          *
3379          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3380          *   63      48 47    40 39       32 31             16 15    8 7      0
3381          *   +----------------------------------------------------------------+
3382          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3383          *   +----------------------------------------------------------------+
3384          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3385          *   +----------------------------------------------------------------+
3386          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3387          *
3388          * Extended Data Descriptor (DTYP=0x1)
3389          *   +----------------------------------------------------------------+
3390          * 0 |                     Buffer Address [63:0]                      |
3391          *   +----------------------------------------------------------------+
3392          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3393          *   +----------------------------------------------------------------+
3394          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3395          */
3396         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3397         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3398
3399         if (!netif_msg_tx_done(adapter))
3400                 goto rx_ring_summary;
3401
3402         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3403                 struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3404                 struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3405                 struct my_u { __le64 a; __le64 b; };
3406                 struct my_u *u = (struct my_u *)tx_desc;
3407                 const char *type;
3408
3409                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3410                         type = "NTC/U";
3411                 else if (i == tx_ring->next_to_use)
3412                         type = "NTU";
3413                 else if (i == tx_ring->next_to_clean)
3414                         type = "NTC";
3415                 else
3416                         type = "";
3417
3418                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3419                         ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3420                         le64_to_cpu(u->a), le64_to_cpu(u->b),
3421                         (u64)buffer_info->dma, buffer_info->length,
3422                         buffer_info->next_to_watch,
3423                         (u64)buffer_info->time_stamp, buffer_info->skb, type);
3424         }
3425
3426 rx_ring_summary:
3427         /* receive dump */
3428         pr_info("\nRX Desc ring dump\n");
3429
3430         /* Legacy Receive Descriptor Format
3431          *
3432          * +-----------------------------------------------------+
3433          * |                Buffer Address [63:0]                |
3434          * +-----------------------------------------------------+
3435          * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3436          * +-----------------------------------------------------+
3437          * 63       48 47    40 39      32 31         16 15      0
3438          */
3439         pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3440
3441         if (!netif_msg_rx_status(adapter))
3442                 goto exit;
3443
3444         for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3445                 struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3446                 struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3447                 struct my_u { __le64 a; __le64 b; };
3448                 struct my_u *u = (struct my_u *)rx_desc;
3449                 const char *type;
3450
3451                 if (i == rx_ring->next_to_use)
3452                         type = "NTU";
3453                 else if (i == rx_ring->next_to_clean)
3454                         type = "NTC";
3455                 else
3456                         type = "";
3457
3458                 pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3459                         i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3460                         (u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3461         } /* for */
3462
3463         /* dump the descriptor caches */
3464         /* rx */
3465         pr_info("Rx descriptor cache in 64bit format\n");
3466         for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3467                 pr_info("R%04X: %08X|%08X %08X|%08X\n",
3468                         i,
3469                         readl(adapter->hw.hw_addr + i+4),
3470                         readl(adapter->hw.hw_addr + i),
3471                         readl(adapter->hw.hw_addr + i+12),
3472                         readl(adapter->hw.hw_addr + i+8));
3473         }
3474         /* tx */
3475         pr_info("Tx descriptor cache in 64bit format\n");
3476         for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3477                 pr_info("T%04X: %08X|%08X %08X|%08X\n",
3478                         i,
3479                         readl(adapter->hw.hw_addr + i+4),
3480                         readl(adapter->hw.hw_addr + i),
3481                         readl(adapter->hw.hw_addr + i+12),
3482                         readl(adapter->hw.hw_addr + i+8));
3483         }
3484 exit:
3485         return;
3486 }
3487
3488 /**
3489  * e1000_tx_timeout - Respond to a Tx Hang
3490  * @netdev: network interface device structure
3491  * @txqueue: number of the Tx queue that hung (unused)
3492  **/
3493 static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
3494 {
3495         struct e1000_adapter *adapter = netdev_priv(netdev);
3496
3497         /* Do the reset outside of interrupt context */
3498         adapter->tx_timeout_count++;
3499         schedule_work(&adapter->reset_task);
3500 }
3501
3502 static void e1000_reset_task(struct work_struct *work)
3503 {
3504         struct e1000_adapter *adapter =
3505                 container_of(work, struct e1000_adapter, reset_task);
3506
3507         e_err(drv, "Reset adapter\n");
3508         e1000_reinit_locked(adapter);
3509 }
3510
3511 /**
3512  * e1000_change_mtu - Change the Maximum Transfer Unit
3513  * @netdev: network interface device structure
3514  * @new_mtu: new value for maximum frame size
3515  *
3516  * Returns 0 on success, negative on failure
3517  **/
3518 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3519 {
3520         struct e1000_adapter *adapter = netdev_priv(netdev);
3521         struct e1000_hw *hw = &adapter->hw;
3522         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3523
3524         /* Adapter-specific max frame size limits. */
3525         switch (hw->mac_type) {
3526         case e1000_undefined ... e1000_82542_rev2_1:
3527                 if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3528                         e_err(probe, "Jumbo Frames not supported.\n");
3529                         return -EINVAL;
3530                 }
3531                 break;
3532         default:
3533                 /* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3534                 break;
3535         }
3536
3537         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3538                 msleep(1);
3539         /* e1000_down has a dependency on max_frame_size */
3540         hw->max_frame_size = max_frame;
3541         if (netif_running(netdev)) {
3542                 /* prevent buffers from being reallocated */
3543                 adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3544                 e1000_down(adapter);
3545         }
3546
3547         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3548          * means we reserve 2 more, this pushes us to allocate from the next
3549          * larger slab size.
3550          * i.e. RXBUFFER_2048 --> size-4096 slab
3551          * however with the new *_jumbo_rx* routines, jumbo receives will use
3552          * fragmented skbs
3553          */
3554
3555         if (max_frame <= E1000_RXBUFFER_2048)
3556                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3557         else
3558 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3559                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3560 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3561                 adapter->rx_buffer_len = PAGE_SIZE;
3562 #endif
3563
3564         /* adjust allocation if LPE protects us, and we aren't using SBP */
3565         if (!hw->tbi_compatibility_on &&
3566             ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3567              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3568                 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3569
3570         netdev_dbg(netdev, "changing MTU from %d to %d\n",
3571                    netdev->mtu, new_mtu);
3572         WRITE_ONCE(netdev->mtu, new_mtu);
3573
3574         if (netif_running(netdev))
3575                 e1000_up(adapter);
3576         else
3577                 e1000_reset(adapter);
3578
3579         clear_bit(__E1000_RESETTING, &adapter->flags);
3580
3581         return 0;
3582 }
3583
3584 /**
3585  * e1000_update_stats - Update the board statistics counters
3586  * @adapter: board private structure
3587  **/
3588 void e1000_update_stats(struct e1000_adapter *adapter)
3589 {
3590         struct net_device *netdev = adapter->netdev;
3591         struct e1000_hw *hw = &adapter->hw;
3592         struct pci_dev *pdev = adapter->pdev;
3593         unsigned long flags;
3594         u16 phy_tmp;
3595
3596 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3597
3598         /* Prevent stats update while adapter is being reset, or if the pci
3599          * connection is down.
3600          */
3601         if (adapter->link_speed == 0)
3602                 return;
3603         if (pci_channel_offline(pdev))
3604                 return;
3605
3606         spin_lock_irqsave(&adapter->stats_lock, flags);
3607
3608         /* these counters are modified from e1000_tbi_adjust_stats,
3609          * called from the interrupt context, so they must only
3610          * be written while holding adapter->stats_lock
3611          */
3612
3613         adapter->stats.crcerrs += er32(CRCERRS);
3614         adapter->stats.gprc += er32(GPRC);
3615         adapter->stats.gorcl += er32(GORCL);
3616         adapter->stats.gorch += er32(GORCH);
3617         adapter->stats.bprc += er32(BPRC);
3618         adapter->stats.mprc += er32(MPRC);
3619         adapter->stats.roc += er32(ROC);
3620
3621         adapter->stats.prc64 += er32(PRC64);
3622         adapter->stats.prc127 += er32(PRC127);
3623         adapter->stats.prc255 += er32(PRC255);
3624         adapter->stats.prc511 += er32(PRC511);
3625         adapter->stats.prc1023 += er32(PRC1023);
3626         adapter->stats.prc1522 += er32(PRC1522);
3627
3628         adapter->stats.symerrs += er32(SYMERRS);
3629         adapter->stats.mpc += er32(MPC);
3630         adapter->stats.scc += er32(SCC);
3631         adapter->stats.ecol += er32(ECOL);
3632         adapter->stats.mcc += er32(MCC);
3633         adapter->stats.latecol += er32(LATECOL);
3634         adapter->stats.dc += er32(DC);
3635         adapter->stats.sec += er32(SEC);
3636         adapter->stats.rlec += er32(RLEC);
3637         adapter->stats.xonrxc += er32(XONRXC);
3638         adapter->stats.xontxc += er32(XONTXC);
3639         adapter->stats.xoffrxc += er32(XOFFRXC);
3640         adapter->stats.xofftxc += er32(XOFFTXC);
3641         adapter->stats.fcruc += er32(FCRUC);
3642         adapter->stats.gptc += er32(GPTC);
3643         adapter->stats.gotcl += er32(GOTCL);
3644         adapter->stats.gotch += er32(GOTCH);
3645         adapter->stats.rnbc += er32(RNBC);
3646         adapter->stats.ruc += er32(RUC);
3647         adapter->stats.rfc += er32(RFC);
3648         adapter->stats.rjc += er32(RJC);
3649         adapter->stats.torl += er32(TORL);
3650         adapter->stats.torh += er32(TORH);
3651         adapter->stats.totl += er32(TOTL);
3652         adapter->stats.toth += er32(TOTH);
3653         adapter->stats.tpr += er32(TPR);
3654
3655         adapter->stats.ptc64 += er32(PTC64);
3656         adapter->stats.ptc127 += er32(PTC127);
3657         adapter->stats.ptc255 += er32(PTC255);
3658         adapter->stats.ptc511 += er32(PTC511);
3659         adapter->stats.ptc1023 += er32(PTC1023);
3660         adapter->stats.ptc1522 += er32(PTC1522);
3661
3662         adapter->stats.mptc += er32(MPTC);
3663         adapter->stats.bptc += er32(BPTC);
3664
3665         /* used for adaptive IFS */
3666
3667         hw->tx_packet_delta = er32(TPT);
3668         adapter->stats.tpt += hw->tx_packet_delta;
3669         hw->collision_delta = er32(COLC);
3670         adapter->stats.colc += hw->collision_delta;
3671
3672         if (hw->mac_type >= e1000_82543) {
3673                 adapter->stats.algnerrc += er32(ALGNERRC);
3674                 adapter->stats.rxerrc += er32(RXERRC);
3675                 adapter->stats.tncrs += er32(TNCRS);
3676                 adapter->stats.cexterr += er32(CEXTERR);
3677                 adapter->stats.tsctc += er32(TSCTC);
3678                 adapter->stats.tsctfc += er32(TSCTFC);
3679         }
3680
3681         /* Fill out the OS statistics structure */
3682         netdev->stats.multicast = adapter->stats.mprc;
3683         netdev->stats.collisions = adapter->stats.colc;
3684
3685         /* Rx Errors */
3686
3687         /* RLEC on some newer hardware can be incorrect so build
3688          * our own version based on RUC and ROC
3689          */
3690         netdev->stats.rx_errors = adapter->stats.rxerrc +
3691                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3692                 adapter->stats.ruc + adapter->stats.roc +
3693                 adapter->stats.cexterr;
3694         adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3695         netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3696         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3697         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3698         netdev->stats.rx_missed_errors = adapter->stats.mpc;
3699
3700         /* Tx Errors */
3701         adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3702         netdev->stats.tx_errors = adapter->stats.txerrc;
3703         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3704         netdev->stats.tx_window_errors = adapter->stats.latecol;
3705         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3706         if (hw->bad_tx_carr_stats_fd &&
3707             adapter->link_duplex == FULL_DUPLEX) {
3708                 netdev->stats.tx_carrier_errors = 0;
3709                 adapter->stats.tncrs = 0;
3710         }
3711
3712         /* Tx Dropped needs to be maintained elsewhere */
3713
3714         /* Phy Stats */
3715         if (hw->media_type == e1000_media_type_copper) {
3716                 if ((adapter->link_speed == SPEED_1000) &&
3717                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3718                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3719                         adapter->phy_stats.idle_errors += phy_tmp;
3720                 }
3721
3722                 if ((hw->mac_type <= e1000_82546) &&
3723                    (hw->phy_type == e1000_phy_m88) &&
3724                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3725                         adapter->phy_stats.receive_errors += phy_tmp;
3726         }
3727
3728         /* Management Stats */
3729         if (hw->has_smbus) {
3730                 adapter->stats.mgptc += er32(MGTPTC);
3731                 adapter->stats.mgprc += er32(MGTPRC);
3732                 adapter->stats.mgpdc += er32(MGTPDC);
3733         }
3734
3735         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3736 }
3737
3738 /**
3739  * e1000_intr - Interrupt Handler
3740  * @irq: interrupt number
3741  * @data: pointer to a network interface device structure
3742  **/
3743 static irqreturn_t e1000_intr(int irq, void *data)
3744 {
3745         struct net_device *netdev = data;
3746         struct e1000_adapter *adapter = netdev_priv(netdev);
3747         struct e1000_hw *hw = &adapter->hw;
3748         u32 icr = er32(ICR);
3749
3750         if (unlikely((!icr)))
3751                 return IRQ_NONE;  /* Not our interrupt */
3752
3753         /* we might have caused the interrupt, but the above
3754          * read cleared it, and just in case the driver is
3755          * down there is nothing to do so return handled
3756          */
3757         if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3758                 return IRQ_HANDLED;
3759
3760         if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3761                 hw->get_link_status = 1;
3762                 /* guard against interrupt when we're going down */
3763                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3764                         schedule_delayed_work(&adapter->watchdog_task, 1);
3765         }
3766
3767         /* disable interrupts, without the synchronize_irq bit */
3768         ew32(IMC, ~0);
3769         E1000_WRITE_FLUSH();
3770
3771         if (likely(napi_schedule_prep(&adapter->napi))) {
3772                 adapter->total_tx_bytes = 0;
3773                 adapter->total_tx_packets = 0;
3774                 adapter->total_rx_bytes = 0;
3775                 adapter->total_rx_packets = 0;
3776                 __napi_schedule(&adapter->napi);
3777         } else {
3778                 /* this really should not happen! if it does it is basically a
3779                  * bug, but not a hard error, so enable ints and continue
3780                  */
3781                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3782                         e1000_irq_enable(adapter);
3783         }
3784
3785         return IRQ_HANDLED;
3786 }
3787
3788 /**
3789  * e1000_clean - NAPI Rx polling callback
3790  * @napi: napi struct containing references to driver info
3791  * @budget: budget given to driver for receive packets
3792  **/
3793 static int e1000_clean(struct napi_struct *napi, int budget)
3794 {
3795         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3796                                                      napi);
3797         int tx_clean_complete = 0, work_done = 0;
3798
3799         tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3800
3801         adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3802
3803         if (!tx_clean_complete || work_done == budget)
3804                 return budget;
3805
3806         /* Exit the polling mode, but don't re-enable interrupts if stack might
3807          * poll us due to busy-polling
3808          */
3809         if (likely(napi_complete_done(napi, work_done))) {
3810                 if (likely(adapter->itr_setting & 3))
3811                         e1000_set_itr(adapter);
3812                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3813                         e1000_irq_enable(adapter);
3814         }
3815
3816         return work_done;
3817 }
3818
3819 /**
3820  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3821  * @adapter: board private structure
3822  * @tx_ring: ring to clean
3823  **/
3824 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3825                                struct e1000_tx_ring *tx_ring)
3826 {
3827         struct e1000_hw *hw = &adapter->hw;
3828         struct net_device *netdev = adapter->netdev;
3829         struct e1000_tx_desc *tx_desc, *eop_desc;
3830         struct e1000_tx_buffer *buffer_info;
3831         unsigned int i, eop;
3832         unsigned int count = 0;
3833         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3834         unsigned int bytes_compl = 0, pkts_compl = 0;
3835
3836         i = tx_ring->next_to_clean;
3837         eop = tx_ring->buffer_info[i].next_to_watch;
3838         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3839
3840         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3841                (count < tx_ring->count)) {
3842                 bool cleaned = false;
3843                 dma_rmb();      /* read buffer_info after eop_desc */
3844                 for ( ; !cleaned; count++) {
3845                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3846                         buffer_info = &tx_ring->buffer_info[i];
3847                         cleaned = (i == eop);
3848
3849                         if (cleaned) {
3850                                 total_tx_packets += buffer_info->segs;
3851                                 total_tx_bytes += buffer_info->bytecount;
3852                                 if (buffer_info->skb) {
3853                                         bytes_compl += buffer_info->skb->len;
3854                                         pkts_compl++;
3855                                 }
3856
3857                         }
3858                         e1000_unmap_and_free_tx_resource(adapter, buffer_info,
3859                                                          64);
3860                         tx_desc->upper.data = 0;
3861
3862                         if (unlikely(++i == tx_ring->count))
3863                                 i = 0;
3864                 }
3865
3866                 eop = tx_ring->buffer_info[i].next_to_watch;
3867                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3868         }
3869
3870         /* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3871          * which will reuse the cleaned buffers.
3872          */
3873         smp_store_release(&tx_ring->next_to_clean, i);
3874
3875         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3876
3877 #define TX_WAKE_THRESHOLD 32
3878         if (unlikely(count && netif_carrier_ok(netdev) &&
3879                      E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3880                 /* Make sure that anybody stopping the queue after this
3881                  * sees the new next_to_clean.
3882                  */
3883                 smp_mb();
3884
3885                 if (netif_queue_stopped(netdev) &&
3886                     !(test_bit(__E1000_DOWN, &adapter->flags))) {
3887                         netif_wake_queue(netdev);
3888                         ++adapter->restart_queue;
3889                 }
3890         }
3891
3892         if (adapter->detect_tx_hung) {
3893                 /* Detect a transmit hang in hardware, this serializes the
3894                  * check with the clearing of time_stamp and movement of i
3895                  */
3896                 adapter->detect_tx_hung = false;
3897                 if (tx_ring->buffer_info[eop].time_stamp &&
3898                     time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3899                                (adapter->tx_timeout_factor * HZ)) &&
3900                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3901
3902                         /* detected Tx unit hang */
3903                         e_err(drv, "Detected Tx Unit Hang\n"
3904                               "  Tx Queue             <%lu>\n"
3905                               "  TDH                  <%x>\n"
3906                               "  TDT                  <%x>\n"
3907                               "  next_to_use          <%x>\n"
3908                               "  next_to_clean        <%x>\n"
3909                               "buffer_info[next_to_clean]\n"
3910                               "  time_stamp           <%lx>\n"
3911                               "  next_to_watch        <%x>\n"
3912                               "  jiffies              <%lx>\n"
3913                               "  next_to_watch.status <%x>\n",
3914                                 (unsigned long)(tx_ring - adapter->tx_ring),
3915                                 readl(hw->hw_addr + tx_ring->tdh),
3916                                 readl(hw->hw_addr + tx_ring->tdt),
3917                                 tx_ring->next_to_use,
3918                                 tx_ring->next_to_clean,
3919                                 tx_ring->buffer_info[eop].time_stamp,
3920                                 eop,
3921                                 jiffies,
3922                                 eop_desc->upper.fields.status);
3923                         e1000_dump(adapter);
3924                         netif_stop_queue(netdev);
3925                 }
3926         }
3927         adapter->total_tx_bytes += total_tx_bytes;
3928         adapter->total_tx_packets += total_tx_packets;
3929         netdev->stats.tx_bytes += total_tx_bytes;
3930         netdev->stats.tx_packets += total_tx_packets;
3931         return count < tx_ring->count;
3932 }
3933
3934 /**
3935  * e1000_rx_checksum - Receive Checksum Offload for 82543
3936  * @adapter:     board private structure
3937  * @status_err:  receive descriptor status and error fields
3938  * @csum:        receive descriptor csum field
3939  * @skb:         socket buffer with received data
3940  **/
3941 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3942                               u32 csum, struct sk_buff *skb)
3943 {
3944         struct e1000_hw *hw = &adapter->hw;
3945         u16 status = (u16)status_err;
3946         u8 errors = (u8)(status_err >> 24);
3947
3948         skb_checksum_none_assert(skb);
3949
3950         /* 82543 or newer only */
3951         if (unlikely(hw->mac_type < e1000_82543))
3952                 return;
3953         /* Ignore Checksum bit is set */
3954         if (unlikely(status & E1000_RXD_STAT_IXSM))
3955                 return;
3956         /* TCP/UDP checksum error bit is set */
3957         if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3958                 /* let the stack verify checksum errors */
3959                 adapter->hw_csum_err++;
3960                 return;
3961         }
3962         /* TCP/UDP Checksum has not been calculated */
3963         if (!(status & E1000_RXD_STAT_TCPCS))
3964                 return;
3965
3966         /* It must be a TCP or UDP packet with a valid checksum */
3967         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3968                 /* TCP checksum is good */
3969                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3970         }
3971         adapter->hw_csum_good++;
3972 }
3973
3974 /**
3975  * e1000_consume_page - helper function for jumbo Rx path
3976  * @bi: software descriptor shadow data
3977  * @skb: skb being modified
3978  * @length: length of data being added
3979  **/
3980 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
3981                                u16 length)
3982 {
3983         bi->rxbuf.page = NULL;
3984         skb->len += length;
3985         skb->data_len += length;
3986         skb->truesize += PAGE_SIZE;
3987 }
3988
3989 /**
3990  * e1000_receive_skb - helper function to handle rx indications
3991  * @adapter: board private structure
3992  * @status: descriptor status field as written by hardware
3993  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3994  * @skb: pointer to sk_buff to be indicated to stack
3995  */
3996 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
3997                               __le16 vlan, struct sk_buff *skb)
3998 {
3999         skb->protocol = eth_type_trans(skb, adapter->netdev);
4000
4001         if (status & E1000_RXD_STAT_VP) {
4002                 u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4003
4004                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4005         }
4006         napi_gro_receive(&adapter->napi, skb);
4007 }
4008
4009 /**
4010  * e1000_tbi_adjust_stats
4011  * @hw: Struct containing variables accessed by shared code
4012  * @stats: point to stats struct
4013  * @frame_len: The length of the frame in question
4014  * @mac_addr: The Ethernet destination address of the frame in question
4015  *
4016  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4017  */
4018 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4019                                    struct e1000_hw_stats *stats,
4020                                    u32 frame_len, const u8 *mac_addr)
4021 {
4022         u64 carry_bit;
4023
4024         /* First adjust the frame length. */
4025         frame_len--;
4026         /* We need to adjust the statistics counters, since the hardware
4027          * counters overcount this packet as a CRC error and undercount
4028          * the packet as a good packet
4029          */
4030         /* This packet should not be counted as a CRC error. */
4031         stats->crcerrs--;
4032         /* This packet does count as a Good Packet Received. */
4033         stats->gprc++;
4034
4035         /* Adjust the Good Octets received counters */
4036         carry_bit = 0x80000000 & stats->gorcl;
4037         stats->gorcl += frame_len;
4038         /* If the high bit of Gorcl (the low 32 bits of the Good Octets
4039          * Received Count) was one before the addition,
4040          * AND it is zero after, then we lost the carry out,
4041          * need to add one to Gorch (Good Octets Received Count High).
4042          * This could be simplified if all environments supported
4043          * 64-bit integers.
4044          */
4045         if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4046                 stats->gorch++;
4047         /* Is this a broadcast or multicast?  Check broadcast first,
4048          * since the test for a multicast frame will test positive on
4049          * a broadcast frame.
4050          */
4051         if (is_broadcast_ether_addr(mac_addr))
4052                 stats->bprc++;
4053         else if (is_multicast_ether_addr(mac_addr))
4054                 stats->mprc++;
4055
4056         if (frame_len == hw->max_frame_size) {
4057                 /* In this case, the hardware has overcounted the number of
4058                  * oversize frames.
4059                  */
4060                 if (stats->roc > 0)
4061                         stats->roc--;
4062         }
4063
4064         /* Adjust the bin counters when the extra byte put the frame in the
4065          * wrong bin. Remember that the frame_len was adjusted above.
4066          */
4067         if (frame_len == 64) {
4068                 stats->prc64++;
4069                 stats->prc127--;
4070         } else if (frame_len == 127) {
4071                 stats->prc127++;
4072                 stats->prc255--;
4073         } else if (frame_len == 255) {
4074                 stats->prc255++;
4075                 stats->prc511--;
4076         } else if (frame_len == 511) {
4077                 stats->prc511++;
4078                 stats->prc1023--;
4079         } else if (frame_len == 1023) {
4080                 stats->prc1023++;
4081                 stats->prc1522--;
4082         } else if (frame_len == 1522) {
4083                 stats->prc1522++;
4084         }
4085 }
4086
4087 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4088                                     u8 status, u8 errors,
4089                                     u32 length, const u8 *data)
4090 {
4091         struct e1000_hw *hw = &adapter->hw;
4092         u8 last_byte = *(data + length - 1);
4093
4094         if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4095                 unsigned long irq_flags;
4096
4097                 spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4098                 e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4099                 spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4100
4101                 return true;
4102         }
4103
4104         return false;
4105 }
4106
4107 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4108                                           unsigned int bufsz)
4109 {
4110         struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4111
4112         if (unlikely(!skb))
4113                 adapter->alloc_rx_buff_failed++;
4114         return skb;
4115 }
4116
4117 /**
4118  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4119  * @adapter: board private structure
4120  * @rx_ring: ring to clean
4121  * @work_done: amount of napi work completed this call
4122  * @work_to_do: max amount of work allowed for this call to do
4123  *
4124  * the return value indicates whether actual cleaning was done, there
4125  * is no guarantee that everything was cleaned
4126  */
4127 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4128                                      struct e1000_rx_ring *rx_ring,
4129                                      int *work_done, int work_to_do)
4130 {
4131         struct net_device *netdev = adapter->netdev;
4132         struct pci_dev *pdev = adapter->pdev;
4133         struct e1000_rx_desc *rx_desc, *next_rxd;
4134         struct e1000_rx_buffer *buffer_info, *next_buffer;
4135         u32 length;
4136         unsigned int i;
4137         int cleaned_count = 0;
4138         bool cleaned = false;
4139         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4140
4141         i = rx_ring->next_to_clean;
4142         rx_desc = E1000_RX_DESC(*rx_ring, i);
4143         buffer_info = &rx_ring->buffer_info[i];
4144
4145         while (rx_desc->status & E1000_RXD_STAT_DD) {
4146                 struct sk_buff *skb;
4147                 u8 status;
4148
4149                 if (*work_done >= work_to_do)
4150                         break;
4151                 (*work_done)++;
4152                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4153
4154                 status = rx_desc->status;
4155
4156                 if (++i == rx_ring->count)
4157                         i = 0;
4158
4159                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4160                 prefetch(next_rxd);
4161
4162                 next_buffer = &rx_ring->buffer_info[i];
4163
4164                 cleaned = true;
4165                 cleaned_count++;
4166                 dma_unmap_page(&pdev->dev, buffer_info->dma,
4167                                adapter->rx_buffer_len, DMA_FROM_DEVICE);
4168                 buffer_info->dma = 0;
4169
4170                 length = le16_to_cpu(rx_desc->length);
4171
4172                 /* errors is only valid for DD + EOP descriptors */
4173                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
4174                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4175                         u8 *mapped = page_address(buffer_info->rxbuf.page);
4176
4177                         if (e1000_tbi_should_accept(adapter, status,
4178                                                     rx_desc->errors,
4179                                                     length, mapped)) {
4180                                 length--;
4181                         } else if (netdev->features & NETIF_F_RXALL) {
4182                                 goto process_skb;
4183                         } else {
4184                                 /* an error means any chain goes out the window
4185                                  * too
4186                                  */
4187                                 dev_kfree_skb(rx_ring->rx_skb_top);
4188                                 rx_ring->rx_skb_top = NULL;
4189                                 goto next_desc;
4190                         }
4191                 }
4192
4193 #define rxtop rx_ring->rx_skb_top
4194 process_skb:
4195                 if (!(status & E1000_RXD_STAT_EOP)) {
4196                         /* this descriptor is only the beginning (or middle) */
4197                         if (!rxtop) {
4198                                 /* this is the beginning of a chain */
4199                                 rxtop = napi_get_frags(&adapter->napi);
4200                                 if (!rxtop)
4201                                         break;
4202
4203                                 skb_fill_page_desc(rxtop, 0,
4204                                                    buffer_info->rxbuf.page,
4205                                                    0, length);
4206                         } else {
4207                                 /* this is the middle of a chain */
4208                                 skb_fill_page_desc(rxtop,
4209                                     skb_shinfo(rxtop)->nr_frags,
4210                                     buffer_info->rxbuf.page, 0, length);
4211                         }
4212                         e1000_consume_page(buffer_info, rxtop, length);
4213                         goto next_desc;
4214                 } else {
4215                         if (rxtop) {
4216                                 /* end of the chain */
4217                                 skb_fill_page_desc(rxtop,
4218                                     skb_shinfo(rxtop)->nr_frags,
4219                                     buffer_info->rxbuf.page, 0, length);
4220                                 skb = rxtop;
4221                                 rxtop = NULL;
4222                                 e1000_consume_page(buffer_info, skb, length);
4223                         } else {
4224                                 struct page *p;
4225                                 /* no chain, got EOP, this buf is the packet
4226                                  * copybreak to save the put_page/alloc_page
4227                                  */
4228                                 p = buffer_info->rxbuf.page;
4229                                 if (length <= copybreak) {
4230                                         if (likely(!(netdev->features & NETIF_F_RXFCS)))
4231                                                 length -= 4;
4232                                         skb = e1000_alloc_rx_skb(adapter,
4233                                                                  length);
4234                                         if (!skb)
4235                                                 break;
4236
4237                                         memcpy(skb_tail_pointer(skb),
4238                                                page_address(p), length);
4239
4240                                         /* re-use the page, so don't erase
4241                                          * buffer_info->rxbuf.page
4242                                          */
4243                                         skb_put(skb, length);
4244                                         e1000_rx_checksum(adapter,
4245                                                           status | rx_desc->errors << 24,
4246                                                           le16_to_cpu(rx_desc->csum), skb);
4247
4248                                         total_rx_bytes += skb->len;
4249                                         total_rx_packets++;
4250
4251                                         e1000_receive_skb(adapter, status,
4252                                                           rx_desc->special, skb);
4253                                         goto next_desc;
4254                                 } else {
4255                                         skb = napi_get_frags(&adapter->napi);
4256                                         if (!skb) {
4257                                                 adapter->alloc_rx_buff_failed++;
4258                                                 break;
4259                                         }
4260                                         skb_fill_page_desc(skb, 0, p, 0,
4261                                                            length);
4262                                         e1000_consume_page(buffer_info, skb,
4263                                                            length);
4264                                 }
4265                         }
4266                 }
4267
4268                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
4269                 e1000_rx_checksum(adapter,
4270                                   (u32)(status) |
4271                                   ((u32)(rx_desc->errors) << 24),
4272                                   le16_to_cpu(rx_desc->csum), skb);
4273
4274                 total_rx_bytes += (skb->len - 4); /* don't count FCS */
4275                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4276                         pskb_trim(skb, skb->len - 4);
4277                 total_rx_packets++;
4278
4279                 if (status & E1000_RXD_STAT_VP) {
4280                         __le16 vlan = rx_desc->special;
4281                         u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4282
4283                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4284                 }
4285
4286                 napi_gro_frags(&adapter->napi);
4287
4288 next_desc:
4289                 rx_desc->status = 0;
4290
4291                 /* return some buffers to hardware, one at a time is too slow */
4292                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4293                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4294                         cleaned_count = 0;
4295                 }
4296
4297                 /* use prefetched values */
4298                 rx_desc = next_rxd;
4299                 buffer_info = next_buffer;
4300         }
4301         rx_ring->next_to_clean = i;
4302
4303         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4304         if (cleaned_count)
4305                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4306
4307         adapter->total_rx_packets += total_rx_packets;
4308         adapter->total_rx_bytes += total_rx_bytes;
4309         netdev->stats.rx_bytes += total_rx_bytes;
4310         netdev->stats.rx_packets += total_rx_packets;
4311         return cleaned;
4312 }
4313
4314 /* this should improve performance for small packets with large amounts
4315  * of reassembly being done in the stack
4316  */
4317 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4318                                        struct e1000_rx_buffer *buffer_info,
4319                                        u32 length, const void *data)
4320 {
4321         struct sk_buff *skb;
4322
4323         if (length > copybreak)
4324                 return NULL;
4325
4326         skb = e1000_alloc_rx_skb(adapter, length);
4327         if (!skb)
4328                 return NULL;
4329
4330         dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4331                                 length, DMA_FROM_DEVICE);
4332
4333         skb_put_data(skb, data, length);
4334
4335         return skb;
4336 }
4337
4338 /**
4339  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4340  * @adapter: board private structure
4341  * @rx_ring: ring to clean
4342  * @work_done: amount of napi work completed this call
4343  * @work_to_do: max amount of work allowed for this call to do
4344  */
4345 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4346                                struct e1000_rx_ring *rx_ring,
4347                                int *work_done, int work_to_do)
4348 {
4349         struct net_device *netdev = adapter->netdev;
4350         struct pci_dev *pdev = adapter->pdev;
4351         struct e1000_rx_desc *rx_desc, *next_rxd;
4352         struct e1000_rx_buffer *buffer_info, *next_buffer;
4353         u32 length;
4354         unsigned int i;
4355         int cleaned_count = 0;
4356         bool cleaned = false;
4357         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4358
4359         i = rx_ring->next_to_clean;
4360         rx_desc = E1000_RX_DESC(*rx_ring, i);
4361         buffer_info = &rx_ring->buffer_info[i];
4362
4363         while (rx_desc->status & E1000_RXD_STAT_DD) {
4364                 struct sk_buff *skb;
4365                 u8 *data;
4366                 u8 status;
4367
4368                 if (*work_done >= work_to_do)
4369                         break;
4370                 (*work_done)++;
4371                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4372
4373                 status = rx_desc->status;
4374                 length = le16_to_cpu(rx_desc->length);
4375
4376                 data = buffer_info->rxbuf.data;
4377                 prefetch(data);
4378                 skb = e1000_copybreak(adapter, buffer_info, length, data);
4379                 if (!skb) {
4380                         unsigned int frag_len = e1000_frag_len(adapter);
4381
4382                         skb = napi_build_skb(data - E1000_HEADROOM, frag_len);
4383                         if (!skb) {
4384                                 adapter->alloc_rx_buff_failed++;
4385                                 break;
4386                         }
4387
4388                         skb_reserve(skb, E1000_HEADROOM);
4389                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4390                                          adapter->rx_buffer_len,
4391                                          DMA_FROM_DEVICE);
4392                         buffer_info->dma = 0;
4393                         buffer_info->rxbuf.data = NULL;
4394                 }
4395
4396                 if (++i == rx_ring->count)
4397                         i = 0;
4398
4399                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4400                 prefetch(next_rxd);
4401
4402                 next_buffer = &rx_ring->buffer_info[i];
4403
4404                 cleaned = true;
4405                 cleaned_count++;
4406
4407                 /* !EOP means multiple descriptors were used to store a single
4408                  * packet, if thats the case we need to toss it.  In fact, we
4409                  * to toss every packet with the EOP bit clear and the next
4410                  * frame that _does_ have the EOP bit set, as it is by
4411                  * definition only a frame fragment
4412                  */
4413                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4414                         adapter->discarding = true;
4415
4416                 if (adapter->discarding) {
4417                         /* All receives must fit into a single buffer */
4418                         netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4419                         dev_kfree_skb(skb);
4420                         if (status & E1000_RXD_STAT_EOP)
4421                                 adapter->discarding = false;
4422                         goto next_desc;
4423                 }
4424
4425                 if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4426                         if (e1000_tbi_should_accept(adapter, status,
4427                                                     rx_desc->errors,
4428                                                     length, data)) {
4429                                 length--;
4430                         } else if (netdev->features & NETIF_F_RXALL) {
4431                                 goto process_skb;
4432                         } else {
4433                                 dev_kfree_skb(skb);
4434                                 goto next_desc;
4435                         }
4436                 }
4437
4438 process_skb:
4439                 total_rx_bytes += (length - 4); /* don't count FCS */
4440                 total_rx_packets++;
4441
4442                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4443                         /* adjust length to remove Ethernet CRC, this must be
4444                          * done after the TBI_ACCEPT workaround above
4445                          */
4446                         length -= 4;
4447
4448                 if (buffer_info->rxbuf.data == NULL)
4449                         skb_put(skb, length);
4450                 else /* copybreak skb */
4451                         skb_trim(skb, length);
4452
4453                 /* Receive Checksum Offload */
4454                 e1000_rx_checksum(adapter,
4455                                   (u32)(status) |
4456                                   ((u32)(rx_desc->errors) << 24),
4457                                   le16_to_cpu(rx_desc->csum), skb);
4458
4459                 e1000_receive_skb(adapter, status, rx_desc->special, skb);
4460
4461 next_desc:
4462                 rx_desc->status = 0;
4463
4464                 /* return some buffers to hardware, one at a time is too slow */
4465                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4466                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4467                         cleaned_count = 0;
4468                 }
4469
4470                 /* use prefetched values */
4471                 rx_desc = next_rxd;
4472                 buffer_info = next_buffer;
4473         }
4474         rx_ring->next_to_clean = i;
4475
4476         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4477         if (cleaned_count)
4478                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4479
4480         adapter->total_rx_packets += total_rx_packets;
4481         adapter->total_rx_bytes += total_rx_bytes;
4482         netdev->stats.rx_bytes += total_rx_bytes;
4483         netdev->stats.rx_packets += total_rx_packets;
4484         return cleaned;
4485 }
4486
4487 /**
4488  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4489  * @adapter: address of board private structure
4490  * @rx_ring: pointer to receive ring structure
4491  * @cleaned_count: number of buffers to allocate this pass
4492  **/
4493 static void
4494 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4495                              struct e1000_rx_ring *rx_ring, int cleaned_count)
4496 {
4497         struct pci_dev *pdev = adapter->pdev;
4498         struct e1000_rx_desc *rx_desc;
4499         struct e1000_rx_buffer *buffer_info;
4500         unsigned int i;
4501
4502         i = rx_ring->next_to_use;
4503         buffer_info = &rx_ring->buffer_info[i];
4504
4505         while (cleaned_count--) {
4506                 /* allocate a new page if necessary */
4507                 if (!buffer_info->rxbuf.page) {
4508                         buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4509                         if (unlikely(!buffer_info->rxbuf.page)) {
4510                                 adapter->alloc_rx_buff_failed++;
4511                                 break;
4512                         }
4513                 }
4514
4515                 if (!buffer_info->dma) {
4516                         buffer_info->dma = dma_map_page(&pdev->dev,
4517                                                         buffer_info->rxbuf.page, 0,
4518                                                         adapter->rx_buffer_len,
4519                                                         DMA_FROM_DEVICE);
4520                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4521                                 put_page(buffer_info->rxbuf.page);
4522                                 buffer_info->rxbuf.page = NULL;
4523                                 buffer_info->dma = 0;
4524                                 adapter->alloc_rx_buff_failed++;
4525                                 break;
4526                         }
4527                 }
4528
4529                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4530                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4531
4532                 if (unlikely(++i == rx_ring->count))
4533                         i = 0;
4534                 buffer_info = &rx_ring->buffer_info[i];
4535         }
4536
4537         if (likely(rx_ring->next_to_use != i)) {
4538                 rx_ring->next_to_use = i;
4539                 if (unlikely(i-- == 0))
4540                         i = (rx_ring->count - 1);
4541
4542                 /* Force memory writes to complete before letting h/w
4543                  * know there are new descriptors to fetch.  (Only
4544                  * applicable for weak-ordered memory model archs,
4545                  * such as IA-64).
4546                  */
4547                 dma_wmb();
4548                 writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4549         }
4550 }
4551
4552 /**
4553  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4554  * @adapter: address of board private structure
4555  * @rx_ring: pointer to ring struct
4556  * @cleaned_count: number of new Rx buffers to try to allocate
4557  **/
4558 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4559                                    struct e1000_rx_ring *rx_ring,
4560                                    int cleaned_count)
4561 {
4562         struct e1000_hw *hw = &adapter->hw;
4563         struct pci_dev *pdev = adapter->pdev;
4564         struct e1000_rx_desc *rx_desc;
4565         struct e1000_rx_buffer *buffer_info;
4566         unsigned int i;
4567         unsigned int bufsz = adapter->rx_buffer_len;
4568
4569         i = rx_ring->next_to_use;
4570         buffer_info = &rx_ring->buffer_info[i];
4571
4572         while (cleaned_count--) {
4573                 void *data;
4574
4575                 if (buffer_info->rxbuf.data)
4576                         goto skip;
4577
4578                 data = e1000_alloc_frag(adapter);
4579                 if (!data) {
4580                         /* Better luck next round */
4581                         adapter->alloc_rx_buff_failed++;
4582                         break;
4583                 }
4584
4585                 /* Fix for errata 23, can't cross 64kB boundary */
4586                 if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4587                         void *olddata = data;
4588                         e_err(rx_err, "skb align check failed: %u bytes at "
4589                               "%p\n", bufsz, data);
4590                         /* Try again, without freeing the previous */
4591                         data = e1000_alloc_frag(adapter);
4592                         /* Failed allocation, critical failure */
4593                         if (!data) {
4594                                 skb_free_frag(olddata);
4595                                 adapter->alloc_rx_buff_failed++;
4596                                 break;
4597                         }
4598
4599                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4600                                 /* give up */
4601                                 skb_free_frag(data);
4602                                 skb_free_frag(olddata);
4603                                 adapter->alloc_rx_buff_failed++;
4604                                 break;
4605                         }
4606
4607                         /* Use new allocation */
4608                         skb_free_frag(olddata);
4609                 }
4610                 buffer_info->dma = dma_map_single(&pdev->dev,
4611                                                   data,
4612                                                   adapter->rx_buffer_len,
4613                                                   DMA_FROM_DEVICE);
4614                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4615                         skb_free_frag(data);
4616                         buffer_info->dma = 0;
4617                         adapter->alloc_rx_buff_failed++;
4618                         break;
4619                 }
4620
4621                 /* XXX if it was allocated cleanly it will never map to a
4622                  * boundary crossing
4623                  */
4624
4625                 /* Fix for errata 23, can't cross 64kB boundary */
4626                 if (!e1000_check_64k_bound(adapter,
4627                                         (void *)(unsigned long)buffer_info->dma,
4628                                         adapter->rx_buffer_len)) {
4629                         e_err(rx_err, "dma align check failed: %u bytes at "
4630                               "%p\n", adapter->rx_buffer_len,
4631                               (void *)(unsigned long)buffer_info->dma);
4632
4633                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4634                                          adapter->rx_buffer_len,
4635                                          DMA_FROM_DEVICE);
4636
4637                         skb_free_frag(data);
4638                         buffer_info->rxbuf.data = NULL;
4639                         buffer_info->dma = 0;
4640
4641                         adapter->alloc_rx_buff_failed++;
4642                         break;
4643                 }
4644                 buffer_info->rxbuf.data = data;
4645  skip:
4646                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4647                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4648
4649                 if (unlikely(++i == rx_ring->count))
4650                         i = 0;
4651                 buffer_info = &rx_ring->buffer_info[i];
4652         }
4653
4654         if (likely(rx_ring->next_to_use != i)) {
4655                 rx_ring->next_to_use = i;
4656                 if (unlikely(i-- == 0))
4657                         i = (rx_ring->count - 1);
4658
4659                 /* Force memory writes to complete before letting h/w
4660                  * know there are new descriptors to fetch.  (Only
4661                  * applicable for weak-ordered memory model archs,
4662                  * such as IA-64).
4663                  */
4664                 dma_wmb();
4665                 writel(i, hw->hw_addr + rx_ring->rdt);
4666         }
4667 }
4668
4669 /**
4670  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4671  * @adapter: address of board private structure
4672  **/
4673 static void e1000_smartspeed(struct e1000_adapter *adapter)
4674 {
4675         struct e1000_hw *hw = &adapter->hw;
4676         u16 phy_status;
4677         u16 phy_ctrl;
4678
4679         if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4680            !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4681                 return;
4682
4683         if (adapter->smartspeed == 0) {
4684                 /* If Master/Slave config fault is asserted twice,
4685                  * we assume back-to-back
4686                  */
4687                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4688                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4689                         return;
4690                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4691                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4692                         return;
4693                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4694                 if (phy_ctrl & CR_1000T_MS_ENABLE) {
4695                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
4696                         e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4697                                             phy_ctrl);
4698                         adapter->smartspeed++;
4699                         if (!e1000_phy_setup_autoneg(hw) &&
4700                            !e1000_read_phy_reg(hw, PHY_CTRL,
4701                                                &phy_ctrl)) {
4702                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4703                                              MII_CR_RESTART_AUTO_NEG);
4704                                 e1000_write_phy_reg(hw, PHY_CTRL,
4705                                                     phy_ctrl);
4706                         }
4707                 }
4708                 return;
4709         } else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4710                 /* If still no link, perhaps using 2/3 pair cable */
4711                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4712                 phy_ctrl |= CR_1000T_MS_ENABLE;
4713                 e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4714                 if (!e1000_phy_setup_autoneg(hw) &&
4715                    !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4716                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4717                                      MII_CR_RESTART_AUTO_NEG);
4718                         e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4719                 }
4720         }
4721         /* Restart process after E1000_SMARTSPEED_MAX iterations */
4722         if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4723                 adapter->smartspeed = 0;
4724 }
4725
4726 /**
4727  * e1000_ioctl - handle ioctl calls
4728  * @netdev: pointer to our netdev
4729  * @ifr: pointer to interface request structure
4730  * @cmd: ioctl data
4731  **/
4732 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4733 {
4734         switch (cmd) {
4735         case SIOCGMIIPHY:
4736         case SIOCGMIIREG:
4737         case SIOCSMIIREG:
4738                 return e1000_mii_ioctl(netdev, ifr, cmd);
4739         default:
4740                 return -EOPNOTSUPP;
4741         }
4742 }
4743
4744 /**
4745  * e1000_mii_ioctl -
4746  * @netdev: pointer to our netdev
4747  * @ifr: pointer to interface request structure
4748  * @cmd: ioctl data
4749  **/
4750 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4751                            int cmd)
4752 {
4753         struct e1000_adapter *adapter = netdev_priv(netdev);
4754         struct e1000_hw *hw = &adapter->hw;
4755         struct mii_ioctl_data *data = if_mii(ifr);
4756         int retval;
4757         u16 mii_reg;
4758         unsigned long flags;
4759
4760         if (hw->media_type != e1000_media_type_copper)
4761                 return -EOPNOTSUPP;
4762
4763         switch (cmd) {
4764         case SIOCGMIIPHY:
4765                 data->phy_id = hw->phy_addr;
4766                 break;
4767         case SIOCGMIIREG:
4768                 spin_lock_irqsave(&adapter->stats_lock, flags);
4769                 if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4770                                    &data->val_out)) {
4771                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4772                         return -EIO;
4773                 }
4774                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4775                 break;
4776         case SIOCSMIIREG:
4777                 if (data->reg_num & ~(0x1F))
4778                         return -EFAULT;
4779                 mii_reg = data->val_in;
4780                 spin_lock_irqsave(&adapter->stats_lock, flags);
4781                 if (e1000_write_phy_reg(hw, data->reg_num,
4782                                         mii_reg)) {
4783                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4784                         return -EIO;
4785                 }
4786                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4787                 if (hw->media_type == e1000_media_type_copper) {
4788                         switch (data->reg_num) {
4789                         case PHY_CTRL:
4790                                 if (mii_reg & MII_CR_POWER_DOWN)
4791                                         break;
4792                                 if (mii_reg & MII_CR_AUTO_NEG_EN) {
4793                                         hw->autoneg = 1;
4794                                         hw->autoneg_advertised = 0x2F;
4795                                 } else {
4796                                         u32 speed;
4797                                         if (mii_reg & 0x40)
4798                                                 speed = SPEED_1000;
4799                                         else if (mii_reg & 0x2000)
4800                                                 speed = SPEED_100;
4801                                         else
4802                                                 speed = SPEED_10;
4803                                         retval = e1000_set_spd_dplx(
4804                                                 adapter, speed,
4805                                                 ((mii_reg & 0x100)
4806                                                  ? DUPLEX_FULL :
4807                                                  DUPLEX_HALF));
4808                                         if (retval)
4809                                                 return retval;
4810                                 }
4811                                 if (netif_running(adapter->netdev))
4812                                         e1000_reinit_locked(adapter);
4813                                 else
4814                                         e1000_reset(adapter);
4815                                 break;
4816                         case M88E1000_PHY_SPEC_CTRL:
4817                         case M88E1000_EXT_PHY_SPEC_CTRL:
4818                                 if (e1000_phy_reset(hw))
4819                                         return -EIO;
4820                                 break;
4821                         }
4822                 } else {
4823                         switch (data->reg_num) {
4824                         case PHY_CTRL:
4825                                 if (mii_reg & MII_CR_POWER_DOWN)
4826                                         break;
4827                                 if (netif_running(adapter->netdev))
4828                                         e1000_reinit_locked(adapter);
4829                                 else
4830                                         e1000_reset(adapter);
4831                                 break;
4832                         }
4833                 }
4834                 break;
4835         default:
4836                 return -EOPNOTSUPP;
4837         }
4838         return E1000_SUCCESS;
4839 }
4840
4841 void e1000_pci_set_mwi(struct e1000_hw *hw)
4842 {
4843         struct e1000_adapter *adapter = hw->back;
4844         int ret_val = pci_set_mwi(adapter->pdev);
4845
4846         if (ret_val)
4847                 e_err(probe, "Error in setting MWI\n");
4848 }
4849
4850 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4851 {
4852         struct e1000_adapter *adapter = hw->back;
4853
4854         pci_clear_mwi(adapter->pdev);
4855 }
4856
4857 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4858 {
4859         struct e1000_adapter *adapter = hw->back;
4860         return pcix_get_mmrbc(adapter->pdev);
4861 }
4862
4863 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4864 {
4865         struct e1000_adapter *adapter = hw->back;
4866         pcix_set_mmrbc(adapter->pdev, mmrbc);
4867 }
4868
4869 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4870 {
4871         outl(value, port);
4872 }
4873
4874 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4875 {
4876         u16 vid;
4877
4878         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4879                 return true;
4880         return false;
4881 }
4882
4883 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4884                               netdev_features_t features)
4885 {
4886         struct e1000_hw *hw = &adapter->hw;
4887         u32 ctrl;
4888
4889         ctrl = er32(CTRL);
4890         if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4891                 /* enable VLAN tag insert/strip */
4892                 ctrl |= E1000_CTRL_VME;
4893         } else {
4894                 /* disable VLAN tag insert/strip */
4895                 ctrl &= ~E1000_CTRL_VME;
4896         }
4897         ew32(CTRL, ctrl);
4898 }
4899 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4900                                      bool filter_on)
4901 {
4902         struct e1000_hw *hw = &adapter->hw;
4903         u32 rctl;
4904
4905         if (!test_bit(__E1000_DOWN, &adapter->flags))
4906                 e1000_irq_disable(adapter);
4907
4908         __e1000_vlan_mode(adapter, adapter->netdev->features);
4909         if (filter_on) {
4910                 /* enable VLAN receive filtering */
4911                 rctl = er32(RCTL);
4912                 rctl &= ~E1000_RCTL_CFIEN;
4913                 if (!(adapter->netdev->flags & IFF_PROMISC))
4914                         rctl |= E1000_RCTL_VFE;
4915                 ew32(RCTL, rctl);
4916                 e1000_update_mng_vlan(adapter);
4917         } else {
4918                 /* disable VLAN receive filtering */
4919                 rctl = er32(RCTL);
4920                 rctl &= ~E1000_RCTL_VFE;
4921                 ew32(RCTL, rctl);
4922         }
4923
4924         if (!test_bit(__E1000_DOWN, &adapter->flags))
4925                 e1000_irq_enable(adapter);
4926 }
4927
4928 static void e1000_vlan_mode(struct net_device *netdev,
4929                             netdev_features_t features)
4930 {
4931         struct e1000_adapter *adapter = netdev_priv(netdev);
4932
4933         if (!test_bit(__E1000_DOWN, &adapter->flags))
4934                 e1000_irq_disable(adapter);
4935
4936         __e1000_vlan_mode(adapter, features);
4937
4938         if (!test_bit(__E1000_DOWN, &adapter->flags))
4939                 e1000_irq_enable(adapter);
4940 }
4941
4942 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4943                                  __be16 proto, u16 vid)
4944 {
4945         struct e1000_adapter *adapter = netdev_priv(netdev);
4946         struct e1000_hw *hw = &adapter->hw;
4947         u32 vfta, index;
4948
4949         if ((hw->mng_cookie.status &
4950              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4951             (vid == adapter->mng_vlan_id))
4952                 return 0;
4953
4954         if (!e1000_vlan_used(adapter))
4955                 e1000_vlan_filter_on_off(adapter, true);
4956
4957         /* add VID to filter table */
4958         index = (vid >> 5) & 0x7F;
4959         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4960         vfta |= (1 << (vid & 0x1F));
4961         e1000_write_vfta(hw, index, vfta);
4962
4963         set_bit(vid, adapter->active_vlans);
4964
4965         return 0;
4966 }
4967
4968 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4969                                   __be16 proto, u16 vid)
4970 {
4971         struct e1000_adapter *adapter = netdev_priv(netdev);
4972         struct e1000_hw *hw = &adapter->hw;
4973         u32 vfta, index;
4974
4975         if (!test_bit(__E1000_DOWN, &adapter->flags))
4976                 e1000_irq_disable(adapter);
4977         if (!test_bit(__E1000_DOWN, &adapter->flags))
4978                 e1000_irq_enable(adapter);
4979
4980         /* remove VID from filter table */
4981         index = (vid >> 5) & 0x7F;
4982         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4983         vfta &= ~(1 << (vid & 0x1F));
4984         e1000_write_vfta(hw, index, vfta);
4985
4986         clear_bit(vid, adapter->active_vlans);
4987
4988         if (!e1000_vlan_used(adapter))
4989                 e1000_vlan_filter_on_off(adapter, false);
4990
4991         return 0;
4992 }
4993
4994 static void e1000_restore_vlan(struct e1000_adapter *adapter)
4995 {
4996         u16 vid;
4997
4998         if (!e1000_vlan_used(adapter))
4999                 return;
5000
5001         e1000_vlan_filter_on_off(adapter, true);
5002         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
5003                 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
5004 }
5005
5006 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
5007 {
5008         struct e1000_hw *hw = &adapter->hw;
5009
5010         hw->autoneg = 0;
5011
5012         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5013          * for the switch() below to work
5014          */
5015         if ((spd & 1) || (dplx & ~1))
5016                 goto err_inval;
5017
5018         /* Fiber NICs only allow 1000 gbps Full duplex */
5019         if ((hw->media_type == e1000_media_type_fiber) &&
5020             spd != SPEED_1000 &&
5021             dplx != DUPLEX_FULL)
5022                 goto err_inval;
5023
5024         switch (spd + dplx) {
5025         case SPEED_10 + DUPLEX_HALF:
5026                 hw->forced_speed_duplex = e1000_10_half;
5027                 break;
5028         case SPEED_10 + DUPLEX_FULL:
5029                 hw->forced_speed_duplex = e1000_10_full;
5030                 break;
5031         case SPEED_100 + DUPLEX_HALF:
5032                 hw->forced_speed_duplex = e1000_100_half;
5033                 break;
5034         case SPEED_100 + DUPLEX_FULL:
5035                 hw->forced_speed_duplex = e1000_100_full;
5036                 break;
5037         case SPEED_1000 + DUPLEX_FULL:
5038                 hw->autoneg = 1;
5039                 hw->autoneg_advertised = ADVERTISE_1000_FULL;
5040                 break;
5041         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5042         default:
5043                 goto err_inval;
5044         }
5045
5046         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5047         hw->mdix = AUTO_ALL_MODES;
5048
5049         return 0;
5050
5051 err_inval:
5052         e_err(probe, "Unsupported Speed/Duplex configuration\n");
5053         return -EINVAL;
5054 }
5055
5056 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5057 {
5058         struct net_device *netdev = pci_get_drvdata(pdev);
5059         struct e1000_adapter *adapter = netdev_priv(netdev);
5060         struct e1000_hw *hw = &adapter->hw;
5061         u32 ctrl, ctrl_ext, rctl, status;
5062         u32 wufc = adapter->wol;
5063
5064         netif_device_detach(netdev);
5065
5066         if (netif_running(netdev)) {
5067                 int count = E1000_CHECK_RESET_COUNT;
5068
5069                 while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5070                         usleep_range(10000, 20000);
5071
5072                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5073                 e1000_down(adapter);
5074         }
5075
5076         status = er32(STATUS);
5077         if (status & E1000_STATUS_LU)
5078                 wufc &= ~E1000_WUFC_LNKC;
5079
5080         if (wufc) {
5081                 e1000_setup_rctl(adapter);
5082                 e1000_set_rx_mode(netdev);
5083
5084                 rctl = er32(RCTL);
5085
5086                 /* turn on all-multi mode if wake on multicast is enabled */
5087                 if (wufc & E1000_WUFC_MC)
5088                         rctl |= E1000_RCTL_MPE;
5089
5090                 /* enable receives in the hardware */
5091                 ew32(RCTL, rctl | E1000_RCTL_EN);
5092
5093                 if (hw->mac_type >= e1000_82540) {
5094                         ctrl = er32(CTRL);
5095                         /* advertise wake from D3Cold */
5096                         #define E1000_CTRL_ADVD3WUC 0x00100000
5097                         /* phy power management enable */
5098                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5099                         ctrl |= E1000_CTRL_ADVD3WUC |
5100                                 E1000_CTRL_EN_PHY_PWR_MGMT;
5101                         ew32(CTRL, ctrl);
5102                 }
5103
5104                 if (hw->media_type == e1000_media_type_fiber ||
5105                     hw->media_type == e1000_media_type_internal_serdes) {
5106                         /* keep the laser running in D3 */
5107                         ctrl_ext = er32(CTRL_EXT);
5108                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5109                         ew32(CTRL_EXT, ctrl_ext);
5110                 }
5111
5112                 ew32(WUC, E1000_WUC_PME_EN);
5113                 ew32(WUFC, wufc);
5114         } else {
5115                 ew32(WUC, 0);
5116                 ew32(WUFC, 0);
5117         }
5118
5119         e1000_release_manageability(adapter);
5120
5121         *enable_wake = !!wufc;
5122
5123         /* make sure adapter isn't asleep if manageability is enabled */
5124         if (adapter->en_mng_pt)
5125                 *enable_wake = true;
5126
5127         if (netif_running(netdev))
5128                 e1000_free_irq(adapter);
5129
5130         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5131                 pci_disable_device(pdev);
5132
5133         return 0;
5134 }
5135
5136 static int e1000_suspend(struct device *dev)
5137 {
5138         int retval;
5139         struct pci_dev *pdev = to_pci_dev(dev);
5140         bool wake;
5141
5142         retval = __e1000_shutdown(pdev, &wake);
5143         device_set_wakeup_enable(dev, wake);
5144
5145         return retval;
5146 }
5147
5148 static int e1000_resume(struct device *dev)
5149 {
5150         struct pci_dev *pdev = to_pci_dev(dev);
5151         struct net_device *netdev = pci_get_drvdata(pdev);
5152         struct e1000_adapter *adapter = netdev_priv(netdev);
5153         struct e1000_hw *hw = &adapter->hw;
5154         u32 err;
5155
5156         if (adapter->need_ioport)
5157                 err = pci_enable_device(pdev);
5158         else
5159                 err = pci_enable_device_mem(pdev);
5160         if (err) {
5161                 pr_err("Cannot enable PCI device from suspend\n");
5162                 return err;
5163         }
5164
5165         /* flush memory to make sure state is correct */
5166         smp_mb__before_atomic();
5167         clear_bit(__E1000_DISABLED, &adapter->flags);
5168         pci_set_master(pdev);
5169
5170         pci_enable_wake(pdev, PCI_D3hot, 0);
5171         pci_enable_wake(pdev, PCI_D3cold, 0);
5172
5173         if (netif_running(netdev)) {
5174                 err = e1000_request_irq(adapter);
5175                 if (err)
5176                         return err;
5177         }
5178
5179         e1000_power_up_phy(adapter);
5180         e1000_reset(adapter);
5181         ew32(WUS, ~0);
5182
5183         e1000_init_manageability(adapter);
5184
5185         if (netif_running(netdev))
5186                 e1000_up(adapter);
5187
5188         netif_device_attach(netdev);
5189
5190         return 0;
5191 }
5192
5193 static void e1000_shutdown(struct pci_dev *pdev)
5194 {
5195         bool wake;
5196
5197         __e1000_shutdown(pdev, &wake);
5198
5199         if (system_state == SYSTEM_POWER_OFF) {
5200                 pci_wake_from_d3(pdev, wake);
5201                 pci_set_power_state(pdev, PCI_D3hot);
5202         }
5203 }
5204
5205 #ifdef CONFIG_NET_POLL_CONTROLLER
5206 /* Polling 'interrupt' - used by things like netconsole to send skbs
5207  * without having to re-enable interrupts. It's not called while
5208  * the interrupt routine is executing.
5209  */
5210 static void e1000_netpoll(struct net_device *netdev)
5211 {
5212         struct e1000_adapter *adapter = netdev_priv(netdev);
5213
5214         if (disable_hardirq(adapter->pdev->irq))
5215                 e1000_intr(adapter->pdev->irq, netdev);
5216         enable_irq(adapter->pdev->irq);
5217 }
5218 #endif
5219
5220 /**
5221  * e1000_io_error_detected - called when PCI error is detected
5222  * @pdev: Pointer to PCI device
5223  * @state: The current pci connection state
5224  *
5225  * This function is called after a PCI bus error affecting
5226  * this device has been detected.
5227  */
5228 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5229                                                 pci_channel_state_t state)
5230 {
5231         struct net_device *netdev = pci_get_drvdata(pdev);
5232         struct e1000_adapter *adapter = netdev_priv(netdev);
5233
5234         netif_device_detach(netdev);
5235
5236         if (state == pci_channel_io_perm_failure)
5237                 return PCI_ERS_RESULT_DISCONNECT;
5238
5239         if (netif_running(netdev))
5240                 e1000_down(adapter);
5241
5242         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5243                 pci_disable_device(pdev);
5244
5245         /* Request a slot reset. */
5246         return PCI_ERS_RESULT_NEED_RESET;
5247 }
5248
5249 /**
5250  * e1000_io_slot_reset - called after the pci bus has been reset.
5251  * @pdev: Pointer to PCI device
5252  *
5253  * Restart the card from scratch, as if from a cold-boot. Implementation
5254  * resembles the first-half of the e1000_resume routine.
5255  */
5256 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5257 {
5258         struct net_device *netdev = pci_get_drvdata(pdev);
5259         struct e1000_adapter *adapter = netdev_priv(netdev);
5260         struct e1000_hw *hw = &adapter->hw;
5261         int err;
5262
5263         if (adapter->need_ioport)
5264                 err = pci_enable_device(pdev);
5265         else
5266                 err = pci_enable_device_mem(pdev);
5267         if (err) {
5268                 pr_err("Cannot re-enable PCI device after reset.\n");
5269                 return PCI_ERS_RESULT_DISCONNECT;
5270         }
5271
5272         /* flush memory to make sure state is correct */
5273         smp_mb__before_atomic();
5274         clear_bit(__E1000_DISABLED, &adapter->flags);
5275         pci_set_master(pdev);
5276
5277         pci_enable_wake(pdev, PCI_D3hot, 0);
5278         pci_enable_wake(pdev, PCI_D3cold, 0);
5279
5280         e1000_reset(adapter);
5281         ew32(WUS, ~0);
5282
5283         return PCI_ERS_RESULT_RECOVERED;
5284 }
5285
5286 /**
5287  * e1000_io_resume - called when traffic can start flowing again.
5288  * @pdev: Pointer to PCI device
5289  *
5290  * This callback is called when the error recovery driver tells us that
5291  * its OK to resume normal operation. Implementation resembles the
5292  * second-half of the e1000_resume routine.
5293  */
5294 static void e1000_io_resume(struct pci_dev *pdev)
5295 {
5296         struct net_device *netdev = pci_get_drvdata(pdev);
5297         struct e1000_adapter *adapter = netdev_priv(netdev);
5298
5299         e1000_init_manageability(adapter);
5300
5301         if (netif_running(netdev)) {
5302                 if (e1000_up(adapter)) {
5303                         pr_info("can't bring device back up after reset\n");
5304                         return;
5305                 }
5306         }
5307
5308         netif_device_attach(netdev);
5309 }
5310
5311 /* e1000_main.c */