projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbevf: fix null pointer dereference due to filter being set for VLAN 0
[linux-block.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
86 {0, }
87};
88
89MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
90
91void igb_reset(struct igb_adapter *);
92static int igb_setup_all_tx_resources(struct igb_adapter *);
93static int igb_setup_all_rx_resources(struct igb_adapter *);
94static void igb_free_all_tx_resources(struct igb_adapter *);
95static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 96static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 97void igb_update_stats(struct igb_adapter *);
98static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99static void __devexit igb_remove(struct pci_dev *pdev);
100static int igb_sw_init(struct igb_adapter *);
101static int igb_open(struct net_device *);
102static int igb_close(struct net_device *);
103static void igb_configure_tx(struct igb_adapter *);
104static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 105static void igb_clean_all_tx_rings(struct igb_adapter *);
106static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 107static void igb_clean_tx_ring(struct igb_ring *);
108static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 109static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 110static void igb_update_phy_info(unsigned long);
111static void igb_watchdog(unsigned long);
112static void igb_watchdog_task(struct work_struct *);
b1a436c3 113static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 114static struct net_device_stats *igb_get_stats(struct net_device *);
115static int igb_change_mtu(struct net_device *, int);
116static int igb_set_mac(struct net_device *, void *);
68d480c4 117static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 118static irqreturn_t igb_intr(int irq, void *);
119static irqreturn_t igb_intr_msi(int irq, void *);
120static irqreturn_t igb_msix_other(int irq, void *);
047e0030 121static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 122#ifdef CONFIG_IGB_DCA
047e0030 123static void igb_update_dca(struct igb_q_vector *);
fe4506b6 124static void igb_setup_dca(struct igb_adapter *);
421e02f0 125#endif /* CONFIG_IGB_DCA */
047e0030 126static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 127static int igb_poll(struct napi_struct *, int);
047e0030 128static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 129static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130static void igb_tx_timeout(struct net_device *);
131static void igb_reset_task(struct work_struct *);
132static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133static void igb_vlan_rx_add_vid(struct net_device *, u16);
134static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135static void igb_restore_vlan(struct igb_adapter *);
26ad9178 136static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 137static void igb_ping_all_vfs(struct igb_adapter *);
138static void igb_msg_task(struct igb_adapter *);
4ae196df 139static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 140static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 141static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 142static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
9d5c8243 148
9d5c8243 149#ifdef CONFIG_PM
3fe7c4c9 150static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 151static int igb_resume(struct pci_dev *);
152#endif
153static void igb_shutdown(struct pci_dev *);
421e02f0 154#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 155static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
158 .next = NULL,
159 .priority = 0
160};
161#endif
9d5c8243
AK		 162#ifdef CONFIG_NET_POLL_CONTROLLER
163/* for netdump / net console */
164static void igb_netpoll(struct net_device *);
165#endif
37680117 166#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 167static unsigned int max_vfs = 0;
168module_param(max_vfs, uint, 0);
169MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171#endif /* CONFIG_PCI_IOV */
172
9d5c8243
AK		 173static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176static void igb_io_resume(struct pci_dev *);
177
178static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
182};
183
184
185static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
188 .probe = igb_probe,
189 .remove = __devexit_p(igb_remove),
190#ifdef CONFIG_PM
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
194#endif
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
197};
198
199MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201MODULE_LICENSE("GPL");
202MODULE_VERSION(DRV_VERSION);
203
c97ec42a
TI		 204struct igb_reg_info {
205 u32 ofs;
206 char *name;
207};
208
209static const struct igb_reg_info igb_reg_info_tbl[] = {
210
211 /* General Registers */
212 {E1000_CTRL, "CTRL"},
213 {E1000_STATUS, "STATUS"},
214 {E1000_CTRL_EXT, "CTRL_EXT"},
215
216 /* Interrupt Registers */
217 {E1000_ICR, "ICR"},
218
219 /* RX Registers */
220 {E1000_RCTL, "RCTL"},
221 {E1000_RDLEN(0), "RDLEN"},
222 {E1000_RDH(0), "RDH"},
223 {E1000_RDT(0), "RDT"},
224 {E1000_RXDCTL(0), "RXDCTL"},
225 {E1000_RDBAL(0), "RDBAL"},
226 {E1000_RDBAH(0), "RDBAH"},
227
228 /* TX Registers */
229 {E1000_TCTL, "TCTL"},
230 {E1000_TDBAL(0), "TDBAL"},
231 {E1000_TDBAH(0), "TDBAH"},
232 {E1000_TDLEN(0), "TDLEN"},
233 {E1000_TDH(0), "TDH"},
234 {E1000_TDT(0), "TDT"},
235 {E1000_TXDCTL(0), "TXDCTL"},
236 {E1000_TDFH, "TDFH"},
237 {E1000_TDFT, "TDFT"},
238 {E1000_TDFHS, "TDFHS"},
239 {E1000_TDFPC, "TDFPC"},
240
241 /* List Terminator */
242 {}
243};
244
245/*
246 * igb_regdump - register printout routine
247 */
248static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
249{
250 int n = 0;
251 char rname[16];
252 u32 regs[8];
253
254 switch (reginfo->ofs) {
255 case E1000_RDLEN(0):
256 for (n = 0; n < 4; n++)
257 regs[n] = rd32(E1000_RDLEN(n));
258 break;
259 case E1000_RDH(0):
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDH(n));
262 break;
263 case E1000_RDT(0):
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDT(n));
266 break;
267 case E1000_RXDCTL(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RXDCTL(n));
270 break;
271 case E1000_RDBAL(0):
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RDBAL(n));
274 break;
275 case E1000_RDBAH(0):
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAH(n));
278 break;
279 case E1000_TDBAL(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAL(n));
282 break;
283 case E1000_TDBAH(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_TDBAH(n));
286 break;
287 case E1000_TDLEN(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDLEN(n));
290 break;
291 case E1000_TDH(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDH(n));
294 break;
295 case E1000_TDT(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDT(n));
298 break;
299 case E1000_TXDCTL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TXDCTL(n));
302 break;
303 default:
304 printk(KERN_INFO "%-15s %08x\n",
305 reginfo->name, rd32(reginfo->ofs));
306 return;
307 }
308
309 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
310 printk(KERN_INFO "%-15s ", rname);
311 for (n = 0; n < 4; n++)
312 printk(KERN_CONT "%08x ", regs[n]);
313 printk(KERN_CONT "\n");
314}
315
316/*
317 * igb_dump - Print registers, tx-rings and rx-rings
318 */
319static void igb_dump(struct igb_adapter *adapter)
320{
321 struct net_device *netdev = adapter->netdev;
322 struct e1000_hw *hw = &adapter->hw;
323 struct igb_reg_info *reginfo;
324 int n = 0;
325 struct igb_ring *tx_ring;
326 union e1000_adv_tx_desc *tx_desc;
327 struct my_u0 { u64 a; u64 b; } *u0;
328 struct igb_buffer *buffer_info;
329 struct igb_ring *rx_ring;
330 union e1000_adv_rx_desc *rx_desc;
331 u32 staterr;
332 int i = 0;
333
334 if (!netif_msg_hw(adapter))
335 return;
336
337 /* Print netdevice Info */
338 if (netdev) {
339 dev_info(&adapter->pdev->dev, "Net device Info\n");
340 printk(KERN_INFO "Device Name state "
341 "trans_start last_rx\n");
342 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
343 netdev->name,
344 netdev->state,
345 netdev->trans_start,
346 netdev->last_rx);
347 }
348
349 /* Print Registers */
350 dev_info(&adapter->pdev->dev, "Register Dump\n");
351 printk(KERN_INFO " Register Name Value\n");
352 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
353 reginfo->name; reginfo++) {
354 igb_regdump(hw, reginfo);
355 }
356
357 /* Print TX Ring Summary */
358 if (!netdev || !netif_running(netdev))
359 goto exit;
360
361 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
362 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
363 " leng ntw timestamp\n");
364 for (n = 0; n < adapter->num_tx_queues; n++) {
365 tx_ring = adapter->tx_ring[n];
366 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
367 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
368 n, tx_ring->next_to_use, tx_ring->next_to_clean,
369 (u64)buffer_info->dma,
370 buffer_info->length,
371 buffer_info->next_to_watch,
372 (u64)buffer_info->time_stamp);
373 }
374
375 /* Print TX Rings */
376 if (!netif_msg_tx_done(adapter))
377 goto rx_ring_summary;
378
379 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
380
381 /* Transmit Descriptor Formats
382 *
383 * Advanced Transmit Descriptor
384 * +--------------------------------------------------------------+
385 * 0 | Buffer Address [63:0] |
386 * +--------------------------------------------------------------+
387 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
388 * +--------------------------------------------------------------+
389 * 63 46 45 40 39 38 36 35 32 31 24 15 0
390 */
391
392 for (n = 0; n < adapter->num_tx_queues; n++) {
393 tx_ring = adapter->tx_ring[n];
394 printk(KERN_INFO "------------------------------------\n");
395 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "T [desc] [address 63:0 ] "
398 "[PlPOCIStDDM Ln] [bi->dma ] "
399 "leng ntw timestamp bi->skb\n");
400
401 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
403 buffer_info = &tx_ring->buffer_info[i];
404 u0 = (struct my_u0 *)tx_desc;
405 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
406 " %04X %3X %016llX %p", i,
407 le64_to_cpu(u0->a),
408 le64_to_cpu(u0->b),
409 (u64)buffer_info->dma,
410 buffer_info->length,
411 buffer_info->next_to_watch,
412 (u64)buffer_info->time_stamp,
413 buffer_info->skb);
414 if (i == tx_ring->next_to_use &&
415 i == tx_ring->next_to_clean)
416 printk(KERN_CONT " NTC/U\n");
417 else if (i == tx_ring->next_to_use)
418 printk(KERN_CONT " NTU\n");
419 else if (i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC\n");
421 else
422 printk(KERN_CONT "\n");
423
424 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
425 print_hex_dump(KERN_INFO, "",
426 DUMP_PREFIX_ADDRESS,
427 16, 1, phys_to_virt(buffer_info->dma),
428 buffer_info->length, true);
429 }
430 }
431
432 /* Print RX Rings Summary */
433rx_ring_summary:
434 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
435 printk(KERN_INFO "Queue [NTU] [NTC]\n");
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
438 printk(KERN_INFO " %5d %5X %5X\n", n,
439 rx_ring->next_to_use, rx_ring->next_to_clean);
440 }
441
442 /* Print RX Rings */
443 if (!netif_msg_rx_status(adapter))
444 goto exit;
445
446 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
447
448 /* Advanced Receive Descriptor (Read) Format
449 * 63 1 0
450 * +-----------------------------------------------------+
451 * 0 | Packet Buffer Address [63:1] |A0/NSE|
452 * +----------------------------------------------+------+
453 * 8 | Header Buffer Address [63:1] | DD |
454 * +-----------------------------------------------------+
455 *
456 *
457 * Advanced Receive Descriptor (Write-Back) Format
458 *
459 * 63 48 47 32 31 30 21 20 17 16 4 3 0
460 * +------------------------------------------------------+
461 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
462 * | Checksum Ident | | | | Type | Type |
463 * +------------------------------------------------------+
464 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
465 * +------------------------------------------------------+
466 * 63 48 47 32 31 20 19 0
467 */
468
469 for (n = 0; n < adapter->num_rx_queues; n++) {
470 rx_ring = adapter->rx_ring[n];
471 printk(KERN_INFO "------------------------------------\n");
472 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "R [desc] [ PktBuf A0] "
475 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
476 "<-- Adv Rx Read format\n");
477 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
478 "[vl er S cks ln] ---------------- [bi->skb] "
479 "<-- Adv Rx Write-Back format\n");
480
481 for (i = 0; i < rx_ring->count; i++) {
482 buffer_info = &rx_ring->buffer_info[i];
483 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
484 u0 = (struct my_u0 *)rx_desc;
485 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
486 if (staterr & E1000_RXD_STAT_DD) {
487 /* Descriptor Done */
488 printk(KERN_INFO "RWB[0x%03X] %016llX "
489 "%016llX ---------------- %p", i,
490 le64_to_cpu(u0->a),
491 le64_to_cpu(u0->b),
492 buffer_info->skb);
493 } else {
494 printk(KERN_INFO "R [0x%03X] %016llX "
495 "%016llX %016llX %p", i,
496 le64_to_cpu(u0->a),
497 le64_to_cpu(u0->b),
498 (u64)buffer_info->dma,
499 buffer_info->skb);
500
501 if (netif_msg_pktdata(adapter)) {
502 print_hex_dump(KERN_INFO, "",
503 DUMP_PREFIX_ADDRESS,
504 16, 1,
505 phys_to_virt(buffer_info->dma),
506 rx_ring->rx_buffer_len, true);
507 if (rx_ring->rx_buffer_len
508 < IGB_RXBUFFER_1024)
509 print_hex_dump(KERN_INFO, "",
510 DUMP_PREFIX_ADDRESS,
511 16, 1,
512 phys_to_virt(
513 buffer_info->page_dma +
514 buffer_info->page_offset),
515 PAGE_SIZE/2, true);
516 }
517 }
518
519 if (i == rx_ring->next_to_use)
520 printk(KERN_CONT " NTU\n");
521 else if (i == rx_ring->next_to_clean)
522 printk(KERN_CONT " NTC\n");
523 else
524 printk(KERN_CONT "\n");
525
526 }
527 }
528
529exit:
530 return;
531}
532
533
38c845c7
PO		 534/**
535 * igb_read_clock - read raw cycle counter (to be used by time counter)
536 */
537static cycle_t igb_read_clock(const struct cyclecounter *tc)
538{
539 struct igb_adapter *adapter =
540 container_of(tc, struct igb_adapter, cycles);
541 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 542 u64 stamp = 0;
543 int shift = 0;
38c845c7 544
55cac248
AD		 545 /*
546 * The timestamp latches on lowest register read. For the 82580
547 * the lowest register is SYSTIMR instead of SYSTIML. However we never
548 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
549 */
550 if (hw->mac.type == e1000_82580) {
551 stamp = rd32(E1000_SYSTIMR) >> 8;
552 shift = IGB_82580_TSYNC_SHIFT;
553 }
554
c5b9bd5e
AD		 555 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
556 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 557 return stamp;
558}
559
9d5c8243 560/**
c041076a 561 * igb_get_hw_dev - return device
9d5c8243
AK		 562 * used by hardware layer to print debugging information
563 **/
c041076a 564struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 565{
566 struct igb_adapter *adapter = hw->back;
c041076a 567 return adapter->netdev;
9d5c8243 568}
38c845c7 569
9d5c8243
AK		 570/**
571 * igb_init_module - Driver Registration Routine
572 *
573 * igb_init_module is the first routine called when the driver is
574 * loaded. All it does is register with the PCI subsystem.
575 **/
576static int __init igb_init_module(void)
577{
578 int ret;
579 printk(KERN_INFO "%s - version %s\n",
580 igb_driver_string, igb_driver_version);
581
582 printk(KERN_INFO "%s\n", igb_copyright);
583
421e02f0 584#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 585 dca_register_notify(&dca_notifier);
586#endif
bbd98fe4 587 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 588 return ret;
589}
590
591module_init(igb_init_module);
592
593/**
594 * igb_exit_module - Driver Exit Cleanup Routine
595 *
596 * igb_exit_module is called just before the driver is removed
597 * from memory.
598 **/
599static void __exit igb_exit_module(void)
600{
421e02f0 601#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 602 dca_unregister_notify(&dca_notifier);
603#endif
9d5c8243
AK		 604 pci_unregister_driver(&igb_driver);
605}
606
607module_exit(igb_exit_module);
608
26bc19ec
AD		 609#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
610/**
611 * igb_cache_ring_register - Descriptor ring to register mapping
612 * @adapter: board private structure to initialize
613 *
614 * Once we know the feature-set enabled for the device, we'll cache
615 * the register offset the descriptor ring is assigned to.
616 **/
617static void igb_cache_ring_register(struct igb_adapter *adapter)
618{
ee1b9f06 619 int i = 0, j = 0;
047e0030 620 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 621
622 switch (adapter->hw.mac.type) {
623 case e1000_82576:
624 /* The queues are allocated for virtualization such that VF 0
625 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
626 * In order to avoid collision we start at the first free queue
627 * and continue consuming queues in the same sequence
628 */
ee1b9f06 629 if (adapter->vfs_allocated_count) {
a99955fc 630 for (; i < adapter->rss_queues; i++)
3025a446
AD		 631 adapter->rx_ring[i]->reg_idx = rbase_offset +
632 Q_IDX_82576(i);
ee1b9f06 633 }
26bc19ec 634 case e1000_82575:
55cac248 635 case e1000_82580:
d2ba2ed8 636 case e1000_i350:
26bc19ec 637 default:
ee1b9f06 638 for (; i < adapter->num_rx_queues; i++)
3025a446 639 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 640 for (; j < adapter->num_tx_queues; j++)
3025a446 641 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 642 break;
643 }
644}
645
047e0030
AD		 646static void igb_free_queues(struct igb_adapter *adapter)
647{
3025a446 648 int i;
047e0030 649
3025a446
AD		 650 for (i = 0; i < adapter->num_tx_queues; i++) {
651 kfree(adapter->tx_ring[i]);
652 adapter->tx_ring[i] = NULL;
653 }
654 for (i = 0; i < adapter->num_rx_queues; i++) {
655 kfree(adapter->rx_ring[i]);
656 adapter->rx_ring[i] = NULL;
657 }
047e0030
AD		 658 adapter->num_rx_queues = 0;
659 adapter->num_tx_queues = 0;
660}
661
9d5c8243
AK		 662/**
663 * igb_alloc_queues - Allocate memory for all rings
664 * @adapter: board private structure to initialize
665 *
666 * We allocate one ring per queue at run-time since we don't know the
667 * number of queues at compile-time.
668 **/
669static int igb_alloc_queues(struct igb_adapter *adapter)
670{
3025a446 671 struct igb_ring *ring;
9d5c8243
AK		 672 int i;
673
661086df 674 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 675 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
676 if (!ring)
677 goto err;
68fd9910 678 ring->count = adapter->tx_ring_count;
661086df 679 ring->queue_index = i;
59d71989 680 ring->dev = &adapter->pdev->dev;
e694e964 681 ring->netdev = adapter->netdev;
85ad76b2
AD		 682 /* For 82575, context index must be unique per ring. */
683 if (adapter->hw.mac.type == e1000_82575)
684 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 685 adapter->tx_ring[i] = ring;
661086df 686 }
85ad76b2 687
9d5c8243 688 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 689 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
690 if (!ring)
691 goto err;
68fd9910 692 ring->count = adapter->rx_ring_count;
844290e5 693 ring->queue_index = i;
59d71989 694 ring->dev = &adapter->pdev->dev;
e694e964 695 ring->netdev = adapter->netdev;
4c844851 696 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 697 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
698 /* set flag indicating ring supports SCTP checksum offload */
699 if (adapter->hw.mac.type >= e1000_82576)
700 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 701 adapter->rx_ring[i] = ring;
9d5c8243 702 }
26bc19ec
AD		 703
704 igb_cache_ring_register(adapter);
9d5c8243 705
047e0030 706 return 0;
a88f10ec 707
047e0030
AD		 708err:
709 igb_free_queues(adapter);
d1a8c9e1 710
047e0030 711 return -ENOMEM;
a88f10ec
AD		 712}
713
9d5c8243 714#define IGB_N0_QUEUE -1
047e0030 715static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 716{
717 u32 msixbm = 0;
047e0030 718 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 719 struct e1000_hw *hw = &adapter->hw;
2d064c06 720 u32 ivar, index;
047e0030
AD		 721 int rx_queue = IGB_N0_QUEUE;
722 int tx_queue = IGB_N0_QUEUE;
723
724 if (q_vector->rx_ring)
725 rx_queue = q_vector->rx_ring->reg_idx;
726 if (q_vector->tx_ring)
727 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 728
729 switch (hw->mac.type) {
730 case e1000_82575:
9d5c8243
AK		 731 /* The 82575 assigns vectors using a bitmask, which matches the
732 bitmask for the EICR/EIMS/EIMC registers. To assign one
733 or more queues to a vector, we write the appropriate bits
734 into the MSIXBM register for that vector. */
047e0030 735 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 736 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 737 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 738 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 739 if (!adapter->msix_entries && msix_vector == 0)
740 msixbm |= E1000_EIMS_OTHER;
9d5c8243 741 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 742 q_vector->eims_value = msixbm;
2d064c06
AD		 743 break;
744 case e1000_82576:
26bc19ec 745 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 746 Each queue has a single entry in the table to which we write
747 a vector number along with a "valid" bit. Sadly, the layout
748 of the table is somewhat counterintuitive. */
749 if (rx_queue > IGB_N0_QUEUE) {
047e0030 750 index = (rx_queue & 0x7);
2d064c06 751 ivar = array_rd32(E1000_IVAR0, index);
047e0030 752 if (rx_queue < 8) {
26bc19ec
AD		 753 /* vector goes into low byte of register */
754 ivar = ivar & 0xFFFFFF00;
755 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 756 } else {
757 /* vector goes into third byte of register */
758 ivar = ivar & 0xFF00FFFF;
759 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 760 }
2d064c06
AD		 761 array_wr32(E1000_IVAR0, index, ivar);
762 }
763 if (tx_queue > IGB_N0_QUEUE) {
047e0030 764 index = (tx_queue & 0x7);
2d064c06 765 ivar = array_rd32(E1000_IVAR0, index);
047e0030 766 if (tx_queue < 8) {
26bc19ec
AD		 767 /* vector goes into second byte of register */
768 ivar = ivar & 0xFFFF00FF;
769 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 770 } else {
771 /* vector goes into high byte of register */
772 ivar = ivar & 0x00FFFFFF;
773 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 774 }
2d064c06
AD		 775 array_wr32(E1000_IVAR0, index, ivar);
776 }
047e0030 777 q_vector->eims_value = 1 << msix_vector;
2d064c06 778 break;
55cac248 779 case e1000_82580:
d2ba2ed8 780 case e1000_i350:
55cac248
AD		 781 /* 82580 uses the same table-based approach as 82576 but has fewer
782 entries as a result we carry over for queues greater than 4. */
783 if (rx_queue > IGB_N0_QUEUE) {
784 index = (rx_queue >> 1);
785 ivar = array_rd32(E1000_IVAR0, index);
786 if (rx_queue & 0x1) {
787 /* vector goes into third byte of register */
788 ivar = ivar & 0xFF00FFFF;
789 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
790 } else {
791 /* vector goes into low byte of register */
792 ivar = ivar & 0xFFFFFF00;
793 ivar |= msix_vector | E1000_IVAR_VALID;
794 }
795 array_wr32(E1000_IVAR0, index, ivar);
796 }
797 if (tx_queue > IGB_N0_QUEUE) {
798 index = (tx_queue >> 1);
799 ivar = array_rd32(E1000_IVAR0, index);
800 if (tx_queue & 0x1) {
801 /* vector goes into high byte of register */
802 ivar = ivar & 0x00FFFFFF;
803 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
804 } else {
805 /* vector goes into second byte of register */
806 ivar = ivar & 0xFFFF00FF;
807 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
808 }
809 array_wr32(E1000_IVAR0, index, ivar);
810 }
811 q_vector->eims_value = 1 << msix_vector;
812 break;
2d064c06
AD		 813 default:
814 BUG();
815 break;
816 }
26b39276
AD		 817
818 /* add q_vector eims value to global eims_enable_mask */
819 adapter->eims_enable_mask |= q_vector->eims_value;
820
821 /* configure q_vector to set itr on first interrupt */
822 q_vector->set_itr = 1;
9d5c8243
AK		 823}
824
825/**
826 * igb_configure_msix - Configure MSI-X hardware
827 *
828 * igb_configure_msix sets up the hardware to properly
829 * generate MSI-X interrupts.
830 **/
831static void igb_configure_msix(struct igb_adapter *adapter)
832{
833 u32 tmp;
834 int i, vector = 0;
835 struct e1000_hw *hw = &adapter->hw;
836
837 adapter->eims_enable_mask = 0;
9d5c8243
AK		 838
839 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 840 switch (hw->mac.type) {
841 case e1000_82575:
9d5c8243
AK		 842 tmp = rd32(E1000_CTRL_EXT);
843 /* enable MSI-X PBA support*/
844 tmp |= E1000_CTRL_EXT_PBA_CLR;
845
846 /* Auto-Mask interrupts upon ICR read. */
847 tmp |= E1000_CTRL_EXT_EIAME;
848 tmp |= E1000_CTRL_EXT_IRCA;
849
850 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 851
852 /* enable msix_other interrupt */
853 array_wr32(E1000_MSIXBM(0), vector++,
854 E1000_EIMS_OTHER);
844290e5 855 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 856
2d064c06
AD		 857 break;
858
859 case e1000_82576:
55cac248 860 case e1000_82580:
d2ba2ed8 861 case e1000_i350:
047e0030
AD		 862 /* Turn on MSI-X capability first, or our settings
863 * won't stick. And it will take days to debug. */
864 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
865 E1000_GPIE_PBA | E1000_GPIE_EIAME |
866 E1000_GPIE_NSICR);
867
868 /* enable msix_other interrupt */
869 adapter->eims_other = 1 << vector;
2d064c06 870 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 871
047e0030 872 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 873 break;
874 default:
875 /* do nothing, since nothing else supports MSI-X */
876 break;
877 } /* switch (hw->mac.type) */
047e0030
AD		 878
879 adapter->eims_enable_mask |= adapter->eims_other;
880
26b39276
AD		 881 for (i = 0; i < adapter->num_q_vectors; i++)
882 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 883
9d5c8243
AK		 884 wrfl();
885}
886
887/**
888 * igb_request_msix - Initialize MSI-X interrupts
889 *
890 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
891 * kernel.
892 **/
893static int igb_request_msix(struct igb_adapter *adapter)
894{
895 struct net_device *netdev = adapter->netdev;
047e0030 896 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 897 int i, err = 0, vector = 0;
898
047e0030 899 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 900 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 901 if (err)
902 goto out;
903 vector++;
904
905 for (i = 0; i < adapter->num_q_vectors; i++) {
906 struct igb_q_vector *q_vector = adapter->q_vector[i];
907
908 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
909
910 if (q_vector->rx_ring && q_vector->tx_ring)
911 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
912 q_vector->rx_ring->queue_index);
913 else if (q_vector->tx_ring)
914 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
915 q_vector->tx_ring->queue_index);
916 else if (q_vector->rx_ring)
917 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
918 q_vector->rx_ring->queue_index);
9d5c8243 919 else
047e0030
AD		 920 sprintf(q_vector->name, "%s-unused", netdev->name);
921
9d5c8243 922 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 923 igb_msix_ring, 0, q_vector->name,
047e0030 924 q_vector);
9d5c8243
AK		 925 if (err)
926 goto out;
9d5c8243
AK		 927 vector++;
928 }
929
9d5c8243
AK		 930 igb_configure_msix(adapter);
931 return 0;
932out:
933 return err;
934}
935
936static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
937{
938 if (adapter->msix_entries) {
939 pci_disable_msix(adapter->pdev);
940 kfree(adapter->msix_entries);
941 adapter->msix_entries = NULL;
047e0030 942 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 943 pci_disable_msi(adapter->pdev);
047e0030 944 }
9d5c8243
AK		 945}
946
047e0030
AD		 947/**
948 * igb_free_q_vectors - Free memory allocated for interrupt vectors
949 * @adapter: board private structure to initialize
950 *
951 * This function frees the memory allocated to the q_vectors. In addition if
952 * NAPI is enabled it will delete any references to the NAPI struct prior
953 * to freeing the q_vector.
954 **/
955static void igb_free_q_vectors(struct igb_adapter *adapter)
956{
957 int v_idx;
958
959 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
960 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
961 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 962 if (!q_vector)
963 continue;
047e0030
AD		 964 netif_napi_del(&q_vector->napi);
965 kfree(q_vector);
966 }
967 adapter->num_q_vectors = 0;
968}
969
970/**
971 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
972 *
973 * This function resets the device so that it has 0 rx queues, tx queues, and
974 * MSI-X interrupts allocated.
975 */
976static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
977{
978 igb_free_queues(adapter);
979 igb_free_q_vectors(adapter);
980 igb_reset_interrupt_capability(adapter);
981}
9d5c8243
AK		 982
983/**
984 * igb_set_interrupt_capability - set MSI or MSI-X if supported
985 *
986 * Attempt to configure interrupts using the best available
987 * capabilities of the hardware and kernel.
988 **/
989static void igb_set_interrupt_capability(struct igb_adapter *adapter)
990{
991 int err;
992 int numvecs, i;
993
83b7180d 994 /* Number of supported queues. */
a99955fc 995 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 996 if (adapter->vfs_allocated_count)
997 adapter->num_tx_queues = 1;
998 else
999 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1000
047e0030
AD		 1001 /* start with one vector for every rx queue */
1002 numvecs = adapter->num_rx_queues;
1003
3ad2f3fb 1004 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1005 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006 numvecs += adapter->num_tx_queues;
047e0030
AD		 1007
1008 /* store the number of vectors reserved for queues */
1009 adapter->num_q_vectors = numvecs;
1010
1011 /* add 1 vector for link status interrupts */
1012 numvecs++;
9d5c8243
AK		 1013 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1014 GFP_KERNEL);
1015 if (!adapter->msix_entries)
1016 goto msi_only;
1017
1018 for (i = 0; i < numvecs; i++)
1019 adapter->msix_entries[i].entry = i;
1020
1021 err = pci_enable_msix(adapter->pdev,
1022 adapter->msix_entries,
1023 numvecs);
1024 if (err == 0)
34a20e89 1025 goto out;
9d5c8243
AK		 1026
1027 igb_reset_interrupt_capability(adapter);
1028
1029 /* If we can't do MSI-X, try MSI */
1030msi_only:
2a3abf6d
AD		 1031#ifdef CONFIG_PCI_IOV
1032 /* disable SR-IOV for non MSI-X configurations */
1033 if (adapter->vf_data) {
1034 struct e1000_hw *hw = &adapter->hw;
1035 /* disable iov and allow time for transactions to clear */
1036 pci_disable_sriov(adapter->pdev);
1037 msleep(500);
1038
1039 kfree(adapter->vf_data);
1040 adapter->vf_data = NULL;
1041 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1042 msleep(100);
1043 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1044 }
1045#endif
4fc82adf 1046 adapter->vfs_allocated_count = 0;
a99955fc 1047 adapter->rss_queues = 1;
4fc82adf 1048 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1049 adapter->num_rx_queues = 1;
661086df 1050 adapter->num_tx_queues = 1;
047e0030 1051 adapter->num_q_vectors = 1;
9d5c8243 1052 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1053 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1054out:
661086df 1055 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 1056 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 1057}
1058
047e0030
AD		 1059/**
1060 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061 * @adapter: board private structure to initialize
1062 *
1063 * We allocate one q_vector per queue interrupt. If allocation fails we
1064 * return -ENOMEM.
1065 **/
1066static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1067{
1068 struct igb_q_vector *q_vector;
1069 struct e1000_hw *hw = &adapter->hw;
1070 int v_idx;
1071
1072 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1074 if (!q_vector)
1075 goto err_out;
1076 q_vector->adapter = adapter;
047e0030
AD		 1077 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1079 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080 adapter->q_vector[v_idx] = q_vector;
1081 }
1082 return 0;
1083
1084err_out:
fe0592b4 1085 igb_free_q_vectors(adapter);
047e0030
AD		 1086 return -ENOMEM;
1087}
1088
1089static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090 int ring_idx, int v_idx)
1091{
3025a446 1092 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1093
3025a446 1094 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1095 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1096 q_vector->itr_val = adapter->rx_itr_setting;
1097 if (q_vector->itr_val && q_vector->itr_val <= 3)
1098 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1099}
1100
1101static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102 int ring_idx, int v_idx)
1103{
3025a446 1104 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1105
3025a446 1106 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1107 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1108 q_vector->itr_val = adapter->tx_itr_setting;
1109 if (q_vector->itr_val && q_vector->itr_val <= 3)
1110 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1111}
1112
1113/**
1114 * igb_map_ring_to_vector - maps allocated queues to vectors
1115 *
1116 * This function maps the recently allocated queues to vectors.
1117 **/
1118static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1119{
1120 int i;
1121 int v_idx = 0;
1122
1123 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124 (adapter->num_q_vectors < adapter->num_tx_queues))
1125 return -ENOMEM;
1126
1127 if (adapter->num_q_vectors >=
1128 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129 for (i = 0; i < adapter->num_rx_queues; i++)
1130 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131 for (i = 0; i < adapter->num_tx_queues; i++)
1132 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1133 } else {
1134 for (i = 0; i < adapter->num_rx_queues; i++) {
1135 if (i < adapter->num_tx_queues)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1138 }
1139 for (; i < adapter->num_tx_queues; i++)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1141 }
1142 return 0;
1143}
1144
1145/**
1146 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1147 *
1148 * This function initializes the interrupts and allocates all of the queues.
1149 **/
1150static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1151{
1152 struct pci_dev *pdev = adapter->pdev;
1153 int err;
1154
1155 igb_set_interrupt_capability(adapter);
1156
1157 err = igb_alloc_q_vectors(adapter);
1158 if (err) {
1159 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160 goto err_alloc_q_vectors;
1161 }
1162
1163 err = igb_alloc_queues(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166 goto err_alloc_queues;
1167 }
1168
1169 err = igb_map_ring_to_vector(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172 goto err_map_queues;
1173 }
1174
1175
1176 return 0;
1177err_map_queues:
1178 igb_free_queues(adapter);
1179err_alloc_queues:
1180 igb_free_q_vectors(adapter);
1181err_alloc_q_vectors:
1182 igb_reset_interrupt_capability(adapter);
1183 return err;
1184}
1185
9d5c8243
AK		 1186/**
1187 * igb_request_irq - initialize interrupts
1188 *
1189 * Attempts to configure interrupts using the best available
1190 * capabilities of the hardware and kernel.
1191 **/
1192static int igb_request_irq(struct igb_adapter *adapter)
1193{
1194 struct net_device *netdev = adapter->netdev;
047e0030 1195 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1196 int err = 0;
1197
1198 if (adapter->msix_entries) {
1199 err = igb_request_msix(adapter);
844290e5 1200 if (!err)
9d5c8243 1201 goto request_done;
9d5c8243 1202 /* fall back to MSI */
047e0030 1203 igb_clear_interrupt_scheme(adapter);
9d5c8243 1204 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1205 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1206 igb_free_all_tx_resources(adapter);
1207 igb_free_all_rx_resources(adapter);
047e0030 1208 adapter->num_tx_queues = 1;
9d5c8243 1209 adapter->num_rx_queues = 1;
047e0030
AD		 1210 adapter->num_q_vectors = 1;
1211 err = igb_alloc_q_vectors(adapter);
1212 if (err) {
1213 dev_err(&pdev->dev,
1214 "Unable to allocate memory for vectors\n");
1215 goto request_done;
1216 }
1217 err = igb_alloc_queues(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for queues\n");
1221 igb_free_q_vectors(adapter);
1222 goto request_done;
1223 }
1224 igb_setup_all_tx_resources(adapter);
1225 igb_setup_all_rx_resources(adapter);
844290e5 1226 } else {
feeb2721 1227 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1228 }
844290e5 1229
7dfc16fa 1230 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1231 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1232 netdev->name, adapter);
9d5c8243
AK		 1233 if (!err)
1234 goto request_done;
047e0030 1235
9d5c8243
AK		 1236 /* fall back to legacy interrupts */
1237 igb_reset_interrupt_capability(adapter);
7dfc16fa 1238 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1239 }
1240
a0607fd3 1241 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1242 netdev->name, adapter);
9d5c8243 1243
6cb5e577 1244 if (err)
9d5c8243
AK		 1245 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1246 err);
9d5c8243
AK		 1247
1248request_done:
1249 return err;
1250}
1251
1252static void igb_free_irq(struct igb_adapter *adapter)
1253{
9d5c8243
AK		 1254 if (adapter->msix_entries) {
1255 int vector = 0, i;
1256
047e0030 1257 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1258
047e0030
AD		 1259 for (i = 0; i < adapter->num_q_vectors; i++) {
1260 struct igb_q_vector *q_vector = adapter->q_vector[i];
1261 free_irq(adapter->msix_entries[vector++].vector,
1262 q_vector);
1263 }
1264 } else {
1265 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1266 }
9d5c8243
AK		 1267}
1268
1269/**
1270 * igb_irq_disable - Mask off interrupt generation on the NIC
1271 * @adapter: board private structure
1272 **/
1273static void igb_irq_disable(struct igb_adapter *adapter)
1274{
1275 struct e1000_hw *hw = &adapter->hw;
1276
25568a53
AD		 1277 /*
1278 * we need to be careful when disabling interrupts. The VFs are also
1279 * mapped into these registers and so clearing the bits can cause
1280 * issues on the VF drivers so we only need to clear what we set
1281 */
9d5c8243 1282 if (adapter->msix_entries) {
2dfd1212
AD		 1283 u32 regval = rd32(E1000_EIAM);
1284 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286 regval = rd32(E1000_EIAC);
1287 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1288 }
844290e5
PW		 1289
1290 wr32(E1000_IAM, 0);
9d5c8243
AK		 1291 wr32(E1000_IMC, ~0);
1292 wrfl();
1293 synchronize_irq(adapter->pdev->irq);
1294}
1295
1296/**
1297 * igb_irq_enable - Enable default interrupt generation settings
1298 * @adapter: board private structure
1299 **/
1300static void igb_irq_enable(struct igb_adapter *adapter)
1301{
1302 struct e1000_hw *hw = &adapter->hw;
1303
1304 if (adapter->msix_entries) {
25568a53 1305 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1306 u32 regval = rd32(E1000_EIAC);
1307 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1308 regval = rd32(E1000_EIAM);
1309 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1310 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1311 if (adapter->vfs_allocated_count) {
4ae196df 1312 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1313 ims |= E1000_IMS_VMMB;
1314 }
55cac248
AD		 1315 if (adapter->hw.mac.type == e1000_82580)
1316 ims |= E1000_IMS_DRSTA;
1317
25568a53 1318 wr32(E1000_IMS, ims);
844290e5 1319 } else {
55cac248
AD		 1320 wr32(E1000_IMS, IMS_ENABLE_MASK |
1321 E1000_IMS_DRSTA);
1322 wr32(E1000_IAM, IMS_ENABLE_MASK |
1323 E1000_IMS_DRSTA);
844290e5 1324 }
9d5c8243
AK		 1325}
1326
1327static void igb_update_mng_vlan(struct igb_adapter *adapter)
1328{
51466239 1329 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1330 u16 vid = adapter->hw.mng_cookie.vlan_id;
1331 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1332
1333 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1334 /* add VID to filter table */
1335 igb_vfta_set(hw, vid, true);
1336 adapter->mng_vlan_id = vid;
1337 } else {
1338 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1339 }
1340
1341 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1342 (vid != old_vid) &&
1343 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1344 /* remove VID from filter table */
1345 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1346 }
1347}
1348
1349/**
1350 * igb_release_hw_control - release control of the h/w to f/w
1351 * @adapter: address of board private structure
1352 *
1353 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1354 * For ASF and Pass Through versions of f/w this means that the
1355 * driver is no longer loaded.
1356 *
1357 **/
1358static void igb_release_hw_control(struct igb_adapter *adapter)
1359{
1360 struct e1000_hw *hw = &adapter->hw;
1361 u32 ctrl_ext;
1362
1363 /* Let firmware take over control of h/w */
1364 ctrl_ext = rd32(E1000_CTRL_EXT);
1365 wr32(E1000_CTRL_EXT,
1366 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1367}
1368
9d5c8243
AK		 1369/**
1370 * igb_get_hw_control - get control of the h/w from f/w
1371 * @adapter: address of board private structure
1372 *
1373 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1374 * For ASF and Pass Through versions of f/w this means that
1375 * the driver is loaded.
1376 *
1377 **/
1378static void igb_get_hw_control(struct igb_adapter *adapter)
1379{
1380 struct e1000_hw *hw = &adapter->hw;
1381 u32 ctrl_ext;
1382
1383 /* Let firmware know the driver has taken over */
1384 ctrl_ext = rd32(E1000_CTRL_EXT);
1385 wr32(E1000_CTRL_EXT,
1386 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1387}
1388
9d5c8243
AK		 1389/**
1390 * igb_configure - configure the hardware for RX and TX
1391 * @adapter: private board structure
1392 **/
1393static void igb_configure(struct igb_adapter *adapter)
1394{
1395 struct net_device *netdev = adapter->netdev;
1396 int i;
1397
1398 igb_get_hw_control(adapter);
ff41f8dc 1399 igb_set_rx_mode(netdev);
9d5c8243
AK		 1400
1401 igb_restore_vlan(adapter);
9d5c8243 1402
85b430b4 1403 igb_setup_tctl(adapter);
06cf2666 1404 igb_setup_mrqc(adapter);
9d5c8243 1405 igb_setup_rctl(adapter);
85b430b4
AD		 1406
1407 igb_configure_tx(adapter);
9d5c8243 1408 igb_configure_rx(adapter);
662d7205
AD		 1409
1410 igb_rx_fifo_flush_82575(&adapter->hw);
1411
c493ea45 1412 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1413 * at least 1 descriptor unused to make sure
1414 * next_to_use != next_to_clean */
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1416 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1417 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1418 }
9d5c8243
AK		 1419}
1420
88a268c1
NN		 1421/**
1422 * igb_power_up_link - Power up the phy/serdes link
1423 * @adapter: address of board private structure
1424 **/
1425void igb_power_up_link(struct igb_adapter *adapter)
1426{
1427 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1428 igb_power_up_phy_copper(&adapter->hw);
1429 else
1430 igb_power_up_serdes_link_82575(&adapter->hw);
1431}
1432
1433/**
1434 * igb_power_down_link - Power down the phy/serdes link
1435 * @adapter: address of board private structure
1436 */
1437static void igb_power_down_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_down_phy_copper_82575(&adapter->hw);
1441 else
1442 igb_shutdown_serdes_link_82575(&adapter->hw);
1443}
9d5c8243
AK		 1444
1445/**
1446 * igb_up - Open the interface and prepare it to handle traffic
1447 * @adapter: board private structure
1448 **/
9d5c8243
AK		 1449int igb_up(struct igb_adapter *adapter)
1450{
1451 struct e1000_hw *hw = &adapter->hw;
1452 int i;
1453
1454 /* hardware has been reset, we need to reload some things */
1455 igb_configure(adapter);
1456
1457 clear_bit(__IGB_DOWN, &adapter->state);
1458
047e0030
AD		 1459 for (i = 0; i < adapter->num_q_vectors; i++) {
1460 struct igb_q_vector *q_vector = adapter->q_vector[i];
1461 napi_enable(&q_vector->napi);
1462 }
844290e5 1463 if (adapter->msix_entries)
9d5c8243 1464 igb_configure_msix(adapter);
feeb2721
AD		 1465 else
1466 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1467
1468 /* Clear any pending interrupts. */
1469 rd32(E1000_ICR);
1470 igb_irq_enable(adapter);
1471
d4960307
AD		 1472 /* notify VFs that reset has been completed */
1473 if (adapter->vfs_allocated_count) {
1474 u32 reg_data = rd32(E1000_CTRL_EXT);
1475 reg_data |= E1000_CTRL_EXT_PFRSTD;
1476 wr32(E1000_CTRL_EXT, reg_data);
1477 }
1478
4cb9be7a
JB		 1479 netif_tx_start_all_queues(adapter->netdev);
1480
25568a53
AD		 1481 /* start the watchdog. */
1482 hw->mac.get_link_status = 1;
1483 schedule_work(&adapter->watchdog_task);
1484
9d5c8243
AK		 1485 return 0;
1486}
1487
1488void igb_down(struct igb_adapter *adapter)
1489{
9d5c8243 1490 struct net_device *netdev = adapter->netdev;
330a6d6a 1491 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1492 u32 tctl, rctl;
1493 int i;
1494
1495 /* signal that we're down so the interrupt handler does not
1496 * reschedule our watchdog timer */
1497 set_bit(__IGB_DOWN, &adapter->state);
1498
1499 /* disable receives in the hardware */
1500 rctl = rd32(E1000_RCTL);
1501 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1502 /* flush and sleep below */
1503
fd2ea0a7 1504 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1505
1506 /* disable transmits in the hardware */
1507 tctl = rd32(E1000_TCTL);
1508 tctl &= ~E1000_TCTL_EN;
1509 wr32(E1000_TCTL, tctl);
1510 /* flush both disables and wait for them to finish */
1511 wrfl();
1512 msleep(10);
1513
047e0030
AD		 1514 for (i = 0; i < adapter->num_q_vectors; i++) {
1515 struct igb_q_vector *q_vector = adapter->q_vector[i];
1516 napi_disable(&q_vector->napi);
1517 }
9d5c8243 1518
9d5c8243
AK		 1519 igb_irq_disable(adapter);
1520
1521 del_timer_sync(&adapter->watchdog_timer);
1522 del_timer_sync(&adapter->phy_info_timer);
1523
9d5c8243 1524 netif_carrier_off(netdev);
04fe6358
AD		 1525
1526 /* record the stats before reset*/
1527 igb_update_stats(adapter);
1528
9d5c8243
AK		 1529 adapter->link_speed = 0;
1530 adapter->link_duplex = 0;
1531
3023682e
JK		 1532 if (!pci_channel_offline(adapter->pdev))
1533 igb_reset(adapter);
9d5c8243
AK		 1534 igb_clean_all_tx_rings(adapter);
1535 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1536#ifdef CONFIG_IGB_DCA
1537
1538 /* since we reset the hardware DCA settings were cleared */
1539 igb_setup_dca(adapter);
1540#endif
9d5c8243
AK		 1541}
1542
1543void igb_reinit_locked(struct igb_adapter *adapter)
1544{
1545 WARN_ON(in_interrupt());
1546 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1547 msleep(1);
1548 igb_down(adapter);
1549 igb_up(adapter);
1550 clear_bit(__IGB_RESETTING, &adapter->state);
1551}
1552
1553void igb_reset(struct igb_adapter *adapter)
1554{
090b1795 1555 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1556 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1557 struct e1000_mac_info *mac = &hw->mac;
1558 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1559 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1560 u16 hwm;
1561
1562 /* Repartition Pba for greater than 9k mtu
1563 * To take effect CTRL.RST is required.
1564 */
fa4dfae0 1565 switch (mac->type) {
d2ba2ed8 1566 case e1000_i350:
55cac248
AD		 1567 case e1000_82580:
1568 pba = rd32(E1000_RXPBS);
1569 pba = igb_rxpbs_adjust_82580(pba);
1570 break;
fa4dfae0 1571 case e1000_82576:
d249be54
AD		 1572 pba = rd32(E1000_RXPBS);
1573 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1574 break;
1575 case e1000_82575:
1576 default:
1577 pba = E1000_PBA_34K;
1578 break;
2d064c06 1579 }
9d5c8243 1580
2d064c06
AD		 1581 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1582 (mac->type < e1000_82576)) {
9d5c8243
AK		 1583 /* adjust PBA for jumbo frames */
1584 wr32(E1000_PBA, pba);
1585
1586 /* To maintain wire speed transmits, the Tx FIFO should be
1587 * large enough to accommodate two full transmit packets,
1588 * rounded up to the next 1KB and expressed in KB. Likewise,
1589 * the Rx FIFO should be large enough to accommodate at least
1590 * one full receive packet and is similarly rounded up and
1591 * expressed in KB. */
1592 pba = rd32(E1000_PBA);
1593 /* upper 16 bits has Tx packet buffer allocation size in KB */
1594 tx_space = pba >> 16;
1595 /* lower 16 bits has Rx packet buffer allocation size in KB */
1596 pba &= 0xffff;
1597 /* the tx fifo also stores 16 bytes of information about the tx
1598 * but don't include ethernet FCS because hardware appends it */
1599 min_tx_space = (adapter->max_frame_size +
85e8d004 1600 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1601 ETH_FCS_LEN) * 2;
1602 min_tx_space = ALIGN(min_tx_space, 1024);
1603 min_tx_space >>= 10;
1604 /* software strips receive CRC, so leave room for it */
1605 min_rx_space = adapter->max_frame_size;
1606 min_rx_space = ALIGN(min_rx_space, 1024);
1607 min_rx_space >>= 10;
1608
1609 /* If current Tx allocation is less than the min Tx FIFO size,
1610 * and the min Tx FIFO size is less than the current Rx FIFO
1611 * allocation, take space away from current Rx allocation */
1612 if (tx_space < min_tx_space &&
1613 ((min_tx_space - tx_space) < pba)) {
1614 pba = pba - (min_tx_space - tx_space);
1615
1616 /* if short on rx space, rx wins and must trump tx
1617 * adjustment */
1618 if (pba < min_rx_space)
1619 pba = min_rx_space;
1620 }
2d064c06 1621 wr32(E1000_PBA, pba);
9d5c8243 1622 }
9d5c8243
AK		 1623
1624 /* flow control settings */
1625 /* The high water mark must be low enough to fit one full frame
1626 * (or the size used for early receive) above it in the Rx FIFO.
1627 * Set it to the lower of:
1628 * - 90% of the Rx FIFO size, or
1629 * - the full Rx FIFO size minus one full frame */
1630 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1631 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1632
d405ea3e
AD		 1633 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1634 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1635 fc->pause_time = 0xFFFF;
1636 fc->send_xon = 1;
0cce119a 1637 fc->current_mode = fc->requested_mode;
9d5c8243 1638
4ae196df
AD		 1639 /* disable receive for all VFs and wait one second */
1640 if (adapter->vfs_allocated_count) {
1641 int i;
1642 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1643 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1644
1645 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1646 igb_ping_all_vfs(adapter);
4ae196df
AD		 1647
1648 /* disable transmits and receives */
1649 wr32(E1000_VFRE, 0);
1650 wr32(E1000_VFTE, 0);
1651 }
1652
9d5c8243 1653 /* Allow time for pending master requests to run */
330a6d6a 1654 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1655 wr32(E1000_WUC, 0);
1656
330a6d6a 1657 if (hw->mac.ops.init_hw(hw))
090b1795 1658 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1659
55cac248
AD		 1660 if (hw->mac.type == e1000_82580) {
1661 u32 reg = rd32(E1000_PCIEMISC);
1662 wr32(E1000_PCIEMISC,
1663 reg & ~E1000_PCIEMISC_LX_DECISION);
1664 }
88a268c1
NN		 1665 if (!netif_running(adapter->netdev))
1666 igb_power_down_link(adapter);
1667
9d5c8243
AK		 1668 igb_update_mng_vlan(adapter);
1669
1670 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1671 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1672
330a6d6a 1673 igb_get_phy_info(hw);
9d5c8243
AK		 1674}
1675
2e5c6922 1676static const struct net_device_ops igb_netdev_ops = {
559e9c49 1677 .ndo_open = igb_open,
2e5c6922 1678 .ndo_stop = igb_close,
00829823 1679 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1680 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1681 .ndo_set_rx_mode = igb_set_rx_mode,
1682 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1683 .ndo_set_mac_address = igb_set_mac,
1684 .ndo_change_mtu = igb_change_mtu,
1685 .ndo_do_ioctl = igb_ioctl,
1686 .ndo_tx_timeout = igb_tx_timeout,
1687 .ndo_validate_addr = eth_validate_addr,
1688 .ndo_vlan_rx_register = igb_vlan_rx_register,
1689 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1690 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1691 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1692 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1693 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1694 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1695#ifdef CONFIG_NET_POLL_CONTROLLER
1696 .ndo_poll_controller = igb_netpoll,
1697#endif
1698};
1699
9d5c8243
AK		 1700/**
1701 * igb_probe - Device Initialization Routine
1702 * @pdev: PCI device information struct
1703 * @ent: entry in igb_pci_tbl
1704 *
1705 * Returns 0 on success, negative on failure
1706 *
1707 * igb_probe initializes an adapter identified by a pci_dev structure.
1708 * The OS initialization, configuring of the adapter private structure,
1709 * and a hardware reset occur.
1710 **/
1711static int __devinit igb_probe(struct pci_dev *pdev,
1712 const struct pci_device_id *ent)
1713{
1714 struct net_device *netdev;
1715 struct igb_adapter *adapter;
1716 struct e1000_hw *hw;
4337e993
AD		 1717 u16 eeprom_data = 0;
1718 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1719 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1720 unsigned long mmio_start, mmio_len;
2d6a5e95 1721 int err, pci_using_dac;
9d5c8243
AK		 1722 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1723 u32 part_num;
1724
bded64a7
AG		 1725 /* Catch broken hardware that put the wrong VF device ID in
1726 * the PCIe SR-IOV capability.
1727 */
1728 if (pdev->is_virtfn) {
1729 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1730 pci_name(pdev), pdev->vendor, pdev->device);
1731 return -EINVAL;
1732 }
1733
aed5dec3 1734 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1735 if (err)
1736 return err;
1737
1738 pci_using_dac = 0;
59d71989 1739 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1740 if (!err) {
59d71989 1741 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1742 if (!err)
1743 pci_using_dac = 1;
1744 } else {
59d71989 1745 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1746 if (err) {
59d71989 1747 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1748 if (err) {
1749 dev_err(&pdev->dev, "No usable DMA "
1750 "configuration, aborting\n");
1751 goto err_dma;
1752 }
1753 }
1754 }
1755
aed5dec3
AD		 1756 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1757 IORESOURCE_MEM),
1758 igb_driver_name);
9d5c8243
AK		 1759 if (err)
1760 goto err_pci_reg;
1761
19d5afd4 1762 pci_enable_pcie_error_reporting(pdev);
40a914fa 1763
9d5c8243 1764 pci_set_master(pdev);
c682fc23 1765 pci_save_state(pdev);
9d5c8243
AK		 1766
1767 err = -ENOMEM;
1bfaf07b
AD		 1768 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1769 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1770 if (!netdev)
1771 goto err_alloc_etherdev;
1772
1773 SET_NETDEV_DEV(netdev, &pdev->dev);
1774
1775 pci_set_drvdata(pdev, netdev);
1776 adapter = netdev_priv(netdev);
1777 adapter->netdev = netdev;
1778 adapter->pdev = pdev;
1779 hw = &adapter->hw;
1780 hw->back = adapter;
1781 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1782
1783 mmio_start = pci_resource_start(pdev, 0);
1784 mmio_len = pci_resource_len(pdev, 0);
1785
1786 err = -EIO;
28b0759c
AD		 1787 hw->hw_addr = ioremap(mmio_start, mmio_len);
1788 if (!hw->hw_addr)
9d5c8243
AK		 1789 goto err_ioremap;
1790
2e5c6922 1791 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1792 igb_set_ethtool_ops(netdev);
9d5c8243 1793 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1794
1795 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1796
1797 netdev->mem_start = mmio_start;
1798 netdev->mem_end = mmio_start + mmio_len;
1799
9d5c8243
AK		 1800 /* PCI config space info */
1801 hw->vendor_id = pdev->vendor;
1802 hw->device_id = pdev->device;
1803 hw->revision_id = pdev->revision;
1804 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1805 hw->subsystem_device_id = pdev->subsystem_device;
1806
9d5c8243
AK		 1807 /* Copy the default MAC, PHY and NVM function pointers */
1808 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1809 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1810 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1811 /* Initialize skew-specific constants */
1812 err = ei->get_invariants(hw);
1813 if (err)
450c87c8 1814 goto err_sw_init;
9d5c8243 1815
450c87c8 1816 /* setup the private structure */
9d5c8243
AK		 1817 err = igb_sw_init(adapter);
1818 if (err)
1819 goto err_sw_init;
1820
1821 igb_get_bus_info_pcie(hw);
1822
1823 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1824
1825 /* Copper options */
1826 if (hw->phy.media_type == e1000_media_type_copper) {
1827 hw->phy.mdix = AUTO_ALL_MODES;
1828 hw->phy.disable_polarity_correction = false;
1829 hw->phy.ms_type = e1000_ms_hw_default;
1830 }
1831
1832 if (igb_check_reset_block(hw))
1833 dev_info(&pdev->dev,
1834 "PHY reset is blocked due to SOL/IDER session.\n");
1835
1836 netdev->features = NETIF_F_SG |
7d8eb29e 1837 NETIF_F_IP_CSUM |
9d5c8243
AK		 1838 NETIF_F_HW_VLAN_TX |
1839 NETIF_F_HW_VLAN_RX |
1840 NETIF_F_HW_VLAN_FILTER;
1841
7d8eb29e 1842 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1843 netdev->features |= NETIF_F_TSO;
9d5c8243 1844 netdev->features |= NETIF_F_TSO6;
5c0999b7 1845 netdev->features |= NETIF_F_GRO;
d3352520 1846
48f29ffc
JK		 1847 netdev->vlan_features |= NETIF_F_TSO;
1848 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1849 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1850 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1851 netdev->vlan_features |= NETIF_F_SG;
1852
9d5c8243
AK		 1853 if (pci_using_dac)
1854 netdev->features |= NETIF_F_HIGHDMA;
1855
5b043fb0 1856 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1857 netdev->features |= NETIF_F_SCTP_CSUM;
1858
330a6d6a 1859 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1860
1861 /* before reading the NVM, reset the controller to put the device in a
1862 * known good starting state */
1863 hw->mac.ops.reset_hw(hw);
1864
1865 /* make sure the NVM is good */
1866 if (igb_validate_nvm_checksum(hw) < 0) {
1867 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1868 err = -EIO;
1869 goto err_eeprom;
1870 }
1871
1872 /* copy the MAC address out of the NVM */
1873 if (hw->mac.ops.read_mac_addr(hw))
1874 dev_err(&pdev->dev, "NVM Read Error\n");
1875
1876 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1877 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1878
1879 if (!is_valid_ether_addr(netdev->perm_addr)) {
1880 dev_err(&pdev->dev, "Invalid MAC Address\n");
1881 err = -EIO;
1882 goto err_eeprom;
1883 }
1884
0e340485
AD		 1885 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1886 (unsigned long) adapter);
1887 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1888 (unsigned long) adapter);
9d5c8243
AK		 1889
1890 INIT_WORK(&adapter->reset_task, igb_reset_task);
1891 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1892
450c87c8 1893 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1894 adapter->fc_autoneg = true;
1895 hw->mac.autoneg = true;
1896 hw->phy.autoneg_advertised = 0x2f;
1897
0cce119a
AD		 1898 hw->fc.requested_mode = e1000_fc_default;
1899 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1900
9d5c8243
AK		 1901 igb_validate_mdi_setting(hw);
1902
9d5c8243
AK		 1903 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1904 * enable the ACPI Magic Packet filter
1905 */
1906
a2cf8b6c 1907 if (hw->bus.func == 0)
312c75ae 1908 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1909 else if (hw->mac.type == e1000_82580)
1910 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1911 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1912 &eeprom_data);
a2cf8b6c
AD		 1913 else if (hw->bus.func == 1)
1914 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1915
1916 if (eeprom_data & eeprom_apme_mask)
1917 adapter->eeprom_wol |= E1000_WUFC_MAG;
1918
1919 /* now that we have the eeprom settings, apply the special cases where
1920 * the eeprom may be wrong or the board simply won't support wake on
1921 * lan on a particular port */
1922 switch (pdev->device) {
1923 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1924 adapter->eeprom_wol = 0;
1925 break;
1926 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1927 case E1000_DEV_ID_82576_FIBER:
1928 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1929 /* Wake events only supported on port A for dual fiber
1930 * regardless of eeprom setting */
1931 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1932 adapter->eeprom_wol = 0;
1933 break;
c8ea5ea9 1934 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1935 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1936 /* if quad port adapter, disable WoL on all but port A */
1937 if (global_quad_port_a != 0)
1938 adapter->eeprom_wol = 0;
1939 else
1940 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1941 /* Reset for multiple quad port adapters */
1942 if (++global_quad_port_a == 4)
1943 global_quad_port_a = 0;
1944 break;
9d5c8243
AK		 1945 }
1946
1947 /* initialize the wol settings based on the eeprom settings */
1948 adapter->wol = adapter->eeprom_wol;
e1b86d84 1949 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1950
1951 /* reset the hardware with the new settings */
1952 igb_reset(adapter);
1953
1954 /* let the f/w know that the h/w is now under the control of the
1955 * driver. */
1956 igb_get_hw_control(adapter);
1957
9d5c8243
AK		 1958 strcpy(netdev->name, "eth%d");
1959 err = register_netdev(netdev);
1960 if (err)
1961 goto err_register;
1962
b168dfc5
JB		 1963 /* carrier off reporting is important to ethtool even BEFORE open */
1964 netif_carrier_off(netdev);
1965
421e02f0 1966#ifdef CONFIG_IGB_DCA
bbd98fe4 1967 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1968 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1969 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1970 igb_setup_dca(adapter);
1971 }
fe4506b6 1972
38c845c7 1973#endif
9d5c8243
AK		 1974 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1975 /* print bus type/speed/width info */
7c510e4b 1976 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1977 netdev->name,
559e9c49 1978 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1979 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1980 "unknown"),
59c3de89
AD		 1981 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1982 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1983 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1984 "unknown"),
7c510e4b 1985 netdev->dev_addr);
9d5c8243
AK		 1986
1987 igb_read_part_num(hw, &part_num);
1988 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1989 (part_num >> 8), (part_num & 0xff));
1990
1991 dev_info(&pdev->dev,
1992 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1993 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1994 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1995 adapter->num_rx_queues, adapter->num_tx_queues);
1996
9d5c8243
AK		 1997 return 0;
1998
1999err_register:
2000 igb_release_hw_control(adapter);
2001err_eeprom:
2002 if (!igb_check_reset_block(hw))
f5f4cf08 2003 igb_reset_phy(hw);
9d5c8243
AK		 2004
2005 if (hw->flash_address)
2006 iounmap(hw->flash_address);
9d5c8243 2007err_sw_init:
047e0030 2008 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2009 iounmap(hw->hw_addr);
2010err_ioremap:
2011 free_netdev(netdev);
2012err_alloc_etherdev:
559e9c49
AD		 2013 pci_release_selected_regions(pdev,
2014 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2015err_pci_reg:
2016err_dma:
2017 pci_disable_device(pdev);
2018 return err;
2019}
2020
2021/**
2022 * igb_remove - Device Removal Routine
2023 * @pdev: PCI device information struct
2024 *
2025 * igb_remove is called by the PCI subsystem to alert the driver
2026 * that it should release a PCI device. The could be caused by a
2027 * Hot-Plug event, or because the driver is going to be removed from
2028 * memory.
2029 **/
2030static void __devexit igb_remove(struct pci_dev *pdev)
2031{
2032 struct net_device *netdev = pci_get_drvdata(pdev);
2033 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2034 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2035
2036 /* flush_scheduled work may reschedule our watchdog task, so
2037 * explicitly disable watchdog tasks from being rescheduled */
2038 set_bit(__IGB_DOWN, &adapter->state);
2039 del_timer_sync(&adapter->watchdog_timer);
2040 del_timer_sync(&adapter->phy_info_timer);
2041
2042 flush_scheduled_work();
2043
421e02f0 2044#ifdef CONFIG_IGB_DCA
7dfc16fa 2045 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2046 dev_info(&pdev->dev, "DCA disabled\n");
2047 dca_remove_requester(&pdev->dev);
7dfc16fa 2048 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2049 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2050 }
2051#endif
2052
9d5c8243
AK		 2053 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2054 * would have already happened in close and is redundant. */
2055 igb_release_hw_control(adapter);
2056
2057 unregister_netdev(netdev);
2058
047e0030 2059 igb_clear_interrupt_scheme(adapter);
9d5c8243 2060
37680117
AD		 2061#ifdef CONFIG_PCI_IOV
2062 /* reclaim resources allocated to VFs */
2063 if (adapter->vf_data) {
2064 /* disable iov and allow time for transactions to clear */
2065 pci_disable_sriov(pdev);
2066 msleep(500);
2067
2068 kfree(adapter->vf_data);
2069 adapter->vf_data = NULL;
2070 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2071 msleep(100);
2072 dev_info(&pdev->dev, "IOV Disabled\n");
2073 }
2074#endif
559e9c49 2075
28b0759c
AD		 2076 iounmap(hw->hw_addr);
2077 if (hw->flash_address)
2078 iounmap(hw->flash_address);
559e9c49
AD		 2079 pci_release_selected_regions(pdev,
2080 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2081
2082 free_netdev(netdev);
2083
19d5afd4 2084 pci_disable_pcie_error_reporting(pdev);
40a914fa 2085
9d5c8243
AK		 2086 pci_disable_device(pdev);
2087}
2088
a6b623e0
AD		 2089/**
2090 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2091 * @adapter: board private structure to initialize
2092 *
2093 * This function initializes the vf specific data storage and then attempts to
2094 * allocate the VFs. The reason for ordering it this way is because it is much
2095 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2096 * the memory for the VFs.
2097 **/
2098static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2099{
2100#ifdef CONFIG_PCI_IOV
2101 struct pci_dev *pdev = adapter->pdev;
2102
a6b623e0
AD		 2103 if (adapter->vfs_allocated_count) {
2104 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2105 sizeof(struct vf_data_storage),
2106 GFP_KERNEL);
2107 /* if allocation failed then we do not support SR-IOV */
2108 if (!adapter->vf_data) {
2109 adapter->vfs_allocated_count = 0;
2110 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2111 "Data Storage\n");
2112 }
2113 }
2114
2115 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2116 kfree(adapter->vf_data);
2117 adapter->vf_data = NULL;
2118#endif /* CONFIG_PCI_IOV */
2119 adapter->vfs_allocated_count = 0;
2120#ifdef CONFIG_PCI_IOV
2121 } else {
2122 unsigned char mac_addr[ETH_ALEN];
2123 int i;
2124 dev_info(&pdev->dev, "%d vfs allocated\n",
2125 adapter->vfs_allocated_count);
2126 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2127 random_ether_addr(mac_addr);
2128 igb_set_vf_mac(adapter, i, mac_addr);
2129 }
2130 }
2131#endif /* CONFIG_PCI_IOV */
2132}
2133
115f459a
AD		 2134
2135/**
2136 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2137 * @adapter: board private structure to initialize
2138 *
2139 * igb_init_hw_timer initializes the function pointer and values for the hw
2140 * timer found in hardware.
2141 **/
2142static void igb_init_hw_timer(struct igb_adapter *adapter)
2143{
2144 struct e1000_hw *hw = &adapter->hw;
2145
2146 switch (hw->mac.type) {
d2ba2ed8 2147 case e1000_i350:
55cac248
AD		 2148 case e1000_82580:
2149 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2150 adapter->cycles.read = igb_read_clock;
2151 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2152 adapter->cycles.mult = 1;
2153 /*
2154 * The 82580 timesync updates the system timer every 8ns by 8ns
2155 * and the value cannot be shifted. Instead we need to shift
2156 * the registers to generate a 64bit timer value. As a result
2157 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2158 * 24 in order to generate a larger value for synchronization.
2159 */
2160 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2161 /* disable system timer temporarily by setting bit 31 */
2162 wr32(E1000_TSAUXC, 0x80000000);
2163 wrfl();
2164
2165 /* Set registers so that rollover occurs soon to test this. */
2166 wr32(E1000_SYSTIMR, 0x00000000);
2167 wr32(E1000_SYSTIML, 0x80000000);
2168 wr32(E1000_SYSTIMH, 0x000000FF);
2169 wrfl();
2170
2171 /* enable system timer by clearing bit 31 */
2172 wr32(E1000_TSAUXC, 0x0);
2173 wrfl();
2174
2175 timecounter_init(&adapter->clock,
2176 &adapter->cycles,
2177 ktime_to_ns(ktime_get_real()));
2178 /*
2179 * Synchronize our NIC clock against system wall clock. NIC
2180 * time stamp reading requires ~3us per sample, each sample
2181 * was pretty stable even under load => only require 10
2182 * samples for each offset comparison.
2183 */
2184 memset(&adapter->compare, 0, sizeof(adapter->compare));
2185 adapter->compare.source = &adapter->clock;
2186 adapter->compare.target = ktime_get_real;
2187 adapter->compare.num_samples = 10;
2188 timecompare_update(&adapter->compare, 0);
2189 break;
115f459a
AD		 2190 case e1000_82576:
2191 /*
2192 * Initialize hardware timer: we keep it running just in case
2193 * that some program needs it later on.
2194 */
2195 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2196 adapter->cycles.read = igb_read_clock;
2197 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2198 adapter->cycles.mult = 1;
2199 /**
2200 * Scale the NIC clock cycle by a large factor so that
2201 * relatively small clock corrections can be added or
2202 * substracted at each clock tick. The drawbacks of a large
2203 * factor are a) that the clock register overflows more quickly
2204 * (not such a big deal) and b) that the increment per tick has
2205 * to fit into 24 bits. As a result we need to use a shift of
2206 * 19 so we can fit a value of 16 into the TIMINCA register.
2207 */
2208 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2209 wr32(E1000_TIMINCA,
2210 (1 << E1000_TIMINCA_16NS_SHIFT) |
2211 (16 << IGB_82576_TSYNC_SHIFT));
2212
2213 /* Set registers so that rollover occurs soon to test this. */
2214 wr32(E1000_SYSTIML, 0x00000000);
2215 wr32(E1000_SYSTIMH, 0xFF800000);
2216 wrfl();
2217
2218 timecounter_init(&adapter->clock,
2219 &adapter->cycles,
2220 ktime_to_ns(ktime_get_real()));
2221 /*
2222 * Synchronize our NIC clock against system wall clock. NIC
2223 * time stamp reading requires ~3us per sample, each sample
2224 * was pretty stable even under load => only require 10
2225 * samples for each offset comparison.
2226 */
2227 memset(&adapter->compare, 0, sizeof(adapter->compare));
2228 adapter->compare.source = &adapter->clock;
2229 adapter->compare.target = ktime_get_real;
2230 adapter->compare.num_samples = 10;
2231 timecompare_update(&adapter->compare, 0);
2232 break;
2233 case e1000_82575:
2234 /* 82575 does not support timesync */
2235 default:
2236 break;
2237 }
2238
2239}
2240
9d5c8243
AK		 2241/**
2242 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2243 * @adapter: board private structure to initialize
2244 *
2245 * igb_sw_init initializes the Adapter private data structure.
2246 * Fields are initialized based on PCI device information and
2247 * OS network device settings (MTU size).
2248 **/
2249static int __devinit igb_sw_init(struct igb_adapter *adapter)
2250{
2251 struct e1000_hw *hw = &adapter->hw;
2252 struct net_device *netdev = adapter->netdev;
2253 struct pci_dev *pdev = adapter->pdev;
2254
2255 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2256
68fd9910
AD		 2257 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2258 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2259 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2260 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2261
9d5c8243
AK		 2262 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2263 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2264
a6b623e0
AD		 2265#ifdef CONFIG_PCI_IOV
2266 if (hw->mac.type == e1000_82576)
c0f2276f 2267 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
a6b623e0
AD		 2268
2269#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2270 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2271
2272 /*
2273 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2274 * then we should combine the queues into a queue pair in order to
2275 * conserve interrupts due to limited supply
2276 */
2277 if ((adapter->rss_queues > 4) ||
2278 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2279 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2280
a6b623e0 2281 /* This call may decrease the number of queues */
047e0030 2282 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2283 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2284 return -ENOMEM;
2285 }
2286
115f459a 2287 igb_init_hw_timer(adapter);
a6b623e0
AD		 2288 igb_probe_vfs(adapter);
2289
9d5c8243
AK		 2290 /* Explicitly disable IRQ since the NIC can be in any state. */
2291 igb_irq_disable(adapter);
2292
2293 set_bit(__IGB_DOWN, &adapter->state);
2294 return 0;
2295}
2296
2297/**
2298 * igb_open - Called when a network interface is made active
2299 * @netdev: network interface device structure
2300 *
2301 * Returns 0 on success, negative value on failure
2302 *
2303 * The open entry point is called when a network interface is made
2304 * active by the system (IFF_UP). At this point all resources needed
2305 * for transmit and receive operations are allocated, the interrupt
2306 * handler is registered with the OS, the watchdog timer is started,
2307 * and the stack is notified that the interface is ready.
2308 **/
2309static int igb_open(struct net_device *netdev)
2310{
2311 struct igb_adapter *adapter = netdev_priv(netdev);
2312 struct e1000_hw *hw = &adapter->hw;
2313 int err;
2314 int i;
2315
2316 /* disallow open during test */
2317 if (test_bit(__IGB_TESTING, &adapter->state))
2318 return -EBUSY;
2319
b168dfc5
JB		 2320 netif_carrier_off(netdev);
2321
9d5c8243
AK		 2322 /* allocate transmit descriptors */
2323 err = igb_setup_all_tx_resources(adapter);
2324 if (err)
2325 goto err_setup_tx;
2326
2327 /* allocate receive descriptors */
2328 err = igb_setup_all_rx_resources(adapter);
2329 if (err)
2330 goto err_setup_rx;
2331
88a268c1 2332 igb_power_up_link(adapter);
9d5c8243 2333
9d5c8243
AK		 2334 /* before we allocate an interrupt, we must be ready to handle it.
2335 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2336 * as soon as we call pci_request_irq, so we have to setup our
2337 * clean_rx handler before we do so. */
2338 igb_configure(adapter);
2339
2340 err = igb_request_irq(adapter);
2341 if (err)
2342 goto err_req_irq;
2343
2344 /* From here on the code is the same as igb_up() */
2345 clear_bit(__IGB_DOWN, &adapter->state);
2346
047e0030
AD		 2347 for (i = 0; i < adapter->num_q_vectors; i++) {
2348 struct igb_q_vector *q_vector = adapter->q_vector[i];
2349 napi_enable(&q_vector->napi);
2350 }
9d5c8243
AK		 2351
2352 /* Clear any pending interrupts. */
2353 rd32(E1000_ICR);
844290e5
PW		 2354
2355 igb_irq_enable(adapter);
2356
d4960307
AD		 2357 /* notify VFs that reset has been completed */
2358 if (adapter->vfs_allocated_count) {
2359 u32 reg_data = rd32(E1000_CTRL_EXT);
2360 reg_data |= E1000_CTRL_EXT_PFRSTD;
2361 wr32(E1000_CTRL_EXT, reg_data);
2362 }
2363
d55b53ff
JK		 2364 netif_tx_start_all_queues(netdev);
2365
25568a53
AD		 2366 /* start the watchdog. */
2367 hw->mac.get_link_status = 1;
2368 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2369
2370 return 0;
2371
2372err_req_irq:
2373 igb_release_hw_control(adapter);
88a268c1 2374 igb_power_down_link(adapter);
9d5c8243
AK		 2375 igb_free_all_rx_resources(adapter);
2376err_setup_rx:
2377 igb_free_all_tx_resources(adapter);
2378err_setup_tx:
2379 igb_reset(adapter);
2380
2381 return err;
2382}
2383
2384/**
2385 * igb_close - Disables a network interface
2386 * @netdev: network interface device structure
2387 *
2388 * Returns 0, this is not allowed to fail
2389 *
2390 * The close entry point is called when an interface is de-activated
2391 * by the OS. The hardware is still under the driver's control, but
2392 * needs to be disabled. A global MAC reset is issued to stop the
2393 * hardware, and all transmit and receive resources are freed.
2394 **/
2395static int igb_close(struct net_device *netdev)
2396{
2397 struct igb_adapter *adapter = netdev_priv(netdev);
2398
2399 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2400 igb_down(adapter);
2401
2402 igb_free_irq(adapter);
2403
2404 igb_free_all_tx_resources(adapter);
2405 igb_free_all_rx_resources(adapter);
2406
9d5c8243
AK		 2407 return 0;
2408}
2409
2410/**
2411 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2412 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2413 *
2414 * Return 0 on success, negative on failure
2415 **/
80785298 2416int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2417{
59d71989 2418 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2419 int size;
2420
2421 size = sizeof(struct igb_buffer) * tx_ring->count;
2422 tx_ring->buffer_info = vmalloc(size);
2423 if (!tx_ring->buffer_info)
2424 goto err;
2425 memset(tx_ring->buffer_info, 0, size);
2426
2427 /* round up to nearest 4K */
85e8d004 2428 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2429 tx_ring->size = ALIGN(tx_ring->size, 4096);
2430
59d71989
AD		 2431 tx_ring->desc = dma_alloc_coherent(dev,
2432 tx_ring->size,
2433 &tx_ring->dma,
2434 GFP_KERNEL);
9d5c8243
AK		 2435
2436 if (!tx_ring->desc)
2437 goto err;
2438
9d5c8243
AK		 2439 tx_ring->next_to_use = 0;
2440 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2441 return 0;
2442
2443err:
2444 vfree(tx_ring->buffer_info);
59d71989 2445 dev_err(dev,
9d5c8243
AK		 2446 "Unable to allocate memory for the transmit descriptor ring\n");
2447 return -ENOMEM;
2448}
2449
2450/**
2451 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2452 * (Descriptors) for all queues
2453 * @adapter: board private structure
2454 *
2455 * Return 0 on success, negative on failure
2456 **/
2457static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2458{
439705e1 2459 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2460 int i, err = 0;
2461
2462 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2463 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2464 if (err) {
439705e1 2465 dev_err(&pdev->dev,
9d5c8243
AK		 2466 "Allocation for Tx Queue %u failed\n", i);
2467 for (i--; i >= 0; i--)
3025a446 2468 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2469 break;
2470 }
2471 }
2472
a99955fc 2473 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2474 int r_idx = i % adapter->num_tx_queues;
3025a446 2475 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2476 }
9d5c8243
AK		 2477 return err;
2478}
2479
2480/**
85b430b4
AD		 2481 * igb_setup_tctl - configure the transmit control registers
2482 * @adapter: Board private structure
9d5c8243 2483 **/
d7ee5b3a 2484void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2485{
9d5c8243
AK		 2486 struct e1000_hw *hw = &adapter->hw;
2487 u32 tctl;
9d5c8243 2488
85b430b4
AD		 2489 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2490 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2491
2492 /* Program the Transmit Control Register */
9d5c8243
AK		 2493 tctl = rd32(E1000_TCTL);
2494 tctl &= ~E1000_TCTL_CT;
2495 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2496 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2497
2498 igb_config_collision_dist(hw);
2499
9d5c8243
AK		 2500 /* Enable transmits */
2501 tctl |= E1000_TCTL_EN;
2502
2503 wr32(E1000_TCTL, tctl);
2504}
2505
85b430b4
AD		 2506/**
2507 * igb_configure_tx_ring - Configure transmit ring after Reset
2508 * @adapter: board private structure
2509 * @ring: tx ring to configure
2510 *
2511 * Configure a transmit ring after a reset.
2512 **/
d7ee5b3a
AD		 2513void igb_configure_tx_ring(struct igb_adapter *adapter,
2514 struct igb_ring *ring)
85b430b4
AD		 2515{
2516 struct e1000_hw *hw = &adapter->hw;
2517 u32 txdctl;
2518 u64 tdba = ring->dma;
2519 int reg_idx = ring->reg_idx;
2520
2521 /* disable the queue */
2522 txdctl = rd32(E1000_TXDCTL(reg_idx));
2523 wr32(E1000_TXDCTL(reg_idx),
2524 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2525 wrfl();
2526 mdelay(10);
2527
2528 wr32(E1000_TDLEN(reg_idx),
2529 ring->count * sizeof(union e1000_adv_tx_desc));
2530 wr32(E1000_TDBAL(reg_idx),
2531 tdba & 0x00000000ffffffffULL);
2532 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2533
fce99e34
AD		 2534 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2535 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2536 writel(0, ring->head);
2537 writel(0, ring->tail);
85b430b4
AD		 2538
2539 txdctl |= IGB_TX_PTHRESH;
2540 txdctl |= IGB_TX_HTHRESH << 8;
2541 txdctl |= IGB_TX_WTHRESH << 16;
2542
2543 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2544 wr32(E1000_TXDCTL(reg_idx), txdctl);
2545}
2546
2547/**
2548 * igb_configure_tx - Configure transmit Unit after Reset
2549 * @adapter: board private structure
2550 *
2551 * Configure the Tx unit of the MAC after a reset.
2552 **/
2553static void igb_configure_tx(struct igb_adapter *adapter)
2554{
2555 int i;
2556
2557 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2558 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2559}
2560
9d5c8243
AK		 2561/**
2562 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2563 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2564 *
2565 * Returns 0 on success, negative on failure
2566 **/
80785298 2567int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2568{
59d71989 2569 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2570 int size, desc_len;
2571
2572 size = sizeof(struct igb_buffer) * rx_ring->count;
2573 rx_ring->buffer_info = vmalloc(size);
2574 if (!rx_ring->buffer_info)
2575 goto err;
2576 memset(rx_ring->buffer_info, 0, size);
2577
2578 desc_len = sizeof(union e1000_adv_rx_desc);
2579
2580 /* Round up to nearest 4K */
2581 rx_ring->size = rx_ring->count * desc_len;
2582 rx_ring->size = ALIGN(rx_ring->size, 4096);
2583
59d71989
AD		 2584 rx_ring->desc = dma_alloc_coherent(dev,
2585 rx_ring->size,
2586 &rx_ring->dma,
2587 GFP_KERNEL);
9d5c8243
AK		 2588
2589 if (!rx_ring->desc)
2590 goto err;
2591
2592 rx_ring->next_to_clean = 0;
2593 rx_ring->next_to_use = 0;
9d5c8243 2594
9d5c8243
AK		 2595 return 0;
2596
2597err:
2598 vfree(rx_ring->buffer_info);
439705e1 2599 rx_ring->buffer_info = NULL;
59d71989
AD		 2600 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2601 " ring\n");
9d5c8243
AK		 2602 return -ENOMEM;
2603}
2604
2605/**
2606 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2607 * (Descriptors) for all queues
2608 * @adapter: board private structure
2609 *
2610 * Return 0 on success, negative on failure
2611 **/
2612static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2613{
439705e1 2614 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2615 int i, err = 0;
2616
2617 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2618 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2619 if (err) {
439705e1 2620 dev_err(&pdev->dev,
9d5c8243
AK		 2621 "Allocation for Rx Queue %u failed\n", i);
2622 for (i--; i >= 0; i--)
3025a446 2623 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2624 break;
2625 }
2626 }
2627
2628 return err;
2629}
2630
06cf2666
AD		 2631/**
2632 * igb_setup_mrqc - configure the multiple receive queue control registers
2633 * @adapter: Board private structure
2634 **/
2635static void igb_setup_mrqc(struct igb_adapter *adapter)
2636{
2637 struct e1000_hw *hw = &adapter->hw;
2638 u32 mrqc, rxcsum;
2639 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2640 union e1000_reta {
2641 u32 dword;
2642 u8 bytes[4];
2643 } reta;
2644 static const u8 rsshash[40] = {
2645 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2646 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2647 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2648 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2649
2650 /* Fill out hash function seeds */
2651 for (j = 0; j < 10; j++) {
2652 u32 rsskey = rsshash[(j * 4)];
2653 rsskey |= rsshash[(j * 4) + 1] << 8;
2654 rsskey |= rsshash[(j * 4) + 2] << 16;
2655 rsskey |= rsshash[(j * 4) + 3] << 24;
2656 array_wr32(E1000_RSSRK(0), j, rsskey);
2657 }
2658
a99955fc 2659 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2660
2661 if (adapter->vfs_allocated_count) {
2662 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2663 switch (hw->mac.type) {
d2ba2ed8 2664 case e1000_i350:
55cac248
AD		 2665 case e1000_82580:
2666 num_rx_queues = 1;
2667 shift = 0;
2668 break;
06cf2666
AD		 2669 case e1000_82576:
2670 shift = 3;
2671 num_rx_queues = 2;
2672 break;
2673 case e1000_82575:
2674 shift = 2;
2675 shift2 = 6;
2676 default:
2677 break;
2678 }
2679 } else {
2680 if (hw->mac.type == e1000_82575)
2681 shift = 6;
2682 }
2683
2684 for (j = 0; j < (32 * 4); j++) {
2685 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2686 if (shift2)
2687 reta.bytes[j & 3] |= num_rx_queues << shift2;
2688 if ((j & 3) == 3)
2689 wr32(E1000_RETA(j >> 2), reta.dword);
2690 }
2691
2692 /*
2693 * Disable raw packet checksumming so that RSS hash is placed in
2694 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2695 * offloads as they are enabled by default
2696 */
2697 rxcsum = rd32(E1000_RXCSUM);
2698 rxcsum |= E1000_RXCSUM_PCSD;
2699
2700 if (adapter->hw.mac.type >= e1000_82576)
2701 /* Enable Receive Checksum Offload for SCTP */
2702 rxcsum |= E1000_RXCSUM_CRCOFL;
2703
2704 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2705 wr32(E1000_RXCSUM, rxcsum);
2706
2707 /* If VMDq is enabled then we set the appropriate mode for that, else
2708 * we default to RSS so that an RSS hash is calculated per packet even
2709 * if we are only using one queue */
2710 if (adapter->vfs_allocated_count) {
2711 if (hw->mac.type > e1000_82575) {
2712 /* Set the default pool for the PF's first queue */
2713 u32 vtctl = rd32(E1000_VT_CTL);
2714 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2715 E1000_VT_CTL_DISABLE_DEF_POOL);
2716 vtctl |= adapter->vfs_allocated_count <<
2717 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2718 wr32(E1000_VT_CTL, vtctl);
2719 }
a99955fc 2720 if (adapter->rss_queues > 1)
06cf2666
AD		 2721 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2722 else
2723 mrqc = E1000_MRQC_ENABLE_VMDQ;
2724 } else {
2725 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2726 }
2727 igb_vmm_control(adapter);
2728
4478a9cd
AD		 2729 /*
2730 * Generate RSS hash based on TCP port numbers and/or
2731 * IPv4/v6 src and dst addresses since UDP cannot be
2732 * hashed reliably due to IP fragmentation
2733 */
2734 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2735 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2736 E1000_MRQC_RSS_FIELD_IPV6 |
2737 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2738 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2739
2740 wr32(E1000_MRQC, mrqc);
2741}
2742
9d5c8243
AK		 2743/**
2744 * igb_setup_rctl - configure the receive control registers
2745 * @adapter: Board private structure
2746 **/
d7ee5b3a 2747void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2748{
2749 struct e1000_hw *hw = &adapter->hw;
2750 u32 rctl;
9d5c8243
AK		 2751
2752 rctl = rd32(E1000_RCTL);
2753
2754 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2755 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2756
69d728ba 2757 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2758 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2759
87cb7e8c
AK		 2760 /*
2761 * enable stripping of CRC. It's unlikely this will break BMC
2762 * redirection as it did with e1000. Newer features require
2763 * that the HW strips the CRC.
73cd78f1 2764 */
87cb7e8c 2765 rctl |= E1000_RCTL_SECRC;
9d5c8243 2766
559e9c49 2767 /* disable store bad packets and clear size bits. */
ec54d7d6 2768 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2769
6ec43fe6
AD		 2770 /* enable LPE to prevent packets larger than max_frame_size */
2771 rctl |= E1000_RCTL_LPE;
9d5c8243 2772
952f72a8
AD		 2773 /* disable queue 0 to prevent tail write w/o re-config */
2774 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2775
e1739522
AD		 2776 /* Attention!!! For SR-IOV PF driver operations you must enable
2777 * queue drop for all VF and PF queues to prevent head of line blocking
2778 * if an un-trusted VF does not provide descriptors to hardware.
2779 */
2780 if (adapter->vfs_allocated_count) {
e1739522
AD		 2781 /* set all queue drop enable bits */
2782 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2783 }
2784
9d5c8243
AK		 2785 wr32(E1000_RCTL, rctl);
2786}
2787
7d5753f0
AD		 2788static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2789 int vfn)
2790{
2791 struct e1000_hw *hw = &adapter->hw;
2792 u32 vmolr;
2793
2794 /* if it isn't the PF check to see if VFs are enabled and
2795 * increase the size to support vlan tags */
2796 if (vfn < adapter->vfs_allocated_count &&
2797 adapter->vf_data[vfn].vlans_enabled)
2798 size += VLAN_TAG_SIZE;
2799
2800 vmolr = rd32(E1000_VMOLR(vfn));
2801 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2802 vmolr |= size | E1000_VMOLR_LPE;
2803 wr32(E1000_VMOLR(vfn), vmolr);
2804
2805 return 0;
2806}
2807
e1739522
AD		 2808/**
2809 * igb_rlpml_set - set maximum receive packet size
2810 * @adapter: board private structure
2811 *
2812 * Configure maximum receivable packet size.
2813 **/
2814static void igb_rlpml_set(struct igb_adapter *adapter)
2815{
2816 u32 max_frame_size = adapter->max_frame_size;
2817 struct e1000_hw *hw = &adapter->hw;
2818 u16 pf_id = adapter->vfs_allocated_count;
2819
2820 if (adapter->vlgrp)
2821 max_frame_size += VLAN_TAG_SIZE;
2822
2823 /* if vfs are enabled we set RLPML to the largest possible request
2824 * size and set the VMOLR RLPML to the size we need */
2825 if (pf_id) {
2826 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2827 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2828 }
2829
2830 wr32(E1000_RLPML, max_frame_size);
2831}
2832
8151d294
WM		 2833static inline void igb_set_vmolr(struct igb_adapter *adapter,
2834 int vfn, bool aupe)
7d5753f0
AD		 2835{
2836 struct e1000_hw *hw = &adapter->hw;
2837 u32 vmolr;
2838
2839 /*
2840 * This register exists only on 82576 and newer so if we are older then
2841 * we should exit and do nothing
2842 */
2843 if (hw->mac.type < e1000_82576)
2844 return;
2845
2846 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2847 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2848 if (aupe)
2849 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2850 else
2851 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2852
2853 /* clear all bits that might not be set */
2854 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2855
a99955fc 2856 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2857 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2858 /*
2859 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2860 * multicast packets
2861 */
2862 if (vfn <= adapter->vfs_allocated_count)
2863 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2864
2865 wr32(E1000_VMOLR(vfn), vmolr);
2866}
2867
85b430b4
AD		 2868/**
2869 * igb_configure_rx_ring - Configure a receive ring after Reset
2870 * @adapter: board private structure
2871 * @ring: receive ring to be configured
2872 *
2873 * Configure the Rx unit of the MAC after a reset.
2874 **/
d7ee5b3a
AD		 2875void igb_configure_rx_ring(struct igb_adapter *adapter,
2876 struct igb_ring *ring)
85b430b4
AD		 2877{
2878 struct e1000_hw *hw = &adapter->hw;
2879 u64 rdba = ring->dma;
2880 int reg_idx = ring->reg_idx;
952f72a8 2881 u32 srrctl, rxdctl;
85b430b4
AD		 2882
2883 /* disable the queue */
2884 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2885 wr32(E1000_RXDCTL(reg_idx),
2886 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2887
2888 /* Set DMA base address registers */
2889 wr32(E1000_RDBAL(reg_idx),
2890 rdba & 0x00000000ffffffffULL);
2891 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2892 wr32(E1000_RDLEN(reg_idx),
2893 ring->count * sizeof(union e1000_adv_rx_desc));
2894
2895 /* initialize head and tail */
fce99e34
AD		 2896 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2897 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2898 writel(0, ring->head);
2899 writel(0, ring->tail);
85b430b4 2900
952f72a8 2901 /* set descriptor configuration */
4c844851
AD		 2902 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2903 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2904 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2905#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2906 srrctl |= IGB_RXBUFFER_16384 >>
2907 E1000_SRRCTL_BSIZEPKT_SHIFT;
2908#else
2909 srrctl |= (PAGE_SIZE / 2) >>
2910 E1000_SRRCTL_BSIZEPKT_SHIFT;
2911#endif
2912 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2913 } else {
4c844851 2914 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2915 E1000_SRRCTL_BSIZEPKT_SHIFT;
2916 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2917 }
757b77e2
NN		 2918 if (hw->mac.type == e1000_82580)
2919 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2920 /* Only set Drop Enable if we are supporting multiple queues */
2921 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2922 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2923
2924 wr32(E1000_SRRCTL(reg_idx), srrctl);
2925
7d5753f0 2926 /* set filtering for VMDQ pools */
8151d294 2927 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2928
85b430b4
AD		 2929 /* enable receive descriptor fetching */
2930 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2931 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2932 rxdctl &= 0xFFF00000;
2933 rxdctl |= IGB_RX_PTHRESH;
2934 rxdctl |= IGB_RX_HTHRESH << 8;
2935 rxdctl |= IGB_RX_WTHRESH << 16;
2936 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2937}
2938
9d5c8243
AK		 2939/**
2940 * igb_configure_rx - Configure receive Unit after Reset
2941 * @adapter: board private structure
2942 *
2943 * Configure the Rx unit of the MAC after a reset.
2944 **/
2945static void igb_configure_rx(struct igb_adapter *adapter)
2946{
9107584e 2947 int i;
9d5c8243 2948
68d480c4
AD		 2949 /* set UTA to appropriate mode */
2950 igb_set_uta(adapter);
2951
26ad9178
AD		 2952 /* set the correct pool for the PF default MAC address in entry 0 */
2953 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2954 adapter->vfs_allocated_count);
2955
06cf2666
AD		 2956 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2957 * the Base and Length of the Rx Descriptor Ring */
2958 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2959 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2960}
2961
2962/**
2963 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2964 * @tx_ring: Tx descriptor ring for a specific queue
2965 *
2966 * Free all transmit software resources
2967 **/
68fd9910 2968void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2969{
3b644cf6 2970 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2971
2972 vfree(tx_ring->buffer_info);
2973 tx_ring->buffer_info = NULL;
2974
439705e1
AD		 2975 /* if not set, then don't free */
2976 if (!tx_ring->desc)
2977 return;
2978
59d71989
AD		 2979 dma_free_coherent(tx_ring->dev, tx_ring->size,
2980 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2981
2982 tx_ring->desc = NULL;
2983}
2984
2985/**
2986 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2987 * @adapter: board private structure
2988 *
2989 * Free all transmit software resources
2990 **/
2991static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2992{
2993 int i;
2994
2995 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2996 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2997}
2998
b1a436c3
AD		 2999void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3000 struct igb_buffer *buffer_info)
9d5c8243 3001{
6366ad33
AD		 3002 if (buffer_info->dma) {
3003 if (buffer_info->mapped_as_page)
59d71989 3004 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 3005 buffer_info->dma,
3006 buffer_info->length,
59d71989 3007 DMA_TO_DEVICE);
6366ad33 3008 else
59d71989 3009 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3010 buffer_info->dma,
3011 buffer_info->length,
59d71989 3012 DMA_TO_DEVICE);
6366ad33
AD		 3013 buffer_info->dma = 0;
3014 }
9d5c8243
AK		 3015 if (buffer_info->skb) {
3016 dev_kfree_skb_any(buffer_info->skb);
3017 buffer_info->skb = NULL;
3018 }
3019 buffer_info->time_stamp = 0;
6366ad33
AD		 3020 buffer_info->length = 0;
3021 buffer_info->next_to_watch = 0;
3022 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3023}
3024
3025/**
3026 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3027 * @tx_ring: ring to be cleaned
3028 **/
3b644cf6 3029static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3030{
3031 struct igb_buffer *buffer_info;
3032 unsigned long size;
3033 unsigned int i;
3034
3035 if (!tx_ring->buffer_info)
3036 return;
3037 /* Free all the Tx ring sk_buffs */
3038
3039 for (i = 0; i < tx_ring->count; i++) {
3040 buffer_info = &tx_ring->buffer_info[i];
80785298 3041 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3042 }
3043
3044 size = sizeof(struct igb_buffer) * tx_ring->count;
3045 memset(tx_ring->buffer_info, 0, size);
3046
3047 /* Zero out the descriptor ring */
9d5c8243
AK		 3048 memset(tx_ring->desc, 0, tx_ring->size);
3049
3050 tx_ring->next_to_use = 0;
3051 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3052}
3053
3054/**
3055 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3056 * @adapter: board private structure
3057 **/
3058static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3059{
3060 int i;
3061
3062 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3063 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3064}
3065
3066/**
3067 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3068 * @rx_ring: ring to clean the resources from
3069 *
3070 * Free all receive software resources
3071 **/
68fd9910 3072void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3073{
3b644cf6 3074 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3075
3076 vfree(rx_ring->buffer_info);
3077 rx_ring->buffer_info = NULL;
3078
439705e1
AD		 3079 /* if not set, then don't free */
3080 if (!rx_ring->desc)
3081 return;
3082
59d71989
AD		 3083 dma_free_coherent(rx_ring->dev, rx_ring->size,
3084 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3085
3086 rx_ring->desc = NULL;
3087}
3088
3089/**
3090 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3091 * @adapter: board private structure
3092 *
3093 * Free all receive software resources
3094 **/
3095static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3096{
3097 int i;
3098
3099 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3100 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3101}
3102
3103/**
3104 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3105 * @rx_ring: ring to free buffers from
3106 **/
3b644cf6 3107static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3108{
3109 struct igb_buffer *buffer_info;
9d5c8243
AK		 3110 unsigned long size;
3111 unsigned int i;
3112
3113 if (!rx_ring->buffer_info)
3114 return;
439705e1 3115
9d5c8243
AK		 3116 /* Free all the Rx ring sk_buffs */
3117 for (i = 0; i < rx_ring->count; i++) {
3118 buffer_info = &rx_ring->buffer_info[i];
3119 if (buffer_info->dma) {
59d71989 3120 dma_unmap_single(rx_ring->dev,
80785298 3121 buffer_info->dma,
4c844851 3122 rx_ring->rx_buffer_len,
59d71989 3123 DMA_FROM_DEVICE);
9d5c8243
AK		 3124 buffer_info->dma = 0;
3125 }
3126
3127 if (buffer_info->skb) {
3128 dev_kfree_skb(buffer_info->skb);
3129 buffer_info->skb = NULL;
3130 }
6ec43fe6 3131 if (buffer_info->page_dma) {
59d71989 3132 dma_unmap_page(rx_ring->dev,
80785298 3133 buffer_info->page_dma,
6ec43fe6 3134 PAGE_SIZE / 2,
59d71989 3135 DMA_FROM_DEVICE);
6ec43fe6
AD		 3136 buffer_info->page_dma = 0;
3137 }
9d5c8243 3138 if (buffer_info->page) {
9d5c8243
AK		 3139 put_page(buffer_info->page);
3140 buffer_info->page = NULL;
bf36c1a0 3141 buffer_info->page_offset = 0;
9d5c8243
AK		 3142 }
3143 }
3144
9d5c8243
AK		 3145 size = sizeof(struct igb_buffer) * rx_ring->count;
3146 memset(rx_ring->buffer_info, 0, size);
3147
3148 /* Zero out the descriptor ring */
3149 memset(rx_ring->desc, 0, rx_ring->size);
3150
3151 rx_ring->next_to_clean = 0;
3152 rx_ring->next_to_use = 0;
9d5c8243
AK		 3153}
3154
3155/**
3156 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3157 * @adapter: board private structure
3158 **/
3159static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3160{
3161 int i;
3162
3163 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3164 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3165}
3166
3167/**
3168 * igb_set_mac - Change the Ethernet Address of the NIC
3169 * @netdev: network interface device structure
3170 * @p: pointer to an address structure
3171 *
3172 * Returns 0 on success, negative on failure
3173 **/
3174static int igb_set_mac(struct net_device *netdev, void *p)
3175{
3176 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3177 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3178 struct sockaddr *addr = p;
3179
3180 if (!is_valid_ether_addr(addr->sa_data))
3181 return -EADDRNOTAVAIL;
3182
3183 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3184 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3185
26ad9178
AD		 3186 /* set the correct pool for the new PF MAC address in entry 0 */
3187 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3188 adapter->vfs_allocated_count);
e1739522 3189
9d5c8243
AK		 3190 return 0;
3191}
3192
3193/**
68d480c4 3194 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3195 * @netdev: network interface device structure
3196 *
68d480c4
AD		 3197 * Writes multicast address list to the MTA hash table.
3198 * Returns: -ENOMEM on failure
3199 * 0 on no addresses written
3200 * X on writing X addresses to MTA
9d5c8243 3201 **/
68d480c4 3202static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3203{
3204 struct igb_adapter *adapter = netdev_priv(netdev);
3205 struct e1000_hw *hw = &adapter->hw;
22bedad3 3206 struct netdev_hw_addr *ha;
68d480c4 3207 u8 *mta_list;
9d5c8243
AK		 3208 int i;
3209
4cd24eaf 3210 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3211 /* nothing to program, so clear mc list */
3212 igb_update_mc_addr_list(hw, NULL, 0);
3213 igb_restore_vf_multicasts(adapter);
3214 return 0;
3215 }
9d5c8243 3216
4cd24eaf 3217 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3218 if (!mta_list)
3219 return -ENOMEM;
ff41f8dc 3220
68d480c4 3221 /* The shared function expects a packed array of only addresses. */
48e2f183 3222 i = 0;
22bedad3
JP		 3223 netdev_for_each_mc_addr(ha, netdev)
3224 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3225
68d480c4
AD		 3226 igb_update_mc_addr_list(hw, mta_list, i);
3227 kfree(mta_list);
3228
4cd24eaf 3229 return netdev_mc_count(netdev);
68d480c4
AD		 3230}
3231
3232/**
3233 * igb_write_uc_addr_list - write unicast addresses to RAR table
3234 * @netdev: network interface device structure
3235 *
3236 * Writes unicast address list to the RAR table.
3237 * Returns: -ENOMEM on failure/insufficient address space
3238 * 0 on no addresses written
3239 * X on writing X addresses to the RAR table
3240 **/
3241static int igb_write_uc_addr_list(struct net_device *netdev)
3242{
3243 struct igb_adapter *adapter = netdev_priv(netdev);
3244 struct e1000_hw *hw = &adapter->hw;
3245 unsigned int vfn = adapter->vfs_allocated_count;
3246 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3247 int count = 0;
3248
3249 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3250 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3251 return -ENOMEM;
9d5c8243 3252
32e7bfc4 3253 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3254 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3255
3256 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3257 if (!rar_entries)
3258 break;
26ad9178
AD		 3259 igb_rar_set_qsel(adapter, ha->addr,
3260 rar_entries--,
68d480c4
AD		 3261 vfn);
3262 count++;
ff41f8dc
AD		 3263 }
3264 }
3265 /* write the addresses in reverse order to avoid write combining */
3266 for (; rar_entries > 0 ; rar_entries--) {
3267 wr32(E1000_RAH(rar_entries), 0);
3268 wr32(E1000_RAL(rar_entries), 0);
3269 }
3270 wrfl();
3271
68d480c4
AD		 3272 return count;
3273}
3274
3275/**
3276 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3277 * @netdev: network interface device structure
3278 *
3279 * The set_rx_mode entry point is called whenever the unicast or multicast
3280 * address lists or the network interface flags are updated. This routine is
3281 * responsible for configuring the hardware for proper unicast, multicast,
3282 * promiscuous mode, and all-multi behavior.
3283 **/
3284static void igb_set_rx_mode(struct net_device *netdev)
3285{
3286 struct igb_adapter *adapter = netdev_priv(netdev);
3287 struct e1000_hw *hw = &adapter->hw;
3288 unsigned int vfn = adapter->vfs_allocated_count;
3289 u32 rctl, vmolr = 0;
3290 int count;
3291
3292 /* Check for Promiscuous and All Multicast modes */
3293 rctl = rd32(E1000_RCTL);
3294
3295 /* clear the effected bits */
3296 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3297
3298 if (netdev->flags & IFF_PROMISC) {
3299 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3300 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3301 } else {
3302 if (netdev->flags & IFF_ALLMULTI) {
3303 rctl |= E1000_RCTL_MPE;
3304 vmolr |= E1000_VMOLR_MPME;
3305 } else {
3306 /*
3307 * Write addresses to the MTA, if the attempt fails
3308 * then we should just turn on promiscous mode so
3309 * that we can at least receive multicast traffic
3310 */
3311 count = igb_write_mc_addr_list(netdev);
3312 if (count < 0) {
3313 rctl |= E1000_RCTL_MPE;
3314 vmolr |= E1000_VMOLR_MPME;
3315 } else if (count) {
3316 vmolr |= E1000_VMOLR_ROMPE;
3317 }
3318 }
3319 /*
3320 * Write addresses to available RAR registers, if there is not
3321 * sufficient space to store all the addresses then enable
3322 * unicast promiscous mode
3323 */
3324 count = igb_write_uc_addr_list(netdev);
3325 if (count < 0) {
3326 rctl |= E1000_RCTL_UPE;
3327 vmolr |= E1000_VMOLR_ROPE;
3328 }
3329 rctl |= E1000_RCTL_VFE;
28fc06f5 3330 }
68d480c4 3331 wr32(E1000_RCTL, rctl);
28fc06f5 3332
68d480c4
AD		 3333 /*
3334 * In order to support SR-IOV and eventually VMDq it is necessary to set
3335 * the VMOLR to enable the appropriate modes. Without this workaround
3336 * we will have issues with VLAN tag stripping not being done for frames
3337 * that are only arriving because we are the default pool
3338 */
3339 if (hw->mac.type < e1000_82576)
28fc06f5 3340 return;
9d5c8243 3341
68d480c4
AD		 3342 vmolr |= rd32(E1000_VMOLR(vfn)) &
3343 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3344 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3345 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3346}
3347
3348/* Need to wait a few seconds after link up to get diagnostic information from
3349 * the phy */
3350static void igb_update_phy_info(unsigned long data)
3351{
3352 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3353 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3354}
3355
4d6b725e
AD		 3356/**
3357 * igb_has_link - check shared code for link and determine up/down
3358 * @adapter: pointer to driver private info
3359 **/
3145535a 3360bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3361{
3362 struct e1000_hw *hw = &adapter->hw;
3363 bool link_active = false;
3364 s32 ret_val = 0;
3365
3366 /* get_link_status is set on LSC (link status) interrupt or
3367 * rx sequence error interrupt. get_link_status will stay
3368 * false until the e1000_check_for_link establishes link
3369 * for copper adapters ONLY
3370 */
3371 switch (hw->phy.media_type) {
3372 case e1000_media_type_copper:
3373 if (hw->mac.get_link_status) {
3374 ret_val = hw->mac.ops.check_for_link(hw);
3375 link_active = !hw->mac.get_link_status;
3376 } else {
3377 link_active = true;
3378 }
3379 break;
4d6b725e
AD		 3380 case e1000_media_type_internal_serdes:
3381 ret_val = hw->mac.ops.check_for_link(hw);
3382 link_active = hw->mac.serdes_has_link;
3383 break;
3384 default:
3385 case e1000_media_type_unknown:
3386 break;
3387 }
3388
3389 return link_active;
3390}
3391
9d5c8243
AK		 3392/**
3393 * igb_watchdog - Timer Call-back
3394 * @data: pointer to adapter cast into an unsigned long
3395 **/
3396static void igb_watchdog(unsigned long data)
3397{
3398 struct igb_adapter *adapter = (struct igb_adapter *)data;
3399 /* Do the rest outside of interrupt context */
3400 schedule_work(&adapter->watchdog_task);
3401}
3402
3403static void igb_watchdog_task(struct work_struct *work)
3404{
3405 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3406 struct igb_adapter,
3407 watchdog_task);
9d5c8243 3408 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3409 struct net_device *netdev = adapter->netdev;
9d5c8243 3410 u32 link;
7a6ea550 3411 int i;
9d5c8243 3412
4d6b725e 3413 link = igb_has_link(adapter);
9d5c8243
AK		 3414 if (link) {
3415 if (!netif_carrier_ok(netdev)) {
3416 u32 ctrl;
330a6d6a
AD		 3417 hw->mac.ops.get_speed_and_duplex(hw,
3418 &adapter->link_speed,
3419 &adapter->link_duplex);
9d5c8243
AK		 3420
3421 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3422 /* Links status message must follow this format */
3423 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3424 "Flow Control: %s\n",
559e9c49
AD		 3425 netdev->name,
3426 adapter->link_speed,
3427 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3428 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3429 ((ctrl & E1000_CTRL_TFCE) &&
3430 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3431 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3432 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3433
d07f3e37 3434 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3435 adapter->tx_timeout_factor = 1;
3436 switch (adapter->link_speed) {
3437 case SPEED_10:
9d5c8243
AK		 3438 adapter->tx_timeout_factor = 14;
3439 break;
3440 case SPEED_100:
9d5c8243
AK		 3441 /* maybe add some timeout factor ? */
3442 break;
3443 }
3444
3445 netif_carrier_on(netdev);
9d5c8243 3446
4ae196df
AD		 3447 igb_ping_all_vfs(adapter);
3448
4b1a9877 3449 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3450 if (!test_bit(__IGB_DOWN, &adapter->state))
3451 mod_timer(&adapter->phy_info_timer,
3452 round_jiffies(jiffies + 2 * HZ));
3453 }
3454 } else {
3455 if (netif_carrier_ok(netdev)) {
3456 adapter->link_speed = 0;
3457 adapter->link_duplex = 0;
527d47c1
AD		 3458 /* Links status message must follow this format */
3459 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3460 netdev->name);
9d5c8243 3461 netif_carrier_off(netdev);
4b1a9877 3462
4ae196df
AD		 3463 igb_ping_all_vfs(adapter);
3464
4b1a9877 3465 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3466 if (!test_bit(__IGB_DOWN, &adapter->state))
3467 mod_timer(&adapter->phy_info_timer,
3468 round_jiffies(jiffies + 2 * HZ));
3469 }
3470 }
3471
9d5c8243 3472 igb_update_stats(adapter);
9d5c8243 3473
dbabb065 3474 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3475 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3476 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3477 /* We've lost link, so the controller stops DMA,
3478 * but we've got queued Tx work that's never going
3479 * to get done, so reset controller to flush Tx.
3480 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3481 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3482 adapter->tx_timeout_count++;
3483 schedule_work(&adapter->reset_task);
3484 /* return immediately since reset is imminent */
3485 return;
3486 }
9d5c8243 3487 }
9d5c8243 3488
dbabb065
AD		 3489 /* Force detection of hung controller every watchdog period */
3490 tx_ring->detect_tx_hung = true;
3491 }
f7ba205e 3492
9d5c8243 3493 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3494 if (adapter->msix_entries) {
047e0030
AD		 3495 u32 eics = 0;
3496 for (i = 0; i < adapter->num_q_vectors; i++) {
3497 struct igb_q_vector *q_vector = adapter->q_vector[i];
3498 eics |= q_vector->eims_value;
3499 }
7a6ea550
AD		 3500 wr32(E1000_EICS, eics);
3501 } else {
3502 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3503 }
9d5c8243 3504
9d5c8243
AK		 3505 /* Reset the timer */
3506 if (!test_bit(__IGB_DOWN, &adapter->state))
3507 mod_timer(&adapter->watchdog_timer,
3508 round_jiffies(jiffies + 2 * HZ));
3509}
3510
3511enum latency_range {
3512 lowest_latency = 0,
3513 low_latency = 1,
3514 bulk_latency = 2,
3515 latency_invalid = 255
3516};
3517
6eb5a7f1
AD		 3518/**
3519 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3520 *
3521 * Stores a new ITR value based on strictly on packet size. This
3522 * algorithm is less sophisticated than that used in igb_update_itr,
3523 * due to the difficulty of synchronizing statistics across multiple
3524 * receive rings. The divisors and thresholds used by this fuction
3525 * were determined based on theoretical maximum wire speed and testing
3526 * data, in order to minimize response time while increasing bulk
3527 * throughput.
3528 * This functionality is controlled by the InterruptThrottleRate module
3529 * parameter (see igb_param.c)
3530 * NOTE: This function is called only when operating in a multiqueue
3531 * receive environment.
047e0030 3532 * @q_vector: pointer to q_vector
6eb5a7f1 3533 **/
047e0030 3534static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3535{
047e0030 3536 int new_val = q_vector->itr_val;
6eb5a7f1 3537 int avg_wire_size = 0;
047e0030 3538 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3539
6eb5a7f1
AD		 3540 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3541 * ints/sec - ITR timer value of 120 ticks.
3542 */
3543 if (adapter->link_speed != SPEED_1000) {
047e0030 3544 new_val = 976;
6eb5a7f1 3545 goto set_itr_val;
9d5c8243 3546 }
047e0030
AD		 3547
3548 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3549 struct igb_ring *ring = q_vector->rx_ring;
3550 avg_wire_size = ring->total_bytes / ring->total_packets;
3551 }
3552
3553 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3554 struct igb_ring *ring = q_vector->tx_ring;
3555 avg_wire_size = max_t(u32, avg_wire_size,
3556 (ring->total_bytes /
3557 ring->total_packets));
3558 }
3559
3560 /* if avg_wire_size isn't set no work was done */
3561 if (!avg_wire_size)
3562 goto clear_counts;
9d5c8243 3563
6eb5a7f1
AD		 3564 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3565 avg_wire_size += 24;
3566
3567 /* Don't starve jumbo frames */
3568 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3569
6eb5a7f1
AD		 3570 /* Give a little boost to mid-size frames */
3571 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3572 new_val = avg_wire_size / 3;
3573 else
3574 new_val = avg_wire_size / 2;
9d5c8243 3575
abe1c363
NN		 3576 /* when in itr mode 3 do not exceed 20K ints/sec */
3577 if (adapter->rx_itr_setting == 3 && new_val < 196)
3578 new_val = 196;
3579
6eb5a7f1 3580set_itr_val:
047e0030
AD		 3581 if (new_val != q_vector->itr_val) {
3582 q_vector->itr_val = new_val;
3583 q_vector->set_itr = 1;
9d5c8243 3584 }
6eb5a7f1 3585clear_counts:
047e0030
AD		 3586 if (q_vector->rx_ring) {
3587 q_vector->rx_ring->total_bytes = 0;
3588 q_vector->rx_ring->total_packets = 0;
3589 }
3590 if (q_vector->tx_ring) {
3591 q_vector->tx_ring->total_bytes = 0;
3592 q_vector->tx_ring->total_packets = 0;
3593 }
9d5c8243
AK		 3594}
3595
3596/**
3597 * igb_update_itr - update the dynamic ITR value based on statistics
3598 * Stores a new ITR value based on packets and byte
3599 * counts during the last interrupt. The advantage of per interrupt
3600 * computation is faster updates and more accurate ITR for the current
3601 * traffic pattern. Constants in this function were computed
3602 * based on theoretical maximum wire speed and thresholds were set based
3603 * on testing data as well as attempting to minimize response time
3604 * while increasing bulk throughput.
3605 * this functionality is controlled by the InterruptThrottleRate module
3606 * parameter (see igb_param.c)
3607 * NOTE: These calculations are only valid when operating in a single-
3608 * queue environment.
3609 * @adapter: pointer to adapter
047e0030 3610 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3611 * @packets: the number of packets during this measurement interval
3612 * @bytes: the number of bytes during this measurement interval
3613 **/
3614static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3615 int packets, int bytes)
3616{
3617 unsigned int retval = itr_setting;
3618
3619 if (packets == 0)
3620 goto update_itr_done;
3621
3622 switch (itr_setting) {
3623 case lowest_latency:
3624 /* handle TSO and jumbo frames */
3625 if (bytes/packets > 8000)
3626 retval = bulk_latency;
3627 else if ((packets < 5) && (bytes > 512))
3628 retval = low_latency;
3629 break;
3630 case low_latency: /* 50 usec aka 20000 ints/s */
3631 if (bytes > 10000) {
3632 /* this if handles the TSO accounting */
3633 if (bytes/packets > 8000) {
3634 retval = bulk_latency;
3635 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3636 retval = bulk_latency;
3637 } else if ((packets > 35)) {
3638 retval = lowest_latency;
3639 }
3640 } else if (bytes/packets > 2000) {
3641 retval = bulk_latency;
3642 } else if (packets <= 2 && bytes < 512) {
3643 retval = lowest_latency;
3644 }
3645 break;
3646 case bulk_latency: /* 250 usec aka 4000 ints/s */
3647 if (bytes > 25000) {
3648 if (packets > 35)
3649 retval = low_latency;
1e5c3d21 3650 } else if (bytes < 1500) {
9d5c8243
AK		 3651 retval = low_latency;
3652 }
3653 break;
3654 }
3655
3656update_itr_done:
3657 return retval;
3658}
3659
6eb5a7f1 3660static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3661{
047e0030 3662 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3663 u16 current_itr;
047e0030 3664 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3665
3666 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3667 if (adapter->link_speed != SPEED_1000) {
3668 current_itr = 0;
3669 new_itr = 4000;
3670 goto set_itr_now;
3671 }
3672
3673 adapter->rx_itr = igb_update_itr(adapter,
3674 adapter->rx_itr,
3025a446
AD		 3675 q_vector->rx_ring->total_packets,
3676 q_vector->rx_ring->total_bytes);
9d5c8243 3677
047e0030
AD		 3678 adapter->tx_itr = igb_update_itr(adapter,
3679 adapter->tx_itr,
3025a446
AD		 3680 q_vector->tx_ring->total_packets,
3681 q_vector->tx_ring->total_bytes);
047e0030 3682 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3683
6eb5a7f1 3684 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3685 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3686 current_itr = low_latency;
3687
9d5c8243
AK		 3688 switch (current_itr) {
3689 /* counts and packets in update_itr are dependent on these numbers */
3690 case lowest_latency:
78b1f607 3691 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3692 break;
3693 case low_latency:
78b1f607 3694 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3695 break;
3696 case bulk_latency:
78b1f607 3697 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3698 break;
3699 default:
3700 break;
3701 }
3702
3703set_itr_now:
3025a446
AD		 3704 q_vector->rx_ring->total_bytes = 0;
3705 q_vector->rx_ring->total_packets = 0;
3706 q_vector->tx_ring->total_bytes = 0;
3707 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3708
047e0030 3709 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3710 /* this attempts to bias the interrupt rate towards Bulk
3711 * by adding intermediate steps when interrupt rate is
3712 * increasing */
047e0030
AD		 3713 new_itr = new_itr > q_vector->itr_val ?
3714 max((new_itr * q_vector->itr_val) /
3715 (new_itr + (q_vector->itr_val >> 2)),
3716 new_itr) :
9d5c8243
AK		 3717 new_itr;
3718 /* Don't write the value here; it resets the adapter's
3719 * internal timer, and causes us to delay far longer than
3720 * we should between interrupts. Instead, we write the ITR
3721 * value at the beginning of the next interrupt so the timing
3722 * ends up being correct.
3723 */
047e0030
AD		 3724 q_vector->itr_val = new_itr;
3725 q_vector->set_itr = 1;
9d5c8243 3726 }
9d5c8243
AK		 3727}
3728
9d5c8243
AK		 3729#define IGB_TX_FLAGS_CSUM 0x00000001
3730#define IGB_TX_FLAGS_VLAN 0x00000002
3731#define IGB_TX_FLAGS_TSO 0x00000004
3732#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3733#define IGB_TX_FLAGS_TSTAMP 0x00000010
3734#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3735#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3736
85ad76b2 3737static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3738 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3739{
3740 struct e1000_adv_tx_context_desc *context_desc;
3741 unsigned int i;
3742 int err;
3743 struct igb_buffer *buffer_info;
3744 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3745 u32 mss_l4len_idx;
3746 u8 l4len;
9d5c8243
AK		 3747
3748 if (skb_header_cloned(skb)) {
3749 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3750 if (err)
3751 return err;
3752 }
3753
3754 l4len = tcp_hdrlen(skb);
3755 *hdr_len += l4len;
3756
3757 if (skb->protocol == htons(ETH_P_IP)) {
3758 struct iphdr *iph = ip_hdr(skb);
3759 iph->tot_len = 0;
3760 iph->check = 0;
3761 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3762 iph->daddr, 0,
3763 IPPROTO_TCP,
3764 0);
8e1e8a47 3765 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3766 ipv6_hdr(skb)->payload_len = 0;
3767 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3768 &ipv6_hdr(skb)->daddr,
3769 0, IPPROTO_TCP, 0);
3770 }
3771
3772 i = tx_ring->next_to_use;
3773
3774 buffer_info = &tx_ring->buffer_info[i];
3775 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3776 /* VLAN MACLEN IPLEN */
3777 if (tx_flags & IGB_TX_FLAGS_VLAN)
3778 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3779 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3780 *hdr_len += skb_network_offset(skb);
3781 info |= skb_network_header_len(skb);
3782 *hdr_len += skb_network_header_len(skb);
3783 context_desc->vlan_macip_lens = cpu_to_le32(info);
3784
3785 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3786 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3787
3788 if (skb->protocol == htons(ETH_P_IP))
3789 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3790 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3791
3792 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3793
3794 /* MSS L4LEN IDX */
3795 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3796 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3797
73cd78f1 3798 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3799 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3800 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3801
3802 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3803 context_desc->seqnum_seed = 0;
3804
3805 buffer_info->time_stamp = jiffies;
0e014cb1 3806 buffer_info->next_to_watch = i;
9d5c8243
AK		 3807 buffer_info->dma = 0;
3808 i++;
3809 if (i == tx_ring->count)
3810 i = 0;
3811
3812 tx_ring->next_to_use = i;
3813
3814 return true;
3815}
3816
85ad76b2
AD		 3817static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3818 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3819{
3820 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3821 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3822 struct igb_buffer *buffer_info;
3823 u32 info = 0, tu_cmd = 0;
80785298 3824 unsigned int i;
9d5c8243
AK		 3825
3826 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3827 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3828 i = tx_ring->next_to_use;
3829 buffer_info = &tx_ring->buffer_info[i];
3830 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3831
3832 if (tx_flags & IGB_TX_FLAGS_VLAN)
3833 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3834
9d5c8243
AK		 3835 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3836 if (skb->ip_summed == CHECKSUM_PARTIAL)
3837 info |= skb_network_header_len(skb);
3838
3839 context_desc->vlan_macip_lens = cpu_to_le32(info);
3840
3841 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3842
3843 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3844 __be16 protocol;
3845
3846 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3847 const struct vlan_ethhdr *vhdr =
3848 (const struct vlan_ethhdr*)skb->data;
3849
3850 protocol = vhdr->h_vlan_encapsulated_proto;
3851 } else {
3852 protocol = skb->protocol;
3853 }
3854
3855 switch (protocol) {
09640e63 3856 case cpu_to_be16(ETH_P_IP):
9d5c8243 3857 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3858 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3859 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3860 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3861 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3862 break;
09640e63 3863 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3864 /* XXX what about other V6 headers?? */
3865 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3866 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3867 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3868 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3869 break;
3870 default:
3871 if (unlikely(net_ratelimit()))
59d71989 3872 dev_warn(dev,
44b0cda3
MW		 3873 "partial checksum but proto=%x!\n",
3874 skb->protocol);
3875 break;
3876 }
9d5c8243
AK		 3877 }
3878
3879 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3880 context_desc->seqnum_seed = 0;
85ad76b2 3881 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3882 context_desc->mss_l4len_idx =
85ad76b2 3883 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3884
3885 buffer_info->time_stamp = jiffies;
0e014cb1 3886 buffer_info->next_to_watch = i;
9d5c8243
AK		 3887 buffer_info->dma = 0;
3888
3889 i++;
3890 if (i == tx_ring->count)
3891 i = 0;
3892 tx_ring->next_to_use = i;
3893
3894 return true;
3895 }
9d5c8243
AK		 3896 return false;
3897}
3898
3899#define IGB_MAX_TXD_PWR 16
3900#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3901
80785298 3902static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3903 unsigned int first)
9d5c8243
AK		 3904{
3905 struct igb_buffer *buffer_info;
59d71989 3906 struct device *dev = tx_ring->dev;
2873957d 3907 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3908 unsigned int count = 0, i;
3909 unsigned int f;
2873957d 3910 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3911
3912 i = tx_ring->next_to_use;
3913
3914 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3915 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3916 buffer_info->length = hlen;
9d5c8243
AK		 3917 /* set time_stamp *before* dma to help avoid a possible race */
3918 buffer_info->time_stamp = jiffies;
0e014cb1 3919 buffer_info->next_to_watch = i;
2873957d 3920 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3921 DMA_TO_DEVICE);
3922 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3923 goto dma_error;
9d5c8243
AK		 3924
3925 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3926 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3927 unsigned int len = frag->size;
9d5c8243 3928
8581145f 3929 count++;
65689fef
AD		 3930 i++;
3931 if (i == tx_ring->count)
3932 i = 0;
3933
9d5c8243
AK		 3934 buffer_info = &tx_ring->buffer_info[i];
3935 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3936 buffer_info->length = len;
3937 buffer_info->time_stamp = jiffies;
0e014cb1 3938 buffer_info->next_to_watch = i;
6366ad33 3939 buffer_info->mapped_as_page = true;
59d71989 3940 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3941 frag->page,
3942 frag->page_offset,
3943 len,
59d71989
AD		 3944 DMA_TO_DEVICE);
3945 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3946 goto dma_error;
3947
9d5c8243
AK		 3948 }
3949
9d5c8243 3950 tx_ring->buffer_info[i].skb = skb;
2873957d
NN		 3951 tx_ring->buffer_info[i].shtx = skb_shinfo(skb)->tx_flags;
3952 /* multiply data chunks by size of headers */
3953 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3954 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3955 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3956
cdfd01fc 3957 return ++count;
6366ad33
AD		 3958
3959dma_error:
59d71989 3960 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3961
3962 /* clear timestamp and dma mappings for failed buffer_info mapping */
3963 buffer_info->dma = 0;
3964 buffer_info->time_stamp = 0;
3965 buffer_info->length = 0;
3966 buffer_info->next_to_watch = 0;
3967 buffer_info->mapped_as_page = false;
6366ad33
AD		 3968
3969 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3970 while (count--) {
3971 if (i == 0)
3972 i = tx_ring->count;
6366ad33 3973 i--;
6366ad33
AD		 3974 buffer_info = &tx_ring->buffer_info[i];
3975 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3976 }
3977
3978 return 0;
9d5c8243
AK		 3979}
3980
85ad76b2 3981static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3982 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3983 u8 hdr_len)
3984{
cdfd01fc 3985 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3986 struct igb_buffer *buffer_info;
3987 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3988 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3989
3990 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3991 E1000_ADVTXD_DCMD_DEXT);
3992
3993 if (tx_flags & IGB_TX_FLAGS_VLAN)
3994 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3995
33af6bcc
PO		 3996 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3997 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3998
9d5c8243
AK		 3999 if (tx_flags & IGB_TX_FLAGS_TSO) {
4000 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4001
4002 /* insert tcp checksum */
4003 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4004
4005 /* insert ip checksum */
4006 if (tx_flags & IGB_TX_FLAGS_IPV4)
4007 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4008
4009 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4010 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4011 }
4012
85ad76b2
AD		 4013 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4014 (tx_flags & (IGB_TX_FLAGS_CSUM |
4015 IGB_TX_FLAGS_TSO |
7dfc16fa 4016 IGB_TX_FLAGS_VLAN)))
85ad76b2 4017 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4018
4019 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4020
cdfd01fc 4021 do {
9d5c8243
AK		 4022 buffer_info = &tx_ring->buffer_info[i];
4023 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4024 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4025 tx_desc->read.cmd_type_len =
4026 cpu_to_le32(cmd_type_len | buffer_info->length);
4027 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4028 count--;
9d5c8243
AK		 4029 i++;
4030 if (i == tx_ring->count)
4031 i = 0;
cdfd01fc 4032 } while (count > 0);
9d5c8243 4033
85ad76b2 4034 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4035 /* Force memory writes to complete before letting h/w
4036 * know there are new descriptors to fetch. (Only
4037 * applicable for weak-ordered memory model archs,
4038 * such as IA-64). */
4039 wmb();
4040
4041 tx_ring->next_to_use = i;
fce99e34 4042 writel(i, tx_ring->tail);
9d5c8243
AK		 4043 /* we need this if more than one processor can write to our tail
4044 * at a time, it syncronizes IO on IA64/Altix systems */
4045 mmiowb();
4046}
4047
e694e964 4048static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4049{
e694e964
AD		 4050 struct net_device *netdev = tx_ring->netdev;
4051
661086df 4052 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4053
9d5c8243
AK		 4054 /* Herbert's original patch had:
4055 * smp_mb__after_netif_stop_queue();
4056 * but since that doesn't exist yet, just open code it. */
4057 smp_mb();
4058
4059 /* We need to check again in a case another CPU has just
4060 * made room available. */
c493ea45 4061 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4062 return -EBUSY;
4063
4064 /* A reprieve! */
661086df 4065 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4066 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 4067 return 0;
4068}
4069
717ba089 4070static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4071{
c493ea45 4072 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4073 return 0;
e694e964 4074 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4075}
4076
b1a436c3
AD		 4077netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4078 struct igb_ring *tx_ring)
9d5c8243 4079{
e694e964 4080 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 4081 int tso = 0, count;
91d4ee33
NN		 4082 u32 tx_flags = 0;
4083 u16 first;
4084 u8 hdr_len = 0;
c5b9bd5e 4085 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 4086
9d5c8243
AK		 4087 /* need: 1 descriptor per page,
4088 * + 2 desc gap to keep tail from touching head,
4089 * + 1 desc for skb->data,
4090 * + 1 desc for context descriptor,
4091 * otherwise try next time */
e694e964 4092 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4093 /* this is a hard error */
9d5c8243
AK		 4094 return NETDEV_TX_BUSY;
4095 }
33af6bcc 4096
33af6bcc
PO		 4097 if (unlikely(shtx->hardware)) {
4098 shtx->in_progress = 1;
4099 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4100 }
9d5c8243 4101
cdfd01fc 4102 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 4103 tx_flags |= IGB_TX_FLAGS_VLAN;
4104 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4105 }
4106
661086df
PWJ		 4107 if (skb->protocol == htons(ETH_P_IP))
4108 tx_flags |= IGB_TX_FLAGS_IPV4;
4109
0e014cb1 4110 first = tx_ring->next_to_use;
85ad76b2
AD		 4111 if (skb_is_gso(skb)) {
4112 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4113
85ad76b2
AD		 4114 if (tso < 0) {
4115 dev_kfree_skb_any(skb);
4116 return NETDEV_TX_OK;
4117 }
9d5c8243
AK		 4118 }
4119
4120 if (tso)
4121 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4122 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4123 (skb->ip_summed == CHECKSUM_PARTIAL))
4124 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4125
65689fef 4126 /*
cdfd01fc 4127 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4128 * has occured and we need to rewind the descriptor queue
4129 */
80785298 4130 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4131 if (!count) {
65689fef
AD		 4132 dev_kfree_skb_any(skb);
4133 tx_ring->buffer_info[first].time_stamp = 0;
4134 tx_ring->next_to_use = first;
85ad76b2 4135 return NETDEV_TX_OK;
65689fef 4136 }
9d5c8243 4137
85ad76b2
AD		 4138 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4139
4140 /* Make sure there is space in the ring for the next send. */
e694e964 4141 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4142
9d5c8243
AK		 4143 return NETDEV_TX_OK;
4144}
4145
3b29a56d
SH		 4146static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4147 struct net_device *netdev)
9d5c8243
AK		 4148{
4149 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4150 struct igb_ring *tx_ring;
661086df 4151 int r_idx = 0;
b1a436c3
AD		 4152
4153 if (test_bit(__IGB_DOWN, &adapter->state)) {
4154 dev_kfree_skb_any(skb);
4155 return NETDEV_TX_OK;
4156 }
4157
4158 if (skb->len <= 0) {
4159 dev_kfree_skb_any(skb);
4160 return NETDEV_TX_OK;
4161 }
4162
1bfaf07b 4163 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4164 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4165
4166 /* This goes back to the question of how to logically map a tx queue
4167 * to a flow. Right now, performance is impacted slightly negatively
4168 * if using multiple tx queues. If the stack breaks away from a
4169 * single qdisc implementation, we can look at this again. */
e694e964 4170 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4171}
4172
4173/**
4174 * igb_tx_timeout - Respond to a Tx Hang
4175 * @netdev: network interface device structure
4176 **/
4177static void igb_tx_timeout(struct net_device *netdev)
4178{
4179 struct igb_adapter *adapter = netdev_priv(netdev);
4180 struct e1000_hw *hw = &adapter->hw;
4181
4182 /* Do the reset outside of interrupt context */
4183 adapter->tx_timeout_count++;
f7ba205e 4184
55cac248
AD		 4185 if (hw->mac.type == e1000_82580)
4186 hw->dev_spec._82575.global_device_reset = true;
4187
9d5c8243 4188 schedule_work(&adapter->reset_task);
265de409
AD		 4189 wr32(E1000_EICS,
4190 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4191}
4192
4193static void igb_reset_task(struct work_struct *work)
4194{
4195 struct igb_adapter *adapter;
4196 adapter = container_of(work, struct igb_adapter, reset_task);
4197
c97ec42a
TI		 4198 igb_dump(adapter);
4199 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4200 igb_reinit_locked(adapter);
4201}
4202
4203/**
4204 * igb_get_stats - Get System Network Statistics
4205 * @netdev: network interface device structure
4206 *
4207 * Returns the address of the device statistics structure.
4208 * The statistics are actually updated from the timer callback.
4209 **/
73cd78f1 4210static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 4211{
9d5c8243 4212 /* only return the current stats */
8d24e933 4213 return &netdev->stats;
9d5c8243
AK		 4214}
4215
4216/**
4217 * igb_change_mtu - Change the Maximum Transfer Unit
4218 * @netdev: network interface device structure
4219 * @new_mtu: new value for maximum frame size
4220 *
4221 * Returns 0 on success, negative on failure
4222 **/
4223static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4224{
4225 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4226 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4227 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4228 u32 rx_buffer_len, i;
9d5c8243 4229
c809d227 4230 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4231 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4232 return -EINVAL;
4233 }
4234
9d5c8243 4235 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4236 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4237 return -EINVAL;
4238 }
4239
4240 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4241 msleep(1);
73cd78f1 4242
9d5c8243
AK		 4243 /* igb_down has a dependency on max_frame_size */
4244 adapter->max_frame_size = max_frame;
559e9c49 4245
9d5c8243
AK		 4246 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4247 * means we reserve 2 more, this pushes us to allocate from the next
4248 * larger slab size.
4249 * i.e. RXBUFFER_2048 --> size-4096 slab
4250 */
4251
757b77e2
NN		 4252 if (adapter->hw.mac.type == e1000_82580)
4253 max_frame += IGB_TS_HDR_LEN;
4254
7d95b717 4255 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4256 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4257 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4258 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4259 else
4c844851
AD		 4260 rx_buffer_len = IGB_RXBUFFER_128;
4261
757b77e2
NN		 4262 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4263 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4264 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4265
4266 if ((adapter->hw.mac.type == e1000_82580) &&
4267 (rx_buffer_len == IGB_RXBUFFER_128))
4268 rx_buffer_len += IGB_RXBUFFER_64;
4269
4c844851
AD		 4270 if (netif_running(netdev))
4271 igb_down(adapter);
9d5c8243 4272
090b1795 4273 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4274 netdev->mtu, new_mtu);
4275 netdev->mtu = new_mtu;
4276
4c844851 4277 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4278 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4279
9d5c8243
AK		 4280 if (netif_running(netdev))
4281 igb_up(adapter);
4282 else
4283 igb_reset(adapter);
4284
4285 clear_bit(__IGB_RESETTING, &adapter->state);
4286
4287 return 0;
4288}
4289
4290/**
4291 * igb_update_stats - Update the board statistics counters
4292 * @adapter: board private structure
4293 **/
4294
4295void igb_update_stats(struct igb_adapter *adapter)
4296{
128e45eb 4297 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 4298 struct e1000_hw *hw = &adapter->hw;
4299 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4300 u32 reg, mpc;
9d5c8243 4301 u16 phy_tmp;
3f9c0164
AD		 4302 int i;
4303 u64 bytes, packets;
9d5c8243
AK		 4304
4305#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4306
4307 /*
4308 * Prevent stats update while adapter is being reset, or if the pci
4309 * connection is down.
4310 */
4311 if (adapter->link_speed == 0)
4312 return;
4313 if (pci_channel_offline(pdev))
4314 return;
4315
3f9c0164
AD		 4316 bytes = 0;
4317 packets = 0;
4318 for (i = 0; i < adapter->num_rx_queues; i++) {
4319 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 4320 struct igb_ring *ring = adapter->rx_ring[i];
4321 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4322 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 4323 bytes += ring->rx_stats.bytes;
4324 packets += ring->rx_stats.packets;
3f9c0164
AD		 4325 }
4326
128e45eb
AD		 4327 net_stats->rx_bytes = bytes;
4328 net_stats->rx_packets = packets;
3f9c0164
AD		 4329
4330 bytes = 0;
4331 packets = 0;
4332 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4333 struct igb_ring *ring = adapter->tx_ring[i];
4334 bytes += ring->tx_stats.bytes;
4335 packets += ring->tx_stats.packets;
3f9c0164 4336 }
128e45eb
AD		 4337 net_stats->tx_bytes = bytes;
4338 net_stats->tx_packets = packets;
3f9c0164
AD		 4339
4340 /* read stats registers */
9d5c8243
AK		 4341 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4342 adapter->stats.gprc += rd32(E1000_GPRC);
4343 adapter->stats.gorc += rd32(E1000_GORCL);
4344 rd32(E1000_GORCH); /* clear GORCL */
4345 adapter->stats.bprc += rd32(E1000_BPRC);
4346 adapter->stats.mprc += rd32(E1000_MPRC);
4347 adapter->stats.roc += rd32(E1000_ROC);
4348
4349 adapter->stats.prc64 += rd32(E1000_PRC64);
4350 adapter->stats.prc127 += rd32(E1000_PRC127);
4351 adapter->stats.prc255 += rd32(E1000_PRC255);
4352 adapter->stats.prc511 += rd32(E1000_PRC511);
4353 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4354 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4355 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4356 adapter->stats.sec += rd32(E1000_SEC);
4357
fa3d9a6d
MW		 4358 mpc = rd32(E1000_MPC);
4359 adapter->stats.mpc += mpc;
4360 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4361 adapter->stats.scc += rd32(E1000_SCC);
4362 adapter->stats.ecol += rd32(E1000_ECOL);
4363 adapter->stats.mcc += rd32(E1000_MCC);
4364 adapter->stats.latecol += rd32(E1000_LATECOL);
4365 adapter->stats.dc += rd32(E1000_DC);
4366 adapter->stats.rlec += rd32(E1000_RLEC);
4367 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4368 adapter->stats.xontxc += rd32(E1000_XONTXC);
4369 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4370 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4371 adapter->stats.fcruc += rd32(E1000_FCRUC);
4372 adapter->stats.gptc += rd32(E1000_GPTC);
4373 adapter->stats.gotc += rd32(E1000_GOTCL);
4374 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4375 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4376 adapter->stats.ruc += rd32(E1000_RUC);
4377 adapter->stats.rfc += rd32(E1000_RFC);
4378 adapter->stats.rjc += rd32(E1000_RJC);
4379 adapter->stats.tor += rd32(E1000_TORH);
4380 adapter->stats.tot += rd32(E1000_TOTH);
4381 adapter->stats.tpr += rd32(E1000_TPR);
4382
4383 adapter->stats.ptc64 += rd32(E1000_PTC64);
4384 adapter->stats.ptc127 += rd32(E1000_PTC127);
4385 adapter->stats.ptc255 += rd32(E1000_PTC255);
4386 adapter->stats.ptc511 += rd32(E1000_PTC511);
4387 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4388 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4389
4390 adapter->stats.mptc += rd32(E1000_MPTC);
4391 adapter->stats.bptc += rd32(E1000_BPTC);
4392
2d0b0f69
NN		 4393 adapter->stats.tpt += rd32(E1000_TPT);
4394 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4395
4396 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4397 /* read internal phy specific stats */
4398 reg = rd32(E1000_CTRL_EXT);
4399 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4400 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4401 adapter->stats.tncrs += rd32(E1000_TNCRS);
4402 }
4403
9d5c8243
AK		 4404 adapter->stats.tsctc += rd32(E1000_TSCTC);
4405 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4406
4407 adapter->stats.iac += rd32(E1000_IAC);
4408 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4409 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4410 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4411 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4412 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4413 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4414 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4415 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4416
4417 /* Fill out the OS statistics structure */
128e45eb
AD		 4418 net_stats->multicast = adapter->stats.mprc;
4419 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4420
4421 /* Rx Errors */
4422
4423 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4424 * our own version based on RUC and ROC */
128e45eb 4425 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4426 adapter->stats.crcerrs + adapter->stats.algnerrc +
4427 adapter->stats.ruc + adapter->stats.roc +
4428 adapter->stats.cexterr;
128e45eb
AD		 4429 net_stats->rx_length_errors = adapter->stats.ruc +
4430 adapter->stats.roc;
4431 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4432 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4433 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4434
4435 /* Tx Errors */
128e45eb
AD		 4436 net_stats->tx_errors = adapter->stats.ecol +
4437 adapter->stats.latecol;
4438 net_stats->tx_aborted_errors = adapter->stats.ecol;
4439 net_stats->tx_window_errors = adapter->stats.latecol;
4440 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4441
4442 /* Tx Dropped needs to be maintained elsewhere */
4443
4444 /* Phy Stats */
4445 if (hw->phy.media_type == e1000_media_type_copper) {
4446 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4447 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4448 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4449 adapter->phy_stats.idle_errors += phy_tmp;
4450 }
4451 }
4452
4453 /* Management Stats */
4454 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4455 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4456 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4457}
4458
9d5c8243
AK		 4459static irqreturn_t igb_msix_other(int irq, void *data)
4460{
047e0030 4461 struct igb_adapter *adapter = data;
9d5c8243 4462 struct e1000_hw *hw = &adapter->hw;
844290e5 4463 u32 icr = rd32(E1000_ICR);
844290e5 4464 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4465
7f081d40
AD		 4466 if (icr & E1000_ICR_DRSTA)
4467 schedule_work(&adapter->reset_task);
4468
047e0030 4469 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4470 /* HW is reporting DMA is out of sync */
4471 adapter->stats.doosync++;
4472 }
eebbbdba 4473
4ae196df
AD		 4474 /* Check for a mailbox event */
4475 if (icr & E1000_ICR_VMMB)
4476 igb_msg_task(adapter);
4477
4478 if (icr & E1000_ICR_LSC) {
4479 hw->mac.get_link_status = 1;
4480 /* guard against interrupt when we're going down */
4481 if (!test_bit(__IGB_DOWN, &adapter->state))
4482 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4483 }
4484
25568a53
AD		 4485 if (adapter->vfs_allocated_count)
4486 wr32(E1000_IMS, E1000_IMS_LSC |
4487 E1000_IMS_VMMB |
4488 E1000_IMS_DOUTSYNC);
4489 else
4490 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4491 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4492
4493 return IRQ_HANDLED;
4494}
4495
047e0030 4496static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4497{
26b39276 4498 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4499 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4500
047e0030
AD		 4501 if (!q_vector->set_itr)
4502 return;
73cd78f1 4503
047e0030
AD		 4504 if (!itr_val)
4505 itr_val = 0x4;
661086df 4506
26b39276
AD		 4507 if (adapter->hw.mac.type == e1000_82575)
4508 itr_val |= itr_val << 16;
661086df 4509 else
047e0030 4510 itr_val |= 0x8000000;
661086df 4511
047e0030
AD		 4512 writel(itr_val, q_vector->itr_register);
4513 q_vector->set_itr = 0;
6eb5a7f1
AD		 4514}
4515
047e0030 4516static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4517{
047e0030 4518 struct igb_q_vector *q_vector = data;
9d5c8243 4519
047e0030
AD		 4520 /* Write the ITR value calculated from the previous interrupt. */
4521 igb_write_itr(q_vector);
9d5c8243 4522
047e0030 4523 napi_schedule(&q_vector->napi);
844290e5 4524
047e0030 4525 return IRQ_HANDLED;
fe4506b6
JC		 4526}
4527
421e02f0 4528#ifdef CONFIG_IGB_DCA
047e0030 4529static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4530{
047e0030 4531 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4532 struct e1000_hw *hw = &adapter->hw;
4533 int cpu = get_cpu();
fe4506b6 4534
047e0030
AD		 4535 if (q_vector->cpu == cpu)
4536 goto out_no_update;
4537
4538 if (q_vector->tx_ring) {
4539 int q = q_vector->tx_ring->reg_idx;
4540 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4541 if (hw->mac.type == e1000_82575) {
4542 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4543 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4544 } else {
047e0030
AD		 4545 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4546 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4547 E1000_DCA_TXCTRL_CPUID_SHIFT;
4548 }
4549 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4550 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4551 }
4552 if (q_vector->rx_ring) {
4553 int q = q_vector->rx_ring->reg_idx;
4554 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4555 if (hw->mac.type == e1000_82575) {
2d064c06 4556 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4557 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4558 } else {
4559 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4560 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4561 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4562 }
fe4506b6
JC		 4563 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4564 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4565 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4566 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4567 }
047e0030
AD		 4568 q_vector->cpu = cpu;
4569out_no_update:
fe4506b6
JC		 4570 put_cpu();
4571}
4572
4573static void igb_setup_dca(struct igb_adapter *adapter)
4574{
7e0e99ef 4575 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4576 int i;
4577
7dfc16fa 4578 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4579 return;
4580
7e0e99ef
AD		 4581 /* Always use CB2 mode, difference is masked in the CB driver. */
4582 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4583
047e0030 4584 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4585 adapter->q_vector[i]->cpu = -1;
4586 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4587 }
4588}
4589
4590static int __igb_notify_dca(struct device *dev, void *data)
4591{
4592 struct net_device *netdev = dev_get_drvdata(dev);
4593 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4594 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4595 struct e1000_hw *hw = &adapter->hw;
4596 unsigned long event = *(unsigned long *)data;
4597
4598 switch (event) {
4599 case DCA_PROVIDER_ADD:
4600 /* if already enabled, don't do it again */
7dfc16fa 4601 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4602 break;
fe4506b6 4603 if (dca_add_requester(dev) == 0) {
bbd98fe4 4604 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4605 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4606 igb_setup_dca(adapter);
4607 break;
4608 }
4609 /* Fall Through since DCA is disabled. */
4610 case DCA_PROVIDER_REMOVE:
7dfc16fa 4611 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4612 /* without this a class_device is left
047e0030 4613 * hanging around in the sysfs model */
fe4506b6 4614 dca_remove_requester(dev);
090b1795 4615 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4616 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4617 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4618 }
4619 break;
4620 }
bbd98fe4 4621
fe4506b6 4622 return 0;
9d5c8243
AK		 4623}
4624
fe4506b6
JC		 4625static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4626 void *p)
4627{
4628 int ret_val;
4629
4630 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4631 __igb_notify_dca);
4632
4633 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4634}
421e02f0 4635#endif /* CONFIG_IGB_DCA */
9d5c8243 4636
4ae196df
AD		 4637static void igb_ping_all_vfs(struct igb_adapter *adapter)
4638{
4639 struct e1000_hw *hw = &adapter->hw;
4640 u32 ping;
4641 int i;
4642
4643 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4644 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4645 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4646 ping |= E1000_VT_MSGTYPE_CTS;
4647 igb_write_mbx(hw, &ping, 1, i);
4648 }
4649}
4650
7d5753f0
AD		 4651static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4652{
4653 struct e1000_hw *hw = &adapter->hw;
4654 u32 vmolr = rd32(E1000_VMOLR(vf));
4655 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4656
4657 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4658 IGB_VF_FLAG_MULTI_PROMISC);
4659 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4660
4661 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4662 vmolr |= E1000_VMOLR_MPME;
4663 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4664 } else {
4665 /*
4666 * if we have hashes and we are clearing a multicast promisc
4667 * flag we need to write the hashes to the MTA as this step
4668 * was previously skipped
4669 */
4670 if (vf_data->num_vf_mc_hashes > 30) {
4671 vmolr |= E1000_VMOLR_MPME;
4672 } else if (vf_data->num_vf_mc_hashes) {
4673 int j;
4674 vmolr |= E1000_VMOLR_ROMPE;
4675 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4676 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4677 }
4678 }
4679
4680 wr32(E1000_VMOLR(vf), vmolr);
4681
4682 /* there are flags left unprocessed, likely not supported */
4683 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4684 return -EINVAL;
4685
4686 return 0;
4687
4688}
4689
4ae196df
AD		 4690static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4691 u32 *msgbuf, u32 vf)
4692{
4693 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4694 u16 *hash_list = (u16 *)&msgbuf[1];
4695 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4696 int i;
4697
7d5753f0 4698 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4699 * to this VF for later use to restore when the PF multi cast
4700 * list changes
4701 */
4702 vf_data->num_vf_mc_hashes = n;
4703
7d5753f0
AD		 4704 /* only up to 30 hash values supported */
4705 if (n > 30)
4706 n = 30;
4707
4708 /* store the hashes for later use */
4ae196df 4709 for (i = 0; i < n; i++)
a419aef8 4710 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4711
4712 /* Flush and reset the mta with the new values */
ff41f8dc 4713 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4714
4715 return 0;
4716}
4717
4718static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4719{
4720 struct e1000_hw *hw = &adapter->hw;
4721 struct vf_data_storage *vf_data;
4722 int i, j;
4723
4724 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4725 u32 vmolr = rd32(E1000_VMOLR(i));
4726 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4727
4ae196df 4728 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4729
4730 if ((vf_data->num_vf_mc_hashes > 30) ||
4731 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4732 vmolr |= E1000_VMOLR_MPME;
4733 } else if (vf_data->num_vf_mc_hashes) {
4734 vmolr |= E1000_VMOLR_ROMPE;
4735 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4736 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4737 }
4738 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4739 }
4740}
4741
4742static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4743{
4744 struct e1000_hw *hw = &adapter->hw;
4745 u32 pool_mask, reg, vid;
4746 int i;
4747
4748 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4749
4750 /* Find the vlan filter for this id */
4751 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4752 reg = rd32(E1000_VLVF(i));
4753
4754 /* remove the vf from the pool */
4755 reg &= ~pool_mask;
4756
4757 /* if pool is empty then remove entry from vfta */
4758 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4759 (reg & E1000_VLVF_VLANID_ENABLE)) {
4760 reg = 0;
4761 vid = reg & E1000_VLVF_VLANID_MASK;
4762 igb_vfta_set(hw, vid, false);
4763 }
4764
4765 wr32(E1000_VLVF(i), reg);
4766 }
ae641bdc
AD		 4767
4768 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4769}
4770
4771static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4772{
4773 struct e1000_hw *hw = &adapter->hw;
4774 u32 reg, i;
4775
51466239
AD		 4776 /* The vlvf table only exists on 82576 hardware and newer */
4777 if (hw->mac.type < e1000_82576)
4778 return -1;
4779
4780 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4781 if (!adapter->vfs_allocated_count)
4782 return -1;
4783
4784 /* Find the vlan filter for this id */
4785 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4786 reg = rd32(E1000_VLVF(i));
4787 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4788 vid == (reg & E1000_VLVF_VLANID_MASK))
4789 break;
4790 }
4791
4792 if (add) {
4793 if (i == E1000_VLVF_ARRAY_SIZE) {
4794 /* Did not find a matching VLAN ID entry that was
4795 * enabled. Search for a free filter entry, i.e.
4796 * one without the enable bit set
4797 */
4798 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4799 reg = rd32(E1000_VLVF(i));
4800 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4801 break;
4802 }
4803 }
4804 if (i < E1000_VLVF_ARRAY_SIZE) {
4805 /* Found an enabled/available entry */
4806 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4807
4808 /* if !enabled we need to set this up in vfta */
4809 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4810 /* add VID to filter table */
4811 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4812 reg |= E1000_VLVF_VLANID_ENABLE;
4813 }
cad6d05f
AD		 4814 reg &= ~E1000_VLVF_VLANID_MASK;
4815 reg |= vid;
4ae196df 4816 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4817
4818 /* do not modify RLPML for PF devices */
4819 if (vf >= adapter->vfs_allocated_count)
4820 return 0;
4821
4822 if (!adapter->vf_data[vf].vlans_enabled) {
4823 u32 size;
4824 reg = rd32(E1000_VMOLR(vf));
4825 size = reg & E1000_VMOLR_RLPML_MASK;
4826 size += 4;
4827 reg &= ~E1000_VMOLR_RLPML_MASK;
4828 reg |= size;
4829 wr32(E1000_VMOLR(vf), reg);
4830 }
ae641bdc 4831
51466239 4832 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4833 return 0;
4834 }
4835 } else {
4836 if (i < E1000_VLVF_ARRAY_SIZE) {
4837 /* remove vf from the pool */
4838 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4839 /* if pool is empty then remove entry from vfta */
4840 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4841 reg = 0;
4842 igb_vfta_set(hw, vid, false);
4843 }
4844 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4845
4846 /* do not modify RLPML for PF devices */
4847 if (vf >= adapter->vfs_allocated_count)
4848 return 0;
4849
4850 adapter->vf_data[vf].vlans_enabled--;
4851 if (!adapter->vf_data[vf].vlans_enabled) {
4852 u32 size;
4853 reg = rd32(E1000_VMOLR(vf));
4854 size = reg & E1000_VMOLR_RLPML_MASK;
4855 size -= 4;
4856 reg &= ~E1000_VMOLR_RLPML_MASK;
4857 reg |= size;
4858 wr32(E1000_VMOLR(vf), reg);
4859 }
4ae196df
AD		 4860 }
4861 }
8151d294
WM		 4862 return 0;
4863}
4864
4865static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4866{
4867 struct e1000_hw *hw = &adapter->hw;
4868
4869 if (vid)
4870 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4871 else
4872 wr32(E1000_VMVIR(vf), 0);
4873}
4874
4875static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4876 int vf, u16 vlan, u8 qos)
4877{
4878 int err = 0;
4879 struct igb_adapter *adapter = netdev_priv(netdev);
4880
4881 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4882 return -EINVAL;
4883 if (vlan || qos) {
4884 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4885 if (err)
4886 goto out;
4887 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4888 igb_set_vmolr(adapter, vf, !vlan);
4889 adapter->vf_data[vf].pf_vlan = vlan;
4890 adapter->vf_data[vf].pf_qos = qos;
4891 dev_info(&adapter->pdev->dev,
4892 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4893 if (test_bit(__IGB_DOWN, &adapter->state)) {
4894 dev_warn(&adapter->pdev->dev,
4895 "The VF VLAN has been set,"
4896 " but the PF device is not up.\n");
4897 dev_warn(&adapter->pdev->dev,
4898 "Bring the PF device up before"
4899 " attempting to use the VF device.\n");
4900 }
4901 } else {
4902 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4903 false, vf);
4904 igb_set_vmvir(adapter, vlan, vf);
4905 igb_set_vmolr(adapter, vf, true);
4906 adapter->vf_data[vf].pf_vlan = 0;
4907 adapter->vf_data[vf].pf_qos = 0;
4908 }
4909out:
4910 return err;
4ae196df
AD		 4911}
4912
4913static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4914{
4915 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4916 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4917
4918 return igb_vlvf_set(adapter, vid, add, vf);
4919}
4920
f2ca0dbe 4921static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4922{
8151d294
WM		 4923 /* clear flags */
4924 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4925 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4926
4927 /* reset offloads to defaults */
8151d294 4928 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4929
4930 /* reset vlans for device */
4931 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4932 if (adapter->vf_data[vf].pf_vlan)
4933 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4934 adapter->vf_data[vf].pf_vlan,
4935 adapter->vf_data[vf].pf_qos);
4936 else
4937 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4938
4939 /* reset multicast table array for vf */
4940 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4941
4942 /* Flush and reset the mta with the new values */
ff41f8dc 4943 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4944}
4945
f2ca0dbe
AD		 4946static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4947{
4948 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4949
4950 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4951 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4952 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4953
4954 /* process remaining reset events */
4955 igb_vf_reset(adapter, vf);
4956}
4957
4958static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4959{
4960 struct e1000_hw *hw = &adapter->hw;
4961 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4962 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4963 u32 reg, msgbuf[3];
4964 u8 *addr = (u8 *)(&msgbuf[1]);
4965
4966 /* process all the same items cleared in a function level reset */
f2ca0dbe 4967 igb_vf_reset(adapter, vf);
4ae196df
AD		 4968
4969 /* set vf mac address */
26ad9178 4970 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4971
4972 /* enable transmit and receive for vf */
4973 reg = rd32(E1000_VFTE);
4974 wr32(E1000_VFTE, reg | (1 << vf));
4975 reg = rd32(E1000_VFRE);
4976 wr32(E1000_VFRE, reg | (1 << vf));
4977
f2ca0dbe 4978 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4979
4980 /* reply to reset with ack and vf mac address */
4981 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4982 memcpy(addr, vf_mac, 6);
4983 igb_write_mbx(hw, msgbuf, 3, vf);
4984}
4985
4986static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4987{
de42edde
GR		 4988 /*
4989 * The VF MAC Address is stored in a packed array of bytes
4990 * starting at the second 32 bit word of the msg array
4991 */
f2ca0dbe
AD		 4992 unsigned char *addr = (char *)&msg[1];
4993 int err = -1;
4ae196df 4994
f2ca0dbe
AD		 4995 if (is_valid_ether_addr(addr))
4996 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4997
f2ca0dbe 4998 return err;
4ae196df
AD		 4999}
5000
5001static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5002{
5003 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5004 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5005 u32 msg = E1000_VT_MSGTYPE_NACK;
5006
5007 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5008 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5009 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5010 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5011 vf_data->last_nack = jiffies;
4ae196df
AD		 5012 }
5013}
5014
f2ca0dbe 5015static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5016{
f2ca0dbe
AD		 5017 struct pci_dev *pdev = adapter->pdev;
5018 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5019 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5020 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5021 s32 retval;
5022
f2ca0dbe 5023 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5024
fef45f4c
AD		 5025 if (retval) {
5026 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5027 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5028 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5029 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5030 return;
5031 goto out;
5032 }
4ae196df
AD		 5033
5034 /* this is a message we already processed, do nothing */
5035 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5036 return;
4ae196df
AD		 5037
5038 /*
5039 * until the vf completes a reset it should not be
5040 * allowed to start any configuration.
5041 */
5042
5043 if (msgbuf[0] == E1000_VF_RESET) {
5044 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5045 return;
4ae196df
AD		 5046 }
5047
f2ca0dbe 5048 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5049 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5050 return;
5051 retval = -1;
5052 goto out;
4ae196df
AD		 5053 }
5054
5055 switch ((msgbuf[0] & 0xFFFF)) {
5056 case E1000_VF_SET_MAC_ADDR:
5057 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5058 break;
7d5753f0
AD		 5059 case E1000_VF_SET_PROMISC:
5060 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5061 break;
4ae196df
AD		 5062 case E1000_VF_SET_MULTICAST:
5063 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5064 break;
5065 case E1000_VF_SET_LPE:
5066 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5067 break;
5068 case E1000_VF_SET_VLAN:
8151d294
WM		 5069 if (adapter->vf_data[vf].pf_vlan)
5070 retval = -1;
5071 else
5072 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5073 break;
5074 default:
090b1795 5075 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5076 retval = -1;
5077 break;
5078 }
5079
fef45f4c
AD		 5080 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5081out:
4ae196df
AD		 5082 /* notify the VF of the results of what it sent us */
5083 if (retval)
5084 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5085 else
5086 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5087
4ae196df 5088 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5089}
4ae196df 5090
f2ca0dbe
AD		 5091static void igb_msg_task(struct igb_adapter *adapter)
5092{
5093 struct e1000_hw *hw = &adapter->hw;
5094 u32 vf;
5095
5096 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5097 /* process any reset requests */
5098 if (!igb_check_for_rst(hw, vf))
5099 igb_vf_reset_event(adapter, vf);
5100
5101 /* process any messages pending */
5102 if (!igb_check_for_msg(hw, vf))
5103 igb_rcv_msg_from_vf(adapter, vf);
5104
5105 /* process any acks */
5106 if (!igb_check_for_ack(hw, vf))
5107 igb_rcv_ack_from_vf(adapter, vf);
5108 }
4ae196df
AD		 5109}
5110
68d480c4
AD		 5111/**
5112 * igb_set_uta - Set unicast filter table address
5113 * @adapter: board private structure
5114 *
5115 * The unicast table address is a register array of 32-bit registers.
5116 * The table is meant to be used in a way similar to how the MTA is used
5117 * however due to certain limitations in the hardware it is necessary to
5118 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5119 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5120 **/
5121static void igb_set_uta(struct igb_adapter *adapter)
5122{
5123 struct e1000_hw *hw = &adapter->hw;
5124 int i;
5125
5126 /* The UTA table only exists on 82576 hardware and newer */
5127 if (hw->mac.type < e1000_82576)
5128 return;
5129
5130 /* we only need to do this if VMDq is enabled */
5131 if (!adapter->vfs_allocated_count)
5132 return;
5133
5134 for (i = 0; i < hw->mac.uta_reg_count; i++)
5135 array_wr32(E1000_UTA, i, ~0);
5136}
5137
9d5c8243
AK		 5138/**
5139 * igb_intr_msi - Interrupt Handler
5140 * @irq: interrupt number
5141 * @data: pointer to a network interface device structure
5142 **/
5143static irqreturn_t igb_intr_msi(int irq, void *data)
5144{
047e0030
AD		 5145 struct igb_adapter *adapter = data;
5146 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5147 struct e1000_hw *hw = &adapter->hw;
5148 /* read ICR disables interrupts using IAM */
5149 u32 icr = rd32(E1000_ICR);
5150
047e0030 5151 igb_write_itr(q_vector);
9d5c8243 5152
7f081d40
AD		 5153 if (icr & E1000_ICR_DRSTA)
5154 schedule_work(&adapter->reset_task);
5155
047e0030 5156 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5157 /* HW is reporting DMA is out of sync */
5158 adapter->stats.doosync++;
5159 }
5160
9d5c8243
AK		 5161 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5162 hw->mac.get_link_status = 1;
5163 if (!test_bit(__IGB_DOWN, &adapter->state))
5164 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5165 }
5166
047e0030 5167 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5168
5169 return IRQ_HANDLED;
5170}
5171
5172/**
4a3c6433 5173 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5174 * @irq: interrupt number
5175 * @data: pointer to a network interface device structure
5176 **/
5177static irqreturn_t igb_intr(int irq, void *data)
5178{
047e0030
AD		 5179 struct igb_adapter *adapter = data;
5180 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5181 struct e1000_hw *hw = &adapter->hw;
5182 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5183 * need for the IMC write */
5184 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5185 if (!icr)
5186 return IRQ_NONE; /* Not our interrupt */
5187
047e0030 5188 igb_write_itr(q_vector);
9d5c8243
AK		 5189
5190 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5191 * not set, then the adapter didn't send an interrupt */
5192 if (!(icr & E1000_ICR_INT_ASSERTED))
5193 return IRQ_NONE;
5194
7f081d40
AD		 5195 if (icr & E1000_ICR_DRSTA)
5196 schedule_work(&adapter->reset_task);
5197
047e0030 5198 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5199 /* HW is reporting DMA is out of sync */
5200 adapter->stats.doosync++;
5201 }
5202
9d5c8243
AK		 5203 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5204 hw->mac.get_link_status = 1;
5205 /* guard against interrupt when we're going down */
5206 if (!test_bit(__IGB_DOWN, &adapter->state))
5207 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5208 }
5209
047e0030 5210 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5211
5212 return IRQ_HANDLED;
5213}
5214
047e0030 5215static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5216{
047e0030 5217 struct igb_adapter *adapter = q_vector->adapter;
46544258 5218 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5219
4fc82adf
AD		 5220 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5221 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5222 if (!adapter->msix_entries)
6eb5a7f1 5223 igb_set_itr(adapter);
46544258 5224 else
047e0030 5225 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5226 }
5227
46544258
AD		 5228 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5229 if (adapter->msix_entries)
047e0030 5230 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5231 else
5232 igb_irq_enable(adapter);
5233 }
9d5c8243
AK		 5234}
5235
46544258
AD		 5236/**
5237 * igb_poll - NAPI Rx polling callback
5238 * @napi: napi polling structure
5239 * @budget: count of how many packets we should handle
5240 **/
5241static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5242{
047e0030
AD		 5243 struct igb_q_vector *q_vector = container_of(napi,
5244 struct igb_q_vector,
5245 napi);
5246 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5247
421e02f0 5248#ifdef CONFIG_IGB_DCA
047e0030
AD		 5249 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5250 igb_update_dca(q_vector);
fe4506b6 5251#endif
047e0030
AD		 5252 if (q_vector->tx_ring)
5253 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5254
047e0030
AD		 5255 if (q_vector->rx_ring)
5256 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5257
5258 if (!tx_clean_complete)
5259 work_done = budget;
46544258 5260
9d5c8243 5261 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5262 if (work_done < budget) {
288379f0 5263 napi_complete(napi);
047e0030 5264 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5265 }
5266
46544258 5267 return work_done;
9d5c8243 5268}
6d8126f9 5269
33af6bcc 5270/**
c5b9bd5e 5271 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5272 * @adapter: board private structure
c5b9bd5e
AD		 5273 * @shhwtstamps: timestamp structure to update
5274 * @regval: unsigned 64bit system time value.
5275 *
5276 * We need to convert the system time value stored in the RX/TXSTMP registers
5277 * into a hwtstamp which can be used by the upper level timestamping functions
5278 */
5279static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5280 struct skb_shared_hwtstamps *shhwtstamps,
5281 u64 regval)
5282{
5283 u64 ns;
5284
55cac248
AD		 5285 /*
5286 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5287 * 24 to match clock shift we setup earlier.
5288 */
5289 if (adapter->hw.mac.type == e1000_82580)
5290 regval <<= IGB_82580_TSYNC_SHIFT;
5291
c5b9bd5e
AD		 5292 ns = timecounter_cyc2time(&adapter->clock, regval);
5293 timecompare_update(&adapter->compare, ns);
5294 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5295 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5296 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5297}
5298
5299/**
5300 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5301 * @q_vector: pointer to q_vector containing needed info
2873957d 5302 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5303 *
5304 * If we were asked to do hardware stamping and such a time stamp is
5305 * available, then it must have been for this skb here because we only
5306 * allow only one such packet into the queue.
5307 */
2873957d 5308static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5309{
c5b9bd5e 5310 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5311 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5312 struct skb_shared_hwtstamps shhwtstamps;
5313 u64 regval;
33af6bcc 5314
c5b9bd5e 5315 /* if skb does not support hw timestamp or TX stamp not valid exit */
2873957d 5316 if (likely(!buffer_info->shtx.hardware) ||
c5b9bd5e
AD		 5317 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5318 return;
5319
5320 regval = rd32(E1000_TXSTMPL);
5321 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5322
5323 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5324 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5325}
5326
9d5c8243
AK		 5327/**
5328 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5329 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5330 * returns true if ring is completely cleaned
5331 **/
047e0030 5332static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5333{
047e0030
AD		 5334 struct igb_adapter *adapter = q_vector->adapter;
5335 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5336 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5337 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5338 struct igb_buffer *buffer_info;
0e014cb1 5339 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5340 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5341 unsigned int i, eop, count = 0;
5342 bool cleaned = false;
9d5c8243 5343
9d5c8243 5344 i = tx_ring->next_to_clean;
0e014cb1
AD		 5345 eop = tx_ring->buffer_info[i].next_to_watch;
5346 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5347
5348 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5349 (count < tx_ring->count)) {
5350 for (cleaned = false; !cleaned; count++) {
5351 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5352 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5353 cleaned = (i == eop);
9d5c8243 5354
2873957d
NN		 5355 if (buffer_info->skb) {
5356 total_bytes += buffer_info->bytecount;
9d5c8243 5357 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5358 total_packets += buffer_info->gso_segs;
5359 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5360 }
5361
80785298 5362 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5363 tx_desc->wb.status = 0;
9d5c8243
AK		 5364
5365 i++;
5366 if (i == tx_ring->count)
5367 i = 0;
9d5c8243 5368 }
0e014cb1
AD		 5369 eop = tx_ring->buffer_info[i].next_to_watch;
5370 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5371 }
5372
9d5c8243
AK		 5373 tx_ring->next_to_clean = i;
5374
fc7d345d 5375 if (unlikely(count &&
9d5c8243 5376 netif_carrier_ok(netdev) &&
c493ea45 5377 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5378 /* Make sure that anybody stopping the queue after this
5379 * sees the new next_to_clean.
5380 */
5381 smp_mb();
661086df
PWJ		 5382 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5383 !(test_bit(__IGB_DOWN, &adapter->state))) {
5384 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5385 tx_ring->tx_stats.restart_queue++;
661086df 5386 }
9d5c8243
AK		 5387 }
5388
5389 if (tx_ring->detect_tx_hung) {
5390 /* Detect a transmit hang in hardware, this serializes the
5391 * check with the clearing of time_stamp and movement of i */
5392 tx_ring->detect_tx_hung = false;
5393 if (tx_ring->buffer_info[i].time_stamp &&
5394 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5395 (adapter->tx_timeout_factor * HZ)) &&
5396 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5397
9d5c8243 5398 /* detected Tx unit hang */
59d71989 5399 dev_err(tx_ring->dev,
9d5c8243 5400 "Detected Tx Unit Hang\n"
2d064c06 5401 " Tx Queue <%d>\n"
9d5c8243
AK		 5402 " TDH <%x>\n"
5403 " TDT <%x>\n"
5404 " next_to_use <%x>\n"
5405 " next_to_clean <%x>\n"
9d5c8243
AK		 5406 "buffer_info[next_to_clean]\n"
5407 " time_stamp <%lx>\n"
0e014cb1 5408 " next_to_watch <%x>\n"
9d5c8243
AK		 5409 " jiffies <%lx>\n"
5410 " desc.status <%x>\n",
2d064c06 5411 tx_ring->queue_index,
fce99e34
AD		 5412 readl(tx_ring->head),
5413 readl(tx_ring->tail),
9d5c8243
AK		 5414 tx_ring->next_to_use,
5415 tx_ring->next_to_clean,
f7ba205e 5416 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5417 eop,
9d5c8243 5418 jiffies,
0e014cb1 5419 eop_desc->wb.status);
661086df 5420 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5421 }
5422 }
5423 tx_ring->total_bytes += total_bytes;
5424 tx_ring->total_packets += total_packets;
e21ed353
AD		 5425 tx_ring->tx_stats.bytes += total_bytes;
5426 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5427 return (count < tx_ring->count);
9d5c8243
AK		 5428}
5429
9d5c8243
AK		 5430/**
5431 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5432 * @q_vector: structure containing interrupt and ring information
5433 * @skb: packet to send up
5434 * @vlan_tag: vlan tag for packet
9d5c8243 5435 **/
047e0030
AD		 5436static void igb_receive_skb(struct igb_q_vector *q_vector,
5437 struct sk_buff *skb,
5438 u16 vlan_tag)
5439{
5440 struct igb_adapter *adapter = q_vector->adapter;
5441
31b24b95 5442 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5443 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5444 vlan_tag, skb);
182ff8df 5445 else
047e0030 5446 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5447}
5448
04a5fcaa 5449static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5450 u32 status_err, struct sk_buff *skb)
5451{
5452 skb->ip_summed = CHECKSUM_NONE;
5453
5454 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5455 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5456 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5457 return;
85ad76b2 5458
9d5c8243
AK		 5459 /* TCP/UDP checksum error bit is set */
5460 if (status_err &
5461 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5462 /*
5463 * work around errata with sctp packets where the TCPE aka
5464 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5465 * packets, (aka let the stack check the crc32c)
5466 */
85ad76b2
AD		 5467 if ((skb->len == 60) &&
5468 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5469 ring->rx_stats.csum_err++;
85ad76b2 5470
9d5c8243 5471 /* let the stack verify checksum errors */
9d5c8243
AK		 5472 return;
5473 }
5474 /* It must be a TCP or UDP packet with a valid checksum */
5475 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5476 skb->ip_summed = CHECKSUM_UNNECESSARY;
5477
59d71989 5478 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5479}
5480
757b77e2 5481static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5482 struct sk_buff *skb)
5483{
5484 struct igb_adapter *adapter = q_vector->adapter;
5485 struct e1000_hw *hw = &adapter->hw;
5486 u64 regval;
5487
5488 /*
5489 * If this bit is set, then the RX registers contain the time stamp. No
5490 * other packet will be time stamped until we read these registers, so
5491 * read the registers to make them available again. Because only one
5492 * packet can be time stamped at a time, we know that the register
5493 * values must belong to this one here and therefore we don't need to
5494 * compare any of the additional attributes stored for it.
5495 *
5496 * If nothing went wrong, then it should have a skb_shared_tx that we
5497 * can turn into a skb_shared_hwtstamps.
5498 */
757b77e2
NN		 5499 if (staterr & E1000_RXDADV_STAT_TSIP) {
5500 u32 *stamp = (u32 *)skb->data;
5501 regval = le32_to_cpu(*(stamp + 2));
5502 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5503 skb_pull(skb, IGB_TS_HDR_LEN);
5504 } else {
5505 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5506 return;
c5b9bd5e 5507
757b77e2
NN		 5508 regval = rd32(E1000_RXSTMPL);
5509 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5510 }
c5b9bd5e
AD		 5511
5512 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5513}
4c844851 5514static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5515 union e1000_adv_rx_desc *rx_desc)
5516{
5517 /* HW will not DMA in data larger than the given buffer, even if it
5518 * parses the (NFS, of course) header to be larger. In that case, it
5519 * fills the header buffer and spills the rest into the page.
5520 */
5521 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5522 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5523 if (hlen > rx_ring->rx_buffer_len)
5524 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5525 return hlen;
5526}
5527
047e0030
AD		 5528static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5529 int *work_done, int budget)
9d5c8243 5530{
047e0030 5531 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5532 struct net_device *netdev = rx_ring->netdev;
59d71989 5533 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5534 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5535 struct igb_buffer *buffer_info , *next_buffer;
5536 struct sk_buff *skb;
9d5c8243
AK		 5537 bool cleaned = false;
5538 int cleaned_count = 0;
d1eff350 5539 int current_node = numa_node_id();
9d5c8243 5540 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5541 unsigned int i;
2d94d8ab
AD		 5542 u32 staterr;
5543 u16 length;
047e0030 5544 u16 vlan_tag;
9d5c8243
AK		 5545
5546 i = rx_ring->next_to_clean;
69d3ca53 5547 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5548 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5549 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5550
5551 while (staterr & E1000_RXD_STAT_DD) {
5552 if (*work_done >= budget)
5553 break;
5554 (*work_done)++;
9d5c8243 5555
69d3ca53
AD		 5556 skb = buffer_info->skb;
5557 prefetch(skb->data - NET_IP_ALIGN);
5558 buffer_info->skb = NULL;
5559
5560 i++;
5561 if (i == rx_ring->count)
5562 i = 0;
42d0781a 5563
69d3ca53
AD		 5564 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5565 prefetch(next_rxd);
5566 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5567
5568 length = le16_to_cpu(rx_desc->wb.upper.length);
5569 cleaned = true;
5570 cleaned_count++;
5571
2d94d8ab 5572 if (buffer_info->dma) {
59d71989 5573 dma_unmap_single(dev, buffer_info->dma,
4c844851 5574 rx_ring->rx_buffer_len,
59d71989 5575 DMA_FROM_DEVICE);
91615f76 5576 buffer_info->dma = 0;
4c844851 5577 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5578 skb_put(skb, length);
5579 goto send_up;
5580 }
4c844851 5581 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5582 }
5583
5584 if (length) {
59d71989
AD		 5585 dma_unmap_page(dev, buffer_info->page_dma,
5586 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5587 buffer_info->page_dma = 0;
bf36c1a0 5588
aa913403 5589 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5590 buffer_info->page,
5591 buffer_info->page_offset,
5592 length);
5593
d1eff350
AD		 5594 if ((page_count(buffer_info->page) != 1) ||
5595 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5596 buffer_info->page = NULL;
5597 else
5598 get_page(buffer_info->page);
9d5c8243
AK		 5599
5600 skb->len += length;
5601 skb->data_len += length;
bf36c1a0 5602 skb->truesize += length;
9d5c8243 5603 }
9d5c8243 5604
bf36c1a0 5605 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5606 buffer_info->skb = next_buffer->skb;
5607 buffer_info->dma = next_buffer->dma;
5608 next_buffer->skb = skb;
5609 next_buffer->dma = 0;
bf36c1a0
AD		 5610 goto next_desc;
5611 }
69d3ca53 5612send_up:
9d5c8243
AK		 5613 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5614 dev_kfree_skb_irq(skb);
5615 goto next_desc;
5616 }
9d5c8243 5617
757b77e2
NN		 5618 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5619 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5620 total_bytes += skb->len;
5621 total_packets++;
5622
85ad76b2 5623 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5624
5625 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5626 skb_record_rx_queue(skb, rx_ring->queue_index);
5627
5628 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5629 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5630
047e0030 5631 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5632
9d5c8243
AK		 5633next_desc:
5634 rx_desc->wb.upper.status_error = 0;
5635
5636 /* return some buffers to hardware, one at a time is too slow */
5637 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5638 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5639 cleaned_count = 0;
5640 }
5641
5642 /* use prefetched values */
5643 rx_desc = next_rxd;
5644 buffer_info = next_buffer;
9d5c8243
AK		 5645 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5646 }
bf36c1a0 5647
9d5c8243 5648 rx_ring->next_to_clean = i;
c493ea45 5649 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5650
5651 if (cleaned_count)
3b644cf6 5652 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5653
5654 rx_ring->total_packets += total_packets;
5655 rx_ring->total_bytes += total_bytes;
5656 rx_ring->rx_stats.packets += total_packets;
5657 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5658 return cleaned;
5659}
5660
9d5c8243
AK		 5661/**
5662 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5663 * @adapter: address of board private structure
5664 **/
d7ee5b3a 5665void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5666{
e694e964 5667 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5668 union e1000_adv_rx_desc *rx_desc;
5669 struct igb_buffer *buffer_info;
5670 struct sk_buff *skb;
5671 unsigned int i;
db761762 5672 int bufsz;
9d5c8243
AK		 5673
5674 i = rx_ring->next_to_use;
5675 buffer_info = &rx_ring->buffer_info[i];
5676
4c844851 5677 bufsz = rx_ring->rx_buffer_len;
db761762 5678
9d5c8243
AK		 5679 while (cleaned_count--) {
5680 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5681
6ec43fe6 5682 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5683 if (!buffer_info->page) {
42d0781a 5684 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5685 if (!buffer_info->page) {
04a5fcaa 5686 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5687 goto no_buffers;
5688 }
5689 buffer_info->page_offset = 0;
5690 } else {
5691 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5692 }
5693 buffer_info->page_dma =
59d71989 5694 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5695 buffer_info->page_offset,
5696 PAGE_SIZE / 2,
59d71989
AD		 5697 DMA_FROM_DEVICE);
5698 if (dma_mapping_error(rx_ring->dev,
5699 buffer_info->page_dma)) {
42d0781a
AD		 5700 buffer_info->page_dma = 0;
5701 rx_ring->rx_stats.alloc_failed++;
5702 goto no_buffers;
5703 }
9d5c8243
AK		 5704 }
5705
42d0781a
AD		 5706 skb = buffer_info->skb;
5707 if (!skb) {
89d71a66 5708 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5709 if (!skb) {
04a5fcaa 5710 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5711 goto no_buffers;
5712 }
5713
9d5c8243 5714 buffer_info->skb = skb;
42d0781a
AD		 5715 }
5716 if (!buffer_info->dma) {
59d71989 5717 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5718 skb->data,
9d5c8243 5719 bufsz,
59d71989
AD		 5720 DMA_FROM_DEVICE);
5721 if (dma_mapping_error(rx_ring->dev,
5722 buffer_info->dma)) {
42d0781a
AD		 5723 buffer_info->dma = 0;
5724 rx_ring->rx_stats.alloc_failed++;
5725 goto no_buffers;
5726 }
9d5c8243
AK		 5727 }
5728 /* Refresh the desc even if buffer_addrs didn't change because
5729 * each write-back erases this info. */
6ec43fe6 5730 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5731 rx_desc->read.pkt_addr =
5732 cpu_to_le64(buffer_info->page_dma);
5733 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5734 } else {
42d0781a 5735 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5736 rx_desc->read.hdr_addr = 0;
5737 }
5738
5739 i++;
5740 if (i == rx_ring->count)
5741 i = 0;
5742 buffer_info = &rx_ring->buffer_info[i];
5743 }
5744
5745no_buffers:
5746 if (rx_ring->next_to_use != i) {
5747 rx_ring->next_to_use = i;
5748 if (i == 0)
5749 i = (rx_ring->count - 1);
5750 else
5751 i--;
5752
5753 /* Force memory writes to complete before letting h/w
5754 * know there are new descriptors to fetch. (Only
5755 * applicable for weak-ordered memory model archs,
5756 * such as IA-64). */
5757 wmb();
fce99e34 5758 writel(i, rx_ring->tail);
9d5c8243
AK		 5759 }
5760}
5761
5762/**
5763 * igb_mii_ioctl -
5764 * @netdev:
5765 * @ifreq:
5766 * @cmd:
5767 **/
5768static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5769{
5770 struct igb_adapter *adapter = netdev_priv(netdev);
5771 struct mii_ioctl_data *data = if_mii(ifr);
5772
5773 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5774 return -EOPNOTSUPP;
5775
5776 switch (cmd) {
5777 case SIOCGMIIPHY:
5778 data->phy_id = adapter->hw.phy.addr;
5779 break;
5780 case SIOCGMIIREG:
f5f4cf08
AD		 5781 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5782 &data->val_out))
9d5c8243
AK		 5783 return -EIO;
5784 break;
5785 case SIOCSMIIREG:
5786 default:
5787 return -EOPNOTSUPP;
5788 }
5789 return 0;
5790}
5791
c6cb090b
PO		 5792/**
5793 * igb_hwtstamp_ioctl - control hardware time stamping
5794 * @netdev:
5795 * @ifreq:
5796 * @cmd:
5797 *
33af6bcc
PO		 5798 * Outgoing time stamping can be enabled and disabled. Play nice and
5799 * disable it when requested, although it shouldn't case any overhead
5800 * when no packet needs it. At most one packet in the queue may be
5801 * marked for time stamping, otherwise it would be impossible to tell
5802 * for sure to which packet the hardware time stamp belongs.
5803 *
5804 * Incoming time stamping has to be configured via the hardware
5805 * filters. Not all combinations are supported, in particular event
5806 * type has to be specified. Matching the kind of event packet is
5807 * not supported, with the exception of "all V2 events regardless of
5808 * level 2 or 4".
5809 *
c6cb090b
PO		 5810 **/
5811static int igb_hwtstamp_ioctl(struct net_device *netdev,
5812 struct ifreq *ifr, int cmd)
5813{
33af6bcc
PO		 5814 struct igb_adapter *adapter = netdev_priv(netdev);
5815 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5816 struct hwtstamp_config config;
c5b9bd5e
AD		 5817 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5818 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5819 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5820 bool is_l4 = false;
5821 bool is_l2 = false;
33af6bcc 5822 u32 regval;
c6cb090b
PO		 5823
5824 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5825 return -EFAULT;
5826
5827 /* reserved for future extensions */
5828 if (config.flags)
5829 return -EINVAL;
5830
33af6bcc
PO		 5831 switch (config.tx_type) {
5832 case HWTSTAMP_TX_OFF:
c5b9bd5e 5833 tsync_tx_ctl = 0;
33af6bcc 5834 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5835 break;
5836 default:
5837 return -ERANGE;
5838 }
5839
5840 switch (config.rx_filter) {
5841 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5842 tsync_rx_ctl = 0;
33af6bcc
PO		 5843 break;
5844 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5845 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5846 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5847 case HWTSTAMP_FILTER_ALL:
5848 /*
5849 * register TSYNCRXCFG must be set, therefore it is not
5850 * possible to time stamp both Sync and Delay_Req messages
5851 * => fall back to time stamping all packets
5852 */
c5b9bd5e 5853 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5854 config.rx_filter = HWTSTAMP_FILTER_ALL;
5855 break;
5856 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5857 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5858 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5859 is_l4 = true;
33af6bcc
PO		 5860 break;
5861 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5862 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5863 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5864 is_l4 = true;
33af6bcc
PO		 5865 break;
5866 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5867 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5868 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5869 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5870 is_l2 = true;
5871 is_l4 = true;
33af6bcc
PO		 5872 config.rx_filter = HWTSTAMP_FILTER_SOME;
5873 break;
5874 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5875 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5876 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5877 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5878 is_l2 = true;
5879 is_l4 = true;
33af6bcc
PO		 5880 config.rx_filter = HWTSTAMP_FILTER_SOME;
5881 break;
5882 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5883 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5884 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5885 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5886 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5887 is_l2 = true;
33af6bcc
PO		 5888 break;
5889 default:
5890 return -ERANGE;
5891 }
5892
c5b9bd5e
AD		 5893 if (hw->mac.type == e1000_82575) {
5894 if (tsync_rx_ctl | tsync_tx_ctl)
5895 return -EINVAL;
5896 return 0;
5897 }
5898
757b77e2
NN		 5899 /*
5900 * Per-packet timestamping only works if all packets are
5901 * timestamped, so enable timestamping in all packets as
5902 * long as one rx filter was configured.
5903 */
5904 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5905 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5906 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5907 }
5908
33af6bcc
PO		 5909 /* enable/disable TX */
5910 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5911 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5912 regval |= tsync_tx_ctl;
33af6bcc
PO		 5913 wr32(E1000_TSYNCTXCTL, regval);
5914
c5b9bd5e 5915 /* enable/disable RX */
33af6bcc 5916 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5917 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5918 regval |= tsync_rx_ctl;
33af6bcc 5919 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5920
c5b9bd5e
AD		 5921 /* define which PTP packets are time stamped */
5922 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5923
c5b9bd5e
AD		 5924 /* define ethertype filter for timestamped packets */
5925 if (is_l2)
5926 wr32(E1000_ETQF(3),
5927 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5928 E1000_ETQF_1588 | /* enable timestamping */
5929 ETH_P_1588)); /* 1588 eth protocol type */
5930 else
5931 wr32(E1000_ETQF(3), 0);
5932
5933#define PTP_PORT 319
5934 /* L4 Queue Filter[3]: filter by destination port and protocol */
5935 if (is_l4) {
5936 u32 ftqf = (IPPROTO_UDP /* UDP */
5937 | E1000_FTQF_VF_BP /* VF not compared */
5938 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5939 | E1000_FTQF_MASK); /* mask all inputs */
5940 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5941
5942 wr32(E1000_IMIR(3), htons(PTP_PORT));
5943 wr32(E1000_IMIREXT(3),
5944 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5945 if (hw->mac.type == e1000_82576) {
5946 /* enable source port check */
5947 wr32(E1000_SPQF(3), htons(PTP_PORT));
5948 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5949 }
5950 wr32(E1000_FTQF(3), ftqf);
5951 } else {
5952 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5953 }
33af6bcc
PO		 5954 wrfl();
5955
5956 adapter->hwtstamp_config = config;
5957
5958 /* clear TX/RX time stamp registers, just to be sure */
5959 regval = rd32(E1000_TXSTMPH);
5960 regval = rd32(E1000_RXSTMPH);
c6cb090b 5961
33af6bcc
PO		 5962 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5963 -EFAULT : 0;
c6cb090b
PO		 5964}
5965
9d5c8243
AK		 5966/**
5967 * igb_ioctl -
5968 * @netdev:
5969 * @ifreq:
5970 * @cmd:
5971 **/
5972static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5973{
5974 switch (cmd) {
5975 case SIOCGMIIPHY:
5976 case SIOCGMIIREG:
5977 case SIOCSMIIREG:
5978 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5979 case SIOCSHWTSTAMP:
5980 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5981 default:
5982 return -EOPNOTSUPP;
5983 }
5984}
5985
009bc06e
AD		 5986s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5987{
5988 struct igb_adapter *adapter = hw->back;
5989 u16 cap_offset;
5990
5991 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5992 if (!cap_offset)
5993 return -E1000_ERR_CONFIG;
5994
5995 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5996
5997 return 0;
5998}
5999
6000s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6001{
6002 struct igb_adapter *adapter = hw->back;
6003 u16 cap_offset;
6004
6005 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6006 if (!cap_offset)
6007 return -E1000_ERR_CONFIG;
6008
6009 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6010
6011 return 0;
6012}
6013
9d5c8243
AK		 6014static void igb_vlan_rx_register(struct net_device *netdev,
6015 struct vlan_group *grp)
6016{
6017 struct igb_adapter *adapter = netdev_priv(netdev);
6018 struct e1000_hw *hw = &adapter->hw;
6019 u32 ctrl, rctl;
6020
6021 igb_irq_disable(adapter);
6022 adapter->vlgrp = grp;
6023
6024 if (grp) {
6025 /* enable VLAN tag insert/strip */
6026 ctrl = rd32(E1000_CTRL);
6027 ctrl |= E1000_CTRL_VME;
6028 wr32(E1000_CTRL, ctrl);
6029
51466239 6030 /* Disable CFI check */
9d5c8243 6031 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6032 rctl &= ~E1000_RCTL_CFIEN;
6033 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6034 } else {
6035 /* disable VLAN tag insert/strip */
6036 ctrl = rd32(E1000_CTRL);
6037 ctrl &= ~E1000_CTRL_VME;
6038 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6039 }
6040
e1739522
AD		 6041 igb_rlpml_set(adapter);
6042
9d5c8243
AK		 6043 if (!test_bit(__IGB_DOWN, &adapter->state))
6044 igb_irq_enable(adapter);
6045}
6046
6047static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6048{
6049 struct igb_adapter *adapter = netdev_priv(netdev);
6050 struct e1000_hw *hw = &adapter->hw;
4ae196df 6051 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6052
51466239
AD		 6053 /* attempt to add filter to vlvf array */
6054 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6055
51466239
AD		 6056 /* add the filter since PF can receive vlans w/o entry in vlvf */
6057 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6058}
6059
6060static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6061{
6062 struct igb_adapter *adapter = netdev_priv(netdev);
6063 struct e1000_hw *hw = &adapter->hw;
4ae196df 6064 int pf_id = adapter->vfs_allocated_count;
51466239 6065 s32 err;
9d5c8243
AK		 6066
6067 igb_irq_disable(adapter);
6068 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6069
6070 if (!test_bit(__IGB_DOWN, &adapter->state))
6071 igb_irq_enable(adapter);
6072
51466239
AD		 6073 /* remove vlan from VLVF table array */
6074 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6075
51466239
AD		 6076 /* if vid was not present in VLVF just remove it from table */
6077 if (err)
4ae196df 6078 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6079}
6080
6081static void igb_restore_vlan(struct igb_adapter *adapter)
6082{
6083 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6084
6085 if (adapter->vlgrp) {
6086 u16 vid;
6087 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6088 if (!vlan_group_get_device(adapter->vlgrp, vid))
6089 continue;
6090 igb_vlan_rx_add_vid(adapter->netdev, vid);
6091 }
6092 }
6093}
6094
6095int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6096{
090b1795 6097 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6098 struct e1000_mac_info *mac = &adapter->hw.mac;
6099
6100 mac->autoneg = 0;
6101
9d5c8243
AK		 6102 switch (spddplx) {
6103 case SPEED_10 + DUPLEX_HALF:
6104 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6105 break;
6106 case SPEED_10 + DUPLEX_FULL:
6107 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6108 break;
6109 case SPEED_100 + DUPLEX_HALF:
6110 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6111 break;
6112 case SPEED_100 + DUPLEX_FULL:
6113 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6114 break;
6115 case SPEED_1000 + DUPLEX_FULL:
6116 mac->autoneg = 1;
6117 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6118 break;
6119 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6120 default:
090b1795 6121 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6122 return -EINVAL;
6123 }
6124 return 0;
6125}
6126
3fe7c4c9 6127static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6128{
6129 struct net_device *netdev = pci_get_drvdata(pdev);
6130 struct igb_adapter *adapter = netdev_priv(netdev);
6131 struct e1000_hw *hw = &adapter->hw;
2d064c06 6132 u32 ctrl, rctl, status;
9d5c8243
AK		 6133 u32 wufc = adapter->wol;
6134#ifdef CONFIG_PM
6135 int retval = 0;
6136#endif
6137
6138 netif_device_detach(netdev);
6139
a88f10ec
AD		 6140 if (netif_running(netdev))
6141 igb_close(netdev);
6142
047e0030 6143 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6144
6145#ifdef CONFIG_PM
6146 retval = pci_save_state(pdev);
6147 if (retval)
6148 return retval;
6149#endif
6150
6151 status = rd32(E1000_STATUS);
6152 if (status & E1000_STATUS_LU)
6153 wufc &= ~E1000_WUFC_LNKC;
6154
6155 if (wufc) {
6156 igb_setup_rctl(adapter);
ff41f8dc 6157 igb_set_rx_mode(netdev);
9d5c8243
AK		 6158
6159 /* turn on all-multi mode if wake on multicast is enabled */
6160 if (wufc & E1000_WUFC_MC) {
6161 rctl = rd32(E1000_RCTL);
6162 rctl |= E1000_RCTL_MPE;
6163 wr32(E1000_RCTL, rctl);
6164 }
6165
6166 ctrl = rd32(E1000_CTRL);
6167 /* advertise wake from D3Cold */
6168 #define E1000_CTRL_ADVD3WUC 0x00100000
6169 /* phy power management enable */
6170 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6171 ctrl |= E1000_CTRL_ADVD3WUC;
6172 wr32(E1000_CTRL, ctrl);
6173
9d5c8243 6174 /* Allow time for pending master requests to run */
330a6d6a 6175 igb_disable_pcie_master(hw);
9d5c8243
AK		 6176
6177 wr32(E1000_WUC, E1000_WUC_PME_EN);
6178 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6179 } else {
6180 wr32(E1000_WUC, 0);
6181 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6182 }
6183
3fe7c4c9
RW		 6184 *enable_wake = wufc || adapter->en_mng_pt;
6185 if (!*enable_wake)
88a268c1
NN		 6186 igb_power_down_link(adapter);
6187 else
6188 igb_power_up_link(adapter);
9d5c8243
AK		 6189
6190 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6191 * would have already happened in close and is redundant. */
6192 igb_release_hw_control(adapter);
6193
6194 pci_disable_device(pdev);
6195
9d5c8243
AK		 6196 return 0;
6197}
6198
6199#ifdef CONFIG_PM
3fe7c4c9
RW		 6200static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6201{
6202 int retval;
6203 bool wake;
6204
6205 retval = __igb_shutdown(pdev, &wake);
6206 if (retval)
6207 return retval;
6208
6209 if (wake) {
6210 pci_prepare_to_sleep(pdev);
6211 } else {
6212 pci_wake_from_d3(pdev, false);
6213 pci_set_power_state(pdev, PCI_D3hot);
6214 }
6215
6216 return 0;
6217}
6218
9d5c8243
AK		 6219static int igb_resume(struct pci_dev *pdev)
6220{
6221 struct net_device *netdev = pci_get_drvdata(pdev);
6222 struct igb_adapter *adapter = netdev_priv(netdev);
6223 struct e1000_hw *hw = &adapter->hw;
6224 u32 err;
6225
6226 pci_set_power_state(pdev, PCI_D0);
6227 pci_restore_state(pdev);
b94f2d77 6228 pci_save_state(pdev);
42bfd33a 6229
aed5dec3 6230 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6231 if (err) {
6232 dev_err(&pdev->dev,
6233 "igb: Cannot enable PCI device from suspend\n");
6234 return err;
6235 }
6236 pci_set_master(pdev);
6237
6238 pci_enable_wake(pdev, PCI_D3hot, 0);
6239 pci_enable_wake(pdev, PCI_D3cold, 0);
6240
047e0030 6241 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6242 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6243 return -ENOMEM;
9d5c8243
AK		 6244 }
6245
9d5c8243 6246 igb_reset(adapter);
a8564f03
AD		 6247
6248 /* let the f/w know that the h/w is now under the control of the
6249 * driver. */
6250 igb_get_hw_control(adapter);
6251
9d5c8243
AK		 6252 wr32(E1000_WUS, ~0);
6253
a88f10ec
AD		 6254 if (netif_running(netdev)) {
6255 err = igb_open(netdev);
6256 if (err)
6257 return err;
6258 }
9d5c8243
AK		 6259
6260 netif_device_attach(netdev);
6261
9d5c8243
AK		 6262 return 0;
6263}
6264#endif
6265
6266static void igb_shutdown(struct pci_dev *pdev)
6267{
3fe7c4c9
RW		 6268 bool wake;
6269
6270 __igb_shutdown(pdev, &wake);
6271
6272 if (system_state == SYSTEM_POWER_OFF) {
6273 pci_wake_from_d3(pdev, wake);
6274 pci_set_power_state(pdev, PCI_D3hot);
6275 }
9d5c8243
AK		 6276}
6277
6278#ifdef CONFIG_NET_POLL_CONTROLLER
6279/*
6280 * Polling 'interrupt' - used by things like netconsole to send skbs
6281 * without having to re-enable interrupts. It's not called while
6282 * the interrupt routine is executing.
6283 */
6284static void igb_netpoll(struct net_device *netdev)
6285{
6286 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6287 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6288 int i;
9d5c8243 6289
eebbbdba 6290 if (!adapter->msix_entries) {
047e0030 6291 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6292 igb_irq_disable(adapter);
047e0030 6293 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6294 return;
6295 }
9d5c8243 6296
047e0030
AD		 6297 for (i = 0; i < adapter->num_q_vectors; i++) {
6298 struct igb_q_vector *q_vector = adapter->q_vector[i];
6299 wr32(E1000_EIMC, q_vector->eims_value);
6300 napi_schedule(&q_vector->napi);
eebbbdba 6301 }
9d5c8243
AK		 6302}
6303#endif /* CONFIG_NET_POLL_CONTROLLER */
6304
6305/**
6306 * igb_io_error_detected - called when PCI error is detected
6307 * @pdev: Pointer to PCI device
6308 * @state: The current pci connection state
6309 *
6310 * This function is called after a PCI bus error affecting
6311 * this device has been detected.
6312 */
6313static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6314 pci_channel_state_t state)
6315{
6316 struct net_device *netdev = pci_get_drvdata(pdev);
6317 struct igb_adapter *adapter = netdev_priv(netdev);
6318
6319 netif_device_detach(netdev);
6320
59ed6eec
AD		 6321 if (state == pci_channel_io_perm_failure)
6322 return PCI_ERS_RESULT_DISCONNECT;
6323
9d5c8243
AK		 6324 if (netif_running(netdev))
6325 igb_down(adapter);
6326 pci_disable_device(pdev);
6327
6328 /* Request a slot slot reset. */
6329 return PCI_ERS_RESULT_NEED_RESET;
6330}
6331
6332/**
6333 * igb_io_slot_reset - called after the pci bus has been reset.
6334 * @pdev: Pointer to PCI device
6335 *
6336 * Restart the card from scratch, as if from a cold-boot. Implementation
6337 * resembles the first-half of the igb_resume routine.
6338 */
6339static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6340{
6341 struct net_device *netdev = pci_get_drvdata(pdev);
6342 struct igb_adapter *adapter = netdev_priv(netdev);
6343 struct e1000_hw *hw = &adapter->hw;
40a914fa 6344 pci_ers_result_t result;
42bfd33a 6345 int err;
9d5c8243 6346
aed5dec3 6347 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6348 dev_err(&pdev->dev,
6349 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6350 result = PCI_ERS_RESULT_DISCONNECT;
6351 } else {
6352 pci_set_master(pdev);
6353 pci_restore_state(pdev);
b94f2d77 6354 pci_save_state(pdev);
9d5c8243 6355
40a914fa
AD		 6356 pci_enable_wake(pdev, PCI_D3hot, 0);
6357 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6358
40a914fa
AD		 6359 igb_reset(adapter);
6360 wr32(E1000_WUS, ~0);
6361 result = PCI_ERS_RESULT_RECOVERED;
6362 }
9d5c8243 6363
ea943d41
JK		 6364 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6365 if (err) {
6366 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6367 "failed 0x%0x\n", err);
6368 /* non-fatal, continue */
6369 }
40a914fa
AD		 6370
6371 return result;
9d5c8243
AK		 6372}
6373
6374/**
6375 * igb_io_resume - called when traffic can start flowing again.
6376 * @pdev: Pointer to PCI device
6377 *
6378 * This callback is called when the error recovery driver tells us that
6379 * its OK to resume normal operation. Implementation resembles the
6380 * second-half of the igb_resume routine.
6381 */
6382static void igb_io_resume(struct pci_dev *pdev)
6383{
6384 struct net_device *netdev = pci_get_drvdata(pdev);
6385 struct igb_adapter *adapter = netdev_priv(netdev);
6386
9d5c8243
AK		 6387 if (netif_running(netdev)) {
6388 if (igb_up(adapter)) {
6389 dev_err(&pdev->dev, "igb_up failed after reset\n");
6390 return;
6391 }
6392 }
6393
6394 netif_device_attach(netdev);
6395
6396 /* let the f/w know that the h/w is now under the control of the
6397 * driver. */
6398 igb_get_hw_control(adapter);
9d5c8243
AK		 6399}
6400
26ad9178
AD		 6401static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6402 u8 qsel)
6403{
6404 u32 rar_low, rar_high;
6405 struct e1000_hw *hw = &adapter->hw;
6406
6407 /* HW expects these in little endian so we reverse the byte order
6408 * from network order (big endian) to little endian
6409 */
6410 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6411 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6412 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6413
6414 /* Indicate to hardware the Address is Valid. */
6415 rar_high |= E1000_RAH_AV;
6416
6417 if (hw->mac.type == e1000_82575)
6418 rar_high |= E1000_RAH_POOL_1 * qsel;
6419 else
6420 rar_high |= E1000_RAH_POOL_1 << qsel;
6421
6422 wr32(E1000_RAL(index), rar_low);
6423 wrfl();
6424 wr32(E1000_RAH(index), rar_high);
6425 wrfl();
6426}
6427
4ae196df
AD		 6428static int igb_set_vf_mac(struct igb_adapter *adapter,
6429 int vf, unsigned char *mac_addr)
6430{
6431 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6432 /* VF MAC addresses start at end of receive addresses and moves
6433 * torwards the first, as a result a collision should not be possible */
6434 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6435
37680117 6436 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6437
26ad9178 6438 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6439
6440 return 0;
6441}
6442
8151d294
WM		 6443static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6444{
6445 struct igb_adapter *adapter = netdev_priv(netdev);
6446 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6447 return -EINVAL;
6448 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6449 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6450 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6451 " change effective.");
6452 if (test_bit(__IGB_DOWN, &adapter->state)) {
6453 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6454 " but the PF device is not up.\n");
6455 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6456 " attempting to use the VF device.\n");
6457 }
6458 return igb_set_vf_mac(adapter, vf, mac);
6459}
6460
6461static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6462{
6463 return -EOPNOTSUPP;
6464}
6465
6466static int igb_ndo_get_vf_config(struct net_device *netdev,
6467 int vf, struct ifla_vf_info *ivi)
6468{
6469 struct igb_adapter *adapter = netdev_priv(netdev);
6470 if (vf >= adapter->vfs_allocated_count)
6471 return -EINVAL;
6472 ivi->vf = vf;
6473 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6474 ivi->tx_rate = 0;
6475 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6476 ivi->qos = adapter->vf_data[vf].pf_qos;
6477 return 0;
6478}
6479
4ae196df
AD		 6480static void igb_vmm_control(struct igb_adapter *adapter)
6481{
6482 struct e1000_hw *hw = &adapter->hw;
10d8e907 6483 u32 reg;
4ae196df 6484
52a1dd4d
AD		 6485 switch (hw->mac.type) {
6486 case e1000_82575:
6487 default:
6488 /* replication is not supported for 82575 */
4ae196df 6489 return;
52a1dd4d
AD		 6490 case e1000_82576:
6491 /* notify HW that the MAC is adding vlan tags */
6492 reg = rd32(E1000_DTXCTL);
6493 reg |= E1000_DTXCTL_VLAN_ADDED;
6494 wr32(E1000_DTXCTL, reg);
6495 case e1000_82580:
6496 /* enable replication vlan tag stripping */
6497 reg = rd32(E1000_RPLOLR);
6498 reg |= E1000_RPLOLR_STRVLAN;
6499 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6500 case e1000_i350:
6501 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6502 break;
6503 }
10d8e907 6504
d4960307
AD		 6505 if (adapter->vfs_allocated_count) {
6506 igb_vmdq_set_loopback_pf(hw, true);
6507 igb_vmdq_set_replication_pf(hw, true);
6508 } else {
6509 igb_vmdq_set_loopback_pf(hw, false);
6510 igb_vmdq_set_replication_pf(hw, false);
6511 }
4ae196df
AD		 6512}
6513
9d5c8243 6514/* igb_main.c */
Linux 6.x block layer and io_uring tree(s)