projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbe: Break out Rx buffer page management
[linux-2.6-block.git] / drivers / net / ethernet / intel / ixgbe / ixgbe_main.c
CommitLineData
9a799d71
AK		 1/*******************************************************************************
2
3 Intel 10 Gigabit PCI Express Linux driver
37689010 4 Copyright(c) 1999 - 2016 Intel Corporation.
9a799d71
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
b89aae71 23 Linux NICS <linux.nics@intel.com>
9a799d71
AK		 24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27*******************************************************************************/
28
29#include <linux/types.h>
30#include <linux/module.h>
31#include <linux/pci.h>
32#include <linux/netdevice.h>
33#include <linux/vmalloc.h>
34#include <linux/string.h>
35#include <linux/in.h>
a6b7a407 36#include <linux/interrupt.h>
9a799d71
AK		 37#include <linux/ip.h>
38#include <linux/tcp.h>
897ab156 39#include <linux/sctp.h>
60127865 40#include <linux/pkt_sched.h>
9a799d71 41#include <linux/ipv6.h>
5a0e3ad6 42#include <linux/slab.h>
9a799d71
AK		 43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
c762dff2 45#include <linux/etherdevice.h>
9a799d71 46#include <linux/ethtool.h>
01789349 47#include <linux/if.h>
9a799d71 48#include <linux/if_vlan.h>
2a47fa45 49#include <linux/if_macvlan.h>
815cccbf 50#include <linux/if_bridge.h>
70c71606 51#include <linux/prefetch.h>
eacd73f7 52#include <scsi/fc/fc_fcoe.h>
b3a49557 53#include <net/udp_tunnel.h>
b82b17d9
JF		 54#include <net/pkt_cls.h>
55#include <net/tc_act/tc_gact.h>
947f8a45 56#include <net/tc_act/tc_mirred.h>
9f3c7504 57#include <net/vxlan.h>
9a799d71
AK		 58
59#include "ixgbe.h"
60#include "ixgbe_common.h"
ee5f784a 61#include "ixgbe_dcb_82599.h"
1cdd1ec8 62#include "ixgbe_sriov.h"
b82b17d9 63#include "ixgbe_model.h"
9a799d71
AK		 64
65char ixgbe_driver_name[] = "ixgbe";
9c8eb720 66static const char ixgbe_driver_string[] =
e8e9f696 67 "Intel(R) 10 Gigabit PCI Express Network Driver";
8af3c33f 68#ifdef IXGBE_FCOE
ea81875a
NP		 69char ixgbe_default_device_descr[] =
70 "Intel(R) 10 Gigabit Network Connection";
8af3c33f
JK		 71#else
72static char ixgbe_default_device_descr[] =
73 "Intel(R) 10 Gigabit Network Connection";
74#endif
1733284d 75#define DRV_VERSION "5.0.0-k"
9c8eb720 76const char ixgbe_driver_version[] = DRV_VERSION;
a52055e0 77static const char ixgbe_copyright[] =
49425dfc 78 "Copyright (c) 1999-2016 Intel Corporation.";
9a799d71 79
f44e751b
DS		 80static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";
81
9a799d71 82static const struct ixgbe_info *ixgbe_info_tbl[] = {
6a14ee0c
DS		 83 [board_82598] = &ixgbe_82598_info,
84 [board_82599] = &ixgbe_82599_info,
85 [board_X540] = &ixgbe_X540_info,
86 [board_X550] = &ixgbe_X550_info,
87 [board_X550EM_x] = &ixgbe_X550EM_x_info,
49425dfc 88 [board_x550em_a] = &ixgbe_x550em_a_info,
b3eb4e18 89 [board_x550em_a_fw] = &ixgbe_x550em_a_fw_info,
9a799d71
AK		 90};
91
92/* ixgbe_pci_tbl - PCI Device ID Table
93 *
94 * Wildcard entries (PCI_ANY_ID) should come last
95 * Last entry must be all 0s
96 *
97 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
98 * Class, Class Mask, private data (not used) }
99 */
9baa3c34 100static const struct pci_device_id ixgbe_pci_tbl[] = {
54239c67
AD		 101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
116 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
117 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
118 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
119 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
120 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
121 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
122 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
123 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
124 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
125 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
126 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
8f58332b 127 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
7d145282 128 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
9e791e4a 129 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
df376f0d 130 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
6a14ee0c 131 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
a711ad89 132 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
6a14ee0c
DS		 133 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
134 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
deda562a 135 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
018d7146 136 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
f572b2c4
MR		 137 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
138 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
49425dfc 139 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
200157c2
MR		 140 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
141 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
92ed8430 142 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
2d40cd17 143 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
b3eb4e18
MR		 144 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },
145 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
9a799d71
AK		 146 /* required last entry */
147 {0, }
148};
149MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
150
5dd2d332 151#ifdef CONFIG_IXGBE_DCA
bd0362dd 152static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
e8e9f696 153 void *p);
bd0362dd
JC		 154static struct notifier_block dca_notifier = {
155 .notifier_call = ixgbe_notify_dca,
156 .next = NULL,
157 .priority = 0
158};
159#endif
160
1cdd1ec8
GR		 161#ifdef CONFIG_PCI_IOV
162static unsigned int max_vfs;
163module_param(max_vfs, uint, 0);
e8e9f696 164MODULE_PARM_DESC(max_vfs,
170e8543 165 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
1cdd1ec8
GR		 166#endif /* CONFIG_PCI_IOV */
167
8ef78adc
PWJ		 168static unsigned int allow_unsupported_sfp;
169module_param(allow_unsupported_sfp, uint, 0);
170MODULE_PARM_DESC(allow_unsupported_sfp,
171 "Allow unsupported and untested SFP+ modules on 82599-based adapters");
172
b3f4d599 173#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
174static int debug = -1;
175module_param(debug, int, 0);
176MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
177
9a799d71
AK		 178MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
179MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
180MODULE_LICENSE("GPL");
181MODULE_VERSION(DRV_VERSION);
182
780484d8
MR		 183static struct workqueue_struct *ixgbe_wq;
184
14438464 185static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
b3eb4e18 186static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);
14438464 187
b8e82001
JK		 188static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
189 u32 reg, u16 *value)
190{
b8e82001
JK		 191 struct pci_dev *parent_dev;
192 struct pci_bus *parent_bus;
193
194 parent_bus = adapter->pdev->bus->parent;
195 if (!parent_bus)
196 return -1;
197
198 parent_dev = parent_bus->self;
199 if (!parent_dev)
200 return -1;
201
c0798edf 202 if (!pci_is_pcie(parent_dev))
b8e82001
JK		 203 return -1;
204
c0798edf 205 pcie_capability_read_word(parent_dev, reg, value);
14438464
MR		 206 if (*value == IXGBE_FAILED_READ_CFG_WORD &&
207 ixgbe_check_cfg_remove(&adapter->hw, parent_dev))
208 return -1;
b8e82001
JK		 209 return 0;
210}
211
212static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
213{
214 struct ixgbe_hw *hw = &adapter->hw;
215 u16 link_status = 0;
216 int err;
217
218 hw->bus.type = ixgbe_bus_type_pci_express;
219
220 /* Get the negotiated link width and speed from PCI config space of the
221 * parent, as this device is behind a switch
222 */
223 err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status);
224
225 /* assume caller will handle error case */
226 if (err)
227 return err;
228
229 hw->bus.width = ixgbe_convert_bus_width(link_status);
230 hw->bus.speed = ixgbe_convert_bus_speed(link_status);
231
232 return 0;
233}
234
e027d1ae
JK		 235/**
236 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent
237 * @hw: hw specific details
238 *
239 * This function is used by probe to determine whether a device's PCI-Express
240 * bandwidth details should be gathered from the parent bus instead of from the
241 * device. Used to ensure that various locations all have the correct device ID
242 * checks.
243 */
244static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
245{
246 switch (hw->device_id) {
247 case IXGBE_DEV_ID_82599_SFP_SF_QP:
8f58332b 248 case IXGBE_DEV_ID_82599_QSFP_SF_QP:
e027d1ae
JK		 249 return true;
250 default:
251 return false;
252 }
253}
254
255static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
256 int expected_gts)
257{
f9328bc6 258 struct ixgbe_hw *hw = &adapter->hw;
e027d1ae
JK		 259 int max_gts = 0;
260 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
261 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
262 struct pci_dev *pdev;
263
f9328bc6
DS		 264 /* Some devices are not connected over PCIe and thus do not negotiate
265 * speed. These devices do not have valid bus info, and thus any report
266 * we generate may not be correct.
267 */
268 if (hw->bus.type == ixgbe_bus_type_internal)
269 return;
270
56d1392f 271 /* determine whether to use the parent device */
e027d1ae
JK		 272 if (ixgbe_pcie_from_parent(&adapter->hw))
273 pdev = adapter->pdev->bus->parent->self;
274 else
275 pdev = adapter->pdev;
276
277 if (pcie_get_minimum_link(pdev, &speed, &width) ||
278 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) {
279 e_dev_warn("Unable to determine PCI Express bandwidth.\n");
280 return;
281 }
282
283 switch (speed) {
284 case PCIE_SPEED_2_5GT:
285 /* 8b/10b encoding reduces max throughput by 20% */
286 max_gts = 2 * width;
287 break;
288 case PCIE_SPEED_5_0GT:
289 /* 8b/10b encoding reduces max throughput by 20% */
290 max_gts = 4 * width;
291 break;
292 case PCIE_SPEED_8_0GT:
9f0a433c 293 /* 128b/130b encoding reduces throughput by less than 2% */
e027d1ae
JK		 294 max_gts = 8 * width;
295 break;
296 default:
297 e_dev_warn("Unable to determine PCI Express bandwidth.\n");
298 return;
299 }
300
301 e_dev_info("PCI Express bandwidth of %dGT/s available\n",
302 max_gts);
303 e_dev_info("(Speed:%s, Width: x%d, Encoding Loss:%s)\n",
304 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
305 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
306 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
307 "Unknown"),
308 width,
309 (speed == PCIE_SPEED_2_5GT ? "20%" :
310 speed == PCIE_SPEED_5_0GT ? "20%" :
9f0a433c 311 speed == PCIE_SPEED_8_0GT ? "<2%" :
e027d1ae
JK		 312 "Unknown"));
313
314 if (max_gts < expected_gts) {
315 e_dev_warn("This is not sufficient for optimal performance of this card.\n");
316 e_dev_warn("For optimal performance, at least %dGT/s of bandwidth is required.\n",
317 expected_gts);
318 e_dev_warn("A slot with more lanes and/or higher speed is suggested.\n");
319 }
320}
321
7086400d
AD		 322static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
323{
324 if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
09f40aed 325 !test_bit(__IXGBE_REMOVING, &adapter->state) &&
7086400d 326 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
780484d8 327 queue_work(ixgbe_wq, &adapter->service_task);
7086400d
AD		 328}
329
2a1a091c
MR		 330static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
331{
332 struct ixgbe_adapter *adapter = hw->back;
333
334 if (!hw->hw_addr)
335 return;
336 hw->hw_addr = NULL;
337 e_dev_err("Adapter removed\n");
58cf663f
MR		 338 if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
339 ixgbe_service_event_schedule(adapter);
2a1a091c
MR		 340}
341
f8e2472f 342static void ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
2a1a091c
MR		 343{
344 u32 value;
345
346 /* The following check not only optimizes a bit by not
347 * performing a read on the status register when the
348 * register just read was a status register read that
349 * returned IXGBE_FAILED_READ_REG. It also blocks any
350 * potential recursion.
351 */
352 if (reg == IXGBE_STATUS) {
353 ixgbe_remove_adapter(hw);
354 return;
355 }
356 value = ixgbe_read_reg(hw, IXGBE_STATUS);
357 if (value == IXGBE_FAILED_READ_REG)
358 ixgbe_remove_adapter(hw);
359}
360
f8e2472f
MR		 361/**
362 * ixgbe_read_reg - Read from device register
363 * @hw: hw specific details
364 * @reg: offset of register to read
365 *
366 * Returns : value read or IXGBE_FAILED_READ_REG if removed
367 *
368 * This function is used to read device registers. It checks for device
369 * removal by confirming any read that returns all ones by checking the
370 * status register value for all ones. This function avoids reading from
371 * the hardware if a removal was previously detected in which case it
372 * returns IXGBE_FAILED_READ_REG (all ones).
373 */
374u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
375{
376 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
377 u32 value;
378
379 if (ixgbe_removed(reg_addr))
380 return IXGBE_FAILED_READ_REG;
2f2219be
MR		 381 if (unlikely(hw->phy.nw_mng_if_sel &
382 IXGBE_NW_MNG_IF_SEL_ENABLE_10_100M)) {
383 struct ixgbe_adapter *adapter;
384 int i;
385
386 for (i = 0; i < 200; ++i) {
387 value = readl(reg_addr + IXGBE_MAC_SGMII_BUSY);
388 if (likely(!value))
389 goto writes_completed;
390 if (value == IXGBE_FAILED_READ_REG) {
391 ixgbe_remove_adapter(hw);
392 return IXGBE_FAILED_READ_REG;
393 }
394 udelay(5);
395 }
396
397 adapter = hw->back;
398 e_warn(hw, "register writes incomplete %08x\n", value);
399 }
400
401writes_completed:
f8e2472f
MR		 402 value = readl(reg_addr + reg);
403 if (unlikely(value == IXGBE_FAILED_READ_REG))
404 ixgbe_check_remove(hw, reg);
405 return value;
406}
407
14438464
MR		 408static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
409{
410 u16 value;
411
412 pci_read_config_word(pdev, PCI_VENDOR_ID, &value);
413 if (value == IXGBE_FAILED_READ_CFG_WORD) {
414 ixgbe_remove_adapter(hw);
415 return true;
416 }
417 return false;
418}
419
420u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
421{
422 struct ixgbe_adapter *adapter = hw->back;
423 u16 value;
424
425 if (ixgbe_removed(hw->hw_addr))
426 return IXGBE_FAILED_READ_CFG_WORD;
427 pci_read_config_word(adapter->pdev, reg, &value);
428 if (value == IXGBE_FAILED_READ_CFG_WORD &&
429 ixgbe_check_cfg_remove(hw, adapter->pdev))
430 return IXGBE_FAILED_READ_CFG_WORD;
431 return value;
432}
433
434#ifdef CONFIG_PCI_IOV
435static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
436{
437 struct ixgbe_adapter *adapter = hw->back;
438 u32 value;
439
440 if (ixgbe_removed(hw->hw_addr))
441 return IXGBE_FAILED_READ_CFG_DWORD;
442 pci_read_config_dword(adapter->pdev, reg, &value);
443 if (value == IXGBE_FAILED_READ_CFG_DWORD &&
444 ixgbe_check_cfg_remove(hw, adapter->pdev))
445 return IXGBE_FAILED_READ_CFG_DWORD;
446 return value;
447}
448#endif /* CONFIG_PCI_IOV */
449
ed19231c
JK		 450void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
451{
452 struct ixgbe_adapter *adapter = hw->back;
453
454 if (ixgbe_removed(hw->hw_addr))
455 return;
456 pci_write_config_word(adapter->pdev, reg, value);
457}
458
7086400d
AD		 459static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
460{
461 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
462
52f33af8 463 /* flush memory to make sure state is correct before next watchdog */
4e857c58 464 smp_mb__before_atomic();
7086400d
AD		 465 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
466}
467
dcd79aeb
TI		 468struct ixgbe_reg_info {
469 u32 ofs;
470 char *name;
471};
472
473static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
474
475 /* General Registers */
476 {IXGBE_CTRL, "CTRL"},
477 {IXGBE_STATUS, "STATUS"},
478 {IXGBE_CTRL_EXT, "CTRL_EXT"},
479
480 /* Interrupt Registers */
481 {IXGBE_EICR, "EICR"},
482
483 /* RX Registers */
484 {IXGBE_SRRCTL(0), "SRRCTL"},
485 {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
486 {IXGBE_RDLEN(0), "RDLEN"},
487 {IXGBE_RDH(0), "RDH"},
488 {IXGBE_RDT(0), "RDT"},
489 {IXGBE_RXDCTL(0), "RXDCTL"},
490 {IXGBE_RDBAL(0), "RDBAL"},
491 {IXGBE_RDBAH(0), "RDBAH"},
492
493 /* TX Registers */
494 {IXGBE_TDBAL(0), "TDBAL"},
495 {IXGBE_TDBAH(0), "TDBAH"},
496 {IXGBE_TDLEN(0), "TDLEN"},
497 {IXGBE_TDH(0), "TDH"},
498 {IXGBE_TDT(0), "TDT"},
499 {IXGBE_TXDCTL(0), "TXDCTL"},
500
501 /* List Terminator */
ca8dfe25 502 { .name = NULL }
dcd79aeb
TI		 503};
504
505
506/*
507 * ixgbe_regdump - register printout routine
508 */
509static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
510{
511 int i = 0, j = 0;
512 char rname[16];
513 u32 regs[64];
514
515 switch (reginfo->ofs) {
516 case IXGBE_SRRCTL(0):
517 for (i = 0; i < 64; i++)
518 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
519 break;
520 case IXGBE_DCA_RXCTRL(0):
521 for (i = 0; i < 64; i++)
522 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
523 break;
524 case IXGBE_RDLEN(0):
525 for (i = 0; i < 64; i++)
526 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
527 break;
528 case IXGBE_RDH(0):
529 for (i = 0; i < 64; i++)
530 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
531 break;
532 case IXGBE_RDT(0):
533 for (i = 0; i < 64; i++)
534 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
535 break;
536 case IXGBE_RXDCTL(0):
537 for (i = 0; i < 64; i++)
538 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
539 break;
540 case IXGBE_RDBAL(0):
541 for (i = 0; i < 64; i++)
542 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
543 break;
544 case IXGBE_RDBAH(0):
545 for (i = 0; i < 64; i++)
546 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
547 break;
548 case IXGBE_TDBAL(0):
549 for (i = 0; i < 64; i++)
550 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
551 break;
552 case IXGBE_TDBAH(0):
553 for (i = 0; i < 64; i++)
554 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
555 break;
556 case IXGBE_TDLEN(0):
557 for (i = 0; i < 64; i++)
558 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
559 break;
560 case IXGBE_TDH(0):
561 for (i = 0; i < 64; i++)
562 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
563 break;
564 case IXGBE_TDT(0):
565 for (i = 0; i < 64; i++)
566 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
567 break;
568 case IXGBE_TXDCTL(0):
569 for (i = 0; i < 64; i++)
570 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
571 break;
572 default:
c7689578 573 pr_info("%-15s %08x\n", reginfo->name,
dcd79aeb
TI		 574 IXGBE_READ_REG(hw, reginfo->ofs));
575 return;
576 }
577
578 for (i = 0; i < 8; i++) {
579 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
c7689578 580 pr_err("%-15s", rname);
dcd79aeb 581 for (j = 0; j < 8; j++)
c7689578
JP		 582 pr_cont(" %08x", regs[i*8+j]);
583 pr_cont("\n");
dcd79aeb
TI		 584 }
585
586}
587
588/*
589 * ixgbe_dump - Print registers, tx-rings and rx-rings
590 */
591static void ixgbe_dump(struct ixgbe_adapter *adapter)
592{
593 struct net_device *netdev = adapter->netdev;
594 struct ixgbe_hw *hw = &adapter->hw;
595 struct ixgbe_reg_info *reginfo;
596 int n = 0;
597 struct ixgbe_ring *tx_ring;
729739b7 598 struct ixgbe_tx_buffer *tx_buffer;
dcd79aeb
TI		 599 union ixgbe_adv_tx_desc *tx_desc;
600 struct my_u0 { u64 a; u64 b; } *u0;
601 struct ixgbe_ring *rx_ring;
602 union ixgbe_adv_rx_desc *rx_desc;
603 struct ixgbe_rx_buffer *rx_buffer_info;
604 u32 staterr;
605 int i = 0;
606
607 if (!netif_msg_hw(adapter))
608 return;
609
610 /* Print netdevice Info */
611 if (netdev) {
612 dev_info(&adapter->pdev->dev, "Net device Info\n");
c7689578 613 pr_info("Device Name state "
4a7c9726
TK		 614 "trans_start\n");
615 pr_info("%-15s %016lX %016lX\n",
c7689578
JP		 616 netdev->name,
617 netdev->state,
4a7c9726 618 dev_trans_start(netdev));
dcd79aeb
TI		 619 }
620
621 /* Print Registers */
622 dev_info(&adapter->pdev->dev, "Register Dump\n");
c7689578 623 pr_info(" Register Name Value\n");
dcd79aeb
TI		 624 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
625 reginfo->name; reginfo++) {
626 ixgbe_regdump(hw, reginfo);
627 }
628
629 /* Print TX Ring Summary */
630 if (!netdev || !netif_running(netdev))
e90dd264 631 return;
dcd79aeb
TI		 632
633 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
8ad88e37
JH		 634 pr_info(" %s %s %s %s\n",
635 "Queue [NTU] [NTC] [bi(ntc)->dma ]",
636 "leng", "ntw", "timestamp");
dcd79aeb
TI		 637 for (n = 0; n < adapter->num_tx_queues; n++) {
638 tx_ring = adapter->tx_ring[n];
729739b7 639 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
8ad88e37 640 pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
dcd79aeb 641 n, tx_ring->next_to_use, tx_ring->next_to_clean,
729739b7
AD		 642 (u64)dma_unmap_addr(tx_buffer, dma),
643 dma_unmap_len(tx_buffer, len),
644 tx_buffer->next_to_watch,
645 (u64)tx_buffer->time_stamp);
dcd79aeb
TI		 646 }
647
648 /* Print TX Rings */
649 if (!netif_msg_tx_done(adapter))
650 goto rx_ring_summary;
651
652 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
653
654 /* Transmit Descriptor Formats
655 *
39ac868a 656 * 82598 Advanced Transmit Descriptor
dcd79aeb
TI		 657 * +--------------------------------------------------------------+
658 * 0 | Buffer Address [63:0] |
659 * +--------------------------------------------------------------+
39ac868a 660 * 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
dcd79aeb
TI		 661 * +--------------------------------------------------------------+
662 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
39ac868a
JH		 663 *
664 * 82598 Advanced Transmit Descriptor (Write-Back Format)
665 * +--------------------------------------------------------------+
666 * 0 | RSV [63:0] |
667 * +--------------------------------------------------------------+
668 * 8 | RSV | STA | NXTSEQ |
669 * +--------------------------------------------------------------+
670 * 63 36 35 32 31 0
671 *
672 * 82599+ Advanced Transmit Descriptor
673 * +--------------------------------------------------------------+
674 * 0 | Buffer Address [63:0] |
675 * +--------------------------------------------------------------+
676 * 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN |
677 * +--------------------------------------------------------------+
678 * 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0
679 *
680 * 82599+ Advanced Transmit Descriptor (Write-Back Format)
681 * +--------------------------------------------------------------+
682 * 0 | RSV [63:0] |
683 * +--------------------------------------------------------------+
684 * 8 | RSV | STA | RSV |
685 * +--------------------------------------------------------------+
686 * 63 36 35 32 31 0
dcd79aeb
TI		 687 */
688
689 for (n = 0; n < adapter->num_tx_queues; n++) {
690 tx_ring = adapter->tx_ring[n];
c7689578
JP		 691 pr_info("------------------------------------\n");
692 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
693 pr_info("------------------------------------\n");
8ad88e37
JH		 694 pr_info("%s%s %s %s %s %s\n",
695 "T [desc] [address 63:0 ] ",
696 "[PlPOIdStDDt Ln] [bi->dma ] ",
697 "leng", "ntw", "timestamp", "bi->skb");
dcd79aeb
TI		 698
699 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
e4f74028 700 tx_desc = IXGBE_TX_DESC(tx_ring, i);
729739b7 701 tx_buffer = &tx_ring->tx_buffer_info[i];
dcd79aeb 702 u0 = (struct my_u0 *)tx_desc;
8ad88e37
JH		 703 if (dma_unmap_len(tx_buffer, len) > 0) {
704 pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p",
705 i,
706 le64_to_cpu(u0->a),
707 le64_to_cpu(u0->b),
708 (u64)dma_unmap_addr(tx_buffer, dma),
729739b7 709 dma_unmap_len(tx_buffer, len),
8ad88e37
JH		 710 tx_buffer->next_to_watch,
711 (u64)tx_buffer->time_stamp,
712 tx_buffer->skb);
713 if (i == tx_ring->next_to_use &&
714 i == tx_ring->next_to_clean)
715 pr_cont(" NTC/U\n");
716 else if (i == tx_ring->next_to_use)
717 pr_cont(" NTU\n");
718 else if (i == tx_ring->next_to_clean)
719 pr_cont(" NTC\n");
720 else
721 pr_cont("\n");
722
723 if (netif_msg_pktdata(adapter) &&
724 tx_buffer->skb)
725 print_hex_dump(KERN_INFO, "",
726 DUMP_PREFIX_ADDRESS, 16, 1,
727 tx_buffer->skb->data,
728 dma_unmap_len(tx_buffer, len),
729 true);
730 }
dcd79aeb
TI		 731 }
732 }
733
734 /* Print RX Rings Summary */
735rx_ring_summary:
736 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
c7689578 737 pr_info("Queue [NTU] [NTC]\n");
dcd79aeb
TI		 738 for (n = 0; n < adapter->num_rx_queues; n++) {
739 rx_ring = adapter->rx_ring[n];
c7689578
JP		 740 pr_info("%5d %5X %5X\n",
741 n, rx_ring->next_to_use, rx_ring->next_to_clean);
dcd79aeb
TI		 742 }
743
744 /* Print RX Rings */
745 if (!netif_msg_rx_status(adapter))
e90dd264 746 return;
dcd79aeb
TI		 747
748 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
749
39ac868a
JH		 750 /* Receive Descriptor Formats
751 *
752 * 82598 Advanced Receive Descriptor (Read) Format
dcd79aeb
TI		 753 * 63 1 0
754 * +-----------------------------------------------------+
755 * 0 | Packet Buffer Address [63:1] |A0/NSE|
756 * +----------------------------------------------+------+
757 * 8 | Header Buffer Address [63:1] | DD |
758 * +-----------------------------------------------------+
759 *
760 *
39ac868a 761 * 82598 Advanced Receive Descriptor (Write-Back) Format
dcd79aeb
TI		 762 *
763 * 63 48 47 32 31 30 21 20 16 15 4 3 0
764 * +------------------------------------------------------+
39ac868a
JH		 765 * 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS |
766 * | Packet | IP | | | | Type | Type |
767 * | Checksum | Ident | | | | | |
dcd79aeb
TI		 768 * +------------------------------------------------------+
769 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
770 * +------------------------------------------------------+
771 * 63 48 47 32 31 20 19 0
39ac868a
JH		 772 *
773 * 82599+ Advanced Receive Descriptor (Read) Format
774 * 63 1 0
775 * +-----------------------------------------------------+
776 * 0 | Packet Buffer Address [63:1] |A0/NSE|
777 * +----------------------------------------------+------+
778 * 8 | Header Buffer Address [63:1] | DD |
779 * +-----------------------------------------------------+
780 *
781 *
782 * 82599+ Advanced Receive Descriptor (Write-Back) Format
783 *
784 * 63 48 47 32 31 30 21 20 17 16 4 3 0
785 * +------------------------------------------------------+
786 * 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS |
787 * |/ RTT / PCoE_PARAM | | | CNT | Type | Type |
788 * |/ Flow Dir Flt ID | | | | | |
789 * +------------------------------------------------------+
790 * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
791 * +------------------------------------------------------+
792 * 63 48 47 32 31 20 19 0
dcd79aeb 793 */
39ac868a 794
dcd79aeb
TI		 795 for (n = 0; n < adapter->num_rx_queues; n++) {
796 rx_ring = adapter->rx_ring[n];
c7689578
JP		 797 pr_info("------------------------------------\n");
798 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
799 pr_info("------------------------------------\n");
8ad88e37
JH		 800 pr_info("%s%s%s",
801 "R [desc] [ PktBuf A0] ",
802 "[ HeadBuf DD] [bi->dma ] [bi->skb ] ",
dcd79aeb 803 "<-- Adv Rx Read format\n");
8ad88e37
JH		 804 pr_info("%s%s%s",
805 "RWB[desc] [PcsmIpSHl PtRs] ",
806 "[vl er S cks ln] ---------------- [bi->skb ] ",
dcd79aeb
TI		 807 "<-- Adv Rx Write-Back format\n");
808
809 for (i = 0; i < rx_ring->count; i++) {
810 rx_buffer_info = &rx_ring->rx_buffer_info[i];
e4f74028 811 rx_desc = IXGBE_RX_DESC(rx_ring, i);
dcd79aeb
TI		 812 u0 = (struct my_u0 *)rx_desc;
813 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
814 if (staterr & IXGBE_RXD_STAT_DD) {
815 /* Descriptor Done */
c7689578 816 pr_info("RWB[0x%03X] %016llX "
dcd79aeb
TI		 817 "%016llX ---------------- %p", i,
818 le64_to_cpu(u0->a),
819 le64_to_cpu(u0->b),
820 rx_buffer_info->skb);
821 } else {
c7689578 822 pr_info("R [0x%03X] %016llX "
dcd79aeb
TI		 823 "%016llX %016llX %p", i,
824 le64_to_cpu(u0->a),
825 le64_to_cpu(u0->b),
826 (u64)rx_buffer_info->dma,
827 rx_buffer_info->skb);
828
9c50c035
ET		 829 if (netif_msg_pktdata(adapter) &&
830 rx_buffer_info->dma) {
dcd79aeb
TI		 831 print_hex_dump(KERN_INFO, "",
832 DUMP_PREFIX_ADDRESS, 16, 1,
9c50c035
ET		 833 page_address(rx_buffer_info->page) +
834 rx_buffer_info->page_offset,
f800326d 835 ixgbe_rx_bufsz(rx_ring), true);
dcd79aeb
TI		 836 }
837 }
838
839 if (i == rx_ring->next_to_use)
c7689578 840 pr_cont(" NTU\n");
dcd79aeb 841 else if (i == rx_ring->next_to_clean)
c7689578 842 pr_cont(" NTC\n");
dcd79aeb 843 else
c7689578 844 pr_cont("\n");
dcd79aeb
TI		 845
846 }
847 }
dcd79aeb
TI		 848}
849
5eba3699
AV		 850static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
851{
852 u32 ctrl_ext;
853
854 /* Let firmware take over control of h/w */
855 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
856 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
e8e9f696 857 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
5eba3699
AV		 858}
859
860static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
861{
862 u32 ctrl_ext;
863
864 /* Let firmware know the driver has taken over */
865 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
866 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
e8e9f696 867 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
5eba3699 868}
9a799d71 869
49ce9c2c 870/**
e8e26350
PW		 871 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
872 * @adapter: pointer to adapter struct
873 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
874 * @queue: queue to map the corresponding interrupt to
875 * @msix_vector: the vector to map to the corresponding queue
876 *
877 */
878static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
e8e9f696 879 u8 queue, u8 msix_vector)
9a799d71
AK		 880{
881 u32 ivar, index;
e8e26350
PW		 882 struct ixgbe_hw *hw = &adapter->hw;
883 switch (hw->mac.type) {
884 case ixgbe_mac_82598EB:
885 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
886 if (direction == -1)
887 direction = 0;
888 index = (((direction * 64) + queue) >> 2) & 0x1F;
889 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
890 ivar &= ~(0xFF << (8 * (queue & 0x3)));
891 ivar |= (msix_vector << (8 * (queue & 0x3)));
892 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
893 break;
894 case ixgbe_mac_82599EB:
b93a2226 895 case ixgbe_mac_X540:
9a75a1ac
DS		 896 case ixgbe_mac_X550:
897 case ixgbe_mac_X550EM_x:
49425dfc 898 case ixgbe_mac_x550em_a:
e8e26350
PW		 899 if (direction == -1) {
900 /* other causes */
901 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
902 index = ((queue & 1) * 8);
903 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
904 ivar &= ~(0xFF << index);
905 ivar |= (msix_vector << index);
906 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
907 break;
908 } else {
909 /* tx or rx causes */
910 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
911 index = ((16 * (queue & 1)) + (8 * direction));
912 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
913 ivar &= ~(0xFF << index);
914 ivar |= (msix_vector << index);
915 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
916 break;
917 }
918 default:
919 break;
920 }
9a799d71
AK		 921}
922
fe49f04a 923static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
e8e9f696 924 u64 qmask)
fe49f04a
AD		 925{
926 u32 mask;
927
bd508178
AD		 928 switch (adapter->hw.mac.type) {
929 case ixgbe_mac_82598EB:
fe49f04a
AD		 930 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
931 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
bd508178
AD		 932 break;
933 case ixgbe_mac_82599EB:
b93a2226 934 case ixgbe_mac_X540:
9a75a1ac
DS		 935 case ixgbe_mac_X550:
936 case ixgbe_mac_X550EM_x:
49425dfc 937 case ixgbe_mac_x550em_a:
fe49f04a
AD		 938 mask = (qmask & 0xFFFFFFFF);
939 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
940 mask = (qmask >> 32);
941 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
bd508178
AD		 942 break;
943 default:
944 break;
fe49f04a
AD		 945 }
946}
947
729739b7
AD		 948void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring,
949 struct ixgbe_tx_buffer *tx_buffer)
9a799d71 950{
729739b7
AD		 951 if (tx_buffer->skb) {
952 dev_kfree_skb_any(tx_buffer->skb);
953 if (dma_unmap_len(tx_buffer, len))
d3d00239 954 dma_unmap_single(ring->dev,
729739b7
AD		 955 dma_unmap_addr(tx_buffer, dma),
956 dma_unmap_len(tx_buffer, len),
957 DMA_TO_DEVICE);
958 } else if (dma_unmap_len(tx_buffer, len)) {
959 dma_unmap_page(ring->dev,
960 dma_unmap_addr(tx_buffer, dma),
961 dma_unmap_len(tx_buffer, len),
962 DMA_TO_DEVICE);
e5a43549 963 }
729739b7
AD		 964 tx_buffer->next_to_watch = NULL;
965 tx_buffer->skb = NULL;
966 dma_unmap_len_set(tx_buffer, len, 0);
967 /* tx_buffer must be completely set up in the transmit path */
9a799d71
AK		 968}
969
943561d3 970static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
c84d324c
JF		 971{
972 struct ixgbe_hw *hw = &adapter->hw;
973 struct ixgbe_hw_stats *hwstats = &adapter->stats;
c84d324c 974 int i;
943561d3 975 u32 data;
c84d324c 976
943561d3
AD		 977 if ((hw->fc.current_mode != ixgbe_fc_full) &&
978 (hw->fc.current_mode != ixgbe_fc_rx_pause))
979 return;
c84d324c 980
943561d3
AD		 981 switch (hw->mac.type) {
982 case ixgbe_mac_82598EB:
983 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
984 break;
985 default:
986 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
987 }
988 hwstats->lxoffrxc += data;
c84d324c 989
943561d3
AD		 990 /* refill credits (no tx hang) if we received xoff */
991 if (!data)
c84d324c 992 return;
943561d3
AD		 993
994 for (i = 0; i < adapter->num_tx_queues; i++)
995 clear_bit(__IXGBE_HANG_CHECK_ARMED,
996 &adapter->tx_ring[i]->state);
997}
998
999static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
1000{
1001 struct ixgbe_hw *hw = &adapter->hw;
1002 struct ixgbe_hw_stats *hwstats = &adapter->stats;
1003 u32 xoff[8] = {0};
2afaa00d 1004 u8 tc;
943561d3
AD		 1005 int i;
1006 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
1007
1008 if (adapter->ixgbe_ieee_pfc)
1009 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
1010
1011 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
1012 ixgbe_update_xoff_rx_lfc(adapter);
c84d324c 1013 return;
943561d3 1014 }
c84d324c
JF		 1015
1016 /* update stats for each tc, only valid with PFC enabled */
1017 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
2afaa00d
PN		 1018 u32 pxoffrxc;
1019
c84d324c
JF		 1020 switch (hw->mac.type) {
1021 case ixgbe_mac_82598EB:
2afaa00d 1022 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
bd508178 1023 break;
c84d324c 1024 default:
2afaa00d 1025 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
26f23d82 1026 }
2afaa00d
PN		 1027 hwstats->pxoffrxc[i] += pxoffrxc;
1028 /* Get the TC for given UP */
1029 tc = netdev_get_prio_tc_map(adapter->netdev, i);
1030 xoff[tc] += pxoffrxc;
c84d324c
JF		 1031 }
1032
1033 /* disarm tx queues that have received xoff frames */
1034 for (i = 0; i < adapter->num_tx_queues; i++) {
1035 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
c84d324c 1036
2afaa00d 1037 tc = tx_ring->dcb_tc;
c84d324c
JF		 1038 if (xoff[tc])
1039 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
26f23d82 1040 }
26f23d82
YZ		 1041}
1042
c84d324c 1043static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
9a799d71 1044{
7d7ce682 1045 return ring->stats.packets;
c84d324c
JF		 1046}
1047
1048static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
1049{
2a47fa45
JF		 1050 struct ixgbe_adapter *adapter;
1051 struct ixgbe_hw *hw;
1052 u32 head, tail;
1053
1054 if (ring->l2_accel_priv)
1055 adapter = ring->l2_accel_priv->real_adapter;
1056 else
1057 adapter = netdev_priv(ring->netdev);
e01c31a5 1058
2a47fa45
JF		 1059 hw = &adapter->hw;
1060 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
1061 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
c84d324c
JF		 1062
1063 if (head != tail)
1064 return (head < tail) ?
1065 tail - head : (tail + ring->count - head);
1066
1067 return 0;
1068}
1069
1070static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
1071{
1072 u32 tx_done = ixgbe_get_tx_completed(tx_ring);
1073 u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
1074 u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
c84d324c 1075
7d637bcc 1076 clear_check_for_tx_hang(tx_ring);
c84d324c
JF		 1077
1078 /*
1079 * Check for a hung queue, but be thorough. This verifies
1080 * that a transmit has been completed since the previous
1081 * check AND there is at least one packet pending. The
1082 * ARMED bit is set to indicate a potential hang. The
1083 * bit is cleared if a pause frame is received to remove
1084 * false hang detection due to PFC or 802.3x frames. By
1085 * requiring this to fail twice we avoid races with
1086 * pfc clearing the ARMED bit and conditions where we
1087 * run the check_tx_hang logic with a transmit completion
1088 * pending but without time to complete it yet.
1089 */
e90dd264 1090 if (tx_done_old == tx_done && tx_pending)
c84d324c 1091 /* make sure it is true for two checks in a row */
e90dd264
MR		 1092 return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
1093 &tx_ring->state);
1094 /* update completed stats and continue */
1095 tx_ring->tx_stats.tx_done_old = tx_done;
1096 /* reset the countdown */
1097 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
9a799d71 1098
e90dd264 1099 return false;
9a799d71
AK		 1100}
1101
c83c6cbd
AD		 1102/**
1103 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
1104 * @adapter: driver private struct
1105 **/
1106static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
1107{
1108
1109 /* Do the reset outside of interrupt context */
1110 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
57ca2a4f 1111 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
12ff3f3b 1112 e_warn(drv, "initiating reset due to tx timeout\n");
c83c6cbd
AD		 1113 ixgbe_service_event_schedule(adapter);
1114 }
1115}
e01c31a5 1116
c04f90e5
RP		 1117/**
1118 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
1119 **/
1120static int ixgbe_tx_maxrate(struct net_device *netdev,
1121 int queue_index, u32 maxrate)
1122{
1123 struct ixgbe_adapter *adapter = netdev_priv(netdev);
1124 struct ixgbe_hw *hw = &adapter->hw;
1125 u32 bcnrc_val = ixgbe_link_mbps(adapter);
1126
1127 if (!maxrate)
1128 return 0;
1129
1130 /* Calculate the rate factor values to set */
1131 bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
1132 bcnrc_val /= maxrate;
1133
1134 /* clear everything but the rate factor */
1135 bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
1136 IXGBE_RTTBCNRC_RF_DEC_MASK;
1137
1138 /* enable the rate scheduler */
1139 bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;
1140
1141 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
1142 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
1143
1144 return 0;
1145}
1146
9a799d71
AK		 1147/**
1148 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
fe49f04a 1149 * @q_vector: structure containing interrupt and ring information
e01c31a5 1150 * @tx_ring: tx ring to clean
8220bbc1 1151 * @napi_budget: Used to determine if we are in netpoll
9a799d71 1152 **/
fe49f04a 1153static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
a3a8749d 1154 struct ixgbe_ring *tx_ring, int napi_budget)
9a799d71 1155{
fe49f04a 1156 struct ixgbe_adapter *adapter = q_vector->adapter;
d3d00239
AD		 1157 struct ixgbe_tx_buffer *tx_buffer;
1158 union ixgbe_adv_tx_desc *tx_desc;
e01c31a5 1159 unsigned int total_bytes = 0, total_packets = 0;
59224555 1160 unsigned int budget = q_vector->tx.work_limit;
729739b7
AD		 1161 unsigned int i = tx_ring->next_to_clean;
1162
1163 if (test_bit(__IXGBE_DOWN, &adapter->state))
1164 return true;
9a799d71 1165
d3d00239 1166 tx_buffer = &tx_ring->tx_buffer_info[i];
e4f74028 1167 tx_desc = IXGBE_TX_DESC(tx_ring, i);
729739b7 1168 i -= tx_ring->count;
12207e49 1169
729739b7 1170 do {
d3d00239
AD		 1171 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1172
1173 /* if next_to_watch is not set then there is no work pending */
1174 if (!eop_desc)
1175 break;
1176
7f83a9e6 1177 /* prevent any other reads prior to eop_desc */
7e63bf49 1178 read_barrier_depends();
7f83a9e6 1179
d3d00239
AD		 1180 /* if DD is not set pending work has not been completed */
1181 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
1182 break;
8ad494b0 1183
d3d00239
AD		 1184 /* clear next_to_watch to prevent false hangs */
1185 tx_buffer->next_to_watch = NULL;
8ad494b0 1186
091a6246
AD		 1187 /* update the statistics for this packet */
1188 total_bytes += tx_buffer->bytecount;
1189 total_packets += tx_buffer->gso_segs;
1190
fd0db0ed 1191 /* free the skb */
a3a8749d 1192 napi_consume_skb(tx_buffer->skb, napi_budget);
fd0db0ed 1193
729739b7
AD		 1194 /* unmap skb header data */
1195 dma_unmap_single(tx_ring->dev,
1196 dma_unmap_addr(tx_buffer, dma),
1197 dma_unmap_len(tx_buffer, len),
1198 DMA_TO_DEVICE);
1199
fd0db0ed
AD		 1200 /* clear tx_buffer data */
1201 tx_buffer->skb = NULL;
729739b7 1202 dma_unmap_len_set(tx_buffer, len, 0);
fd0db0ed 1203
729739b7
AD		 1204 /* unmap remaining buffers */
1205 while (tx_desc != eop_desc) {
d3d00239
AD		 1206 tx_buffer++;
1207 tx_desc++;
8ad494b0 1208 i++;
729739b7
AD		 1209 if (unlikely(!i)) {
1210 i -= tx_ring->count;
d3d00239 1211 tx_buffer = tx_ring->tx_buffer_info;
e4f74028 1212 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
e092be60 1213 }
e01c31a5 1214
729739b7
AD		 1215 /* unmap any remaining paged data */
1216 if (dma_unmap_len(tx_buffer, len)) {
1217 dma_unmap_page(tx_ring->dev,
1218 dma_unmap_addr(tx_buffer, dma),
1219 dma_unmap_len(tx_buffer, len),
1220 DMA_TO_DEVICE);
1221 dma_unmap_len_set(tx_buffer, len, 0);
1222 }
1223 }
1224
1225 /* move us one more past the eop_desc for start of next pkt */
1226 tx_buffer++;
1227 tx_desc++;
1228 i++;
1229 if (unlikely(!i)) {
1230 i -= tx_ring->count;
1231 tx_buffer = tx_ring->tx_buffer_info;
1232 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1233 }
1234
1235 /* issue prefetch for next Tx descriptor */
1236 prefetch(tx_desc);
12207e49 1237
729739b7
AD		 1238 /* update budget accounting */
1239 budget--;
1240 } while (likely(budget));
1241
1242 i += tx_ring->count;
9a799d71 1243 tx_ring->next_to_clean = i;
d3d00239 1244 u64_stats_update_begin(&tx_ring->syncp);
b953799e 1245 tx_ring->stats.bytes += total_bytes;
bd198058 1246 tx_ring->stats.packets += total_packets;
d3d00239 1247 u64_stats_update_end(&tx_ring->syncp);
bd198058
AD		 1248 q_vector->tx.total_bytes += total_bytes;
1249 q_vector->tx.total_packets += total_packets;
b953799e 1250
c84d324c
JF		 1251 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
1252 /* schedule immediate reset if we believe we hung */
1253 struct ixgbe_hw *hw = &adapter->hw;
c84d324c
JF		 1254 e_err(drv, "Detected Tx Unit Hang\n"
1255 " Tx Queue <%d>\n"
1256 " TDH, TDT <%x>, <%x>\n"
1257 " next_to_use <%x>\n"
1258 " next_to_clean <%x>\n"
1259 "tx_buffer_info[next_to_clean]\n"
1260 " time_stamp <%lx>\n"
1261 " jiffies <%lx>\n",
1262 tx_ring->queue_index,
1263 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
1264 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
d3d00239
AD		 1265 tx_ring->next_to_use, i,
1266 tx_ring->tx_buffer_info[i].time_stamp, jiffies);
c84d324c
JF		 1267
1268 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
1269
1270 e_info(probe,
1271 "tx hang %d detected on queue %d, resetting adapter\n",
1272 adapter->tx_timeout_count + 1, tx_ring->queue_index);
1273
b953799e 1274 /* schedule immediate reset if we believe we hung */
c83c6cbd 1275 ixgbe_tx_timeout_reset(adapter);
b953799e
AD		 1276
1277 /* the adapter is about to reset, no point in enabling stuff */
59224555 1278 return true;
b953799e 1279 }
9a799d71 1280
b2d96e0a
AD		 1281 netdev_tx_completed_queue(txring_txq(tx_ring),
1282 total_packets, total_bytes);
1283
e092be60 1284#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
30065e63 1285 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
7d4987de 1286 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
e092be60
AV		 1287 /* Make sure that anybody stopping the queue after this
1288 * sees the new next_to_clean.
1289 */
1290 smp_mb();
729739b7
AD		 1291 if (__netif_subqueue_stopped(tx_ring->netdev,
1292 tx_ring->queue_index)
1293 && !test_bit(__IXGBE_DOWN, &adapter->state)) {
1294 netif_wake_subqueue(tx_ring->netdev,
1295 tx_ring->queue_index);
5b7da515 1296 ++tx_ring->tx_stats.restart_queue;
30eba97a 1297 }
e092be60 1298 }
9a799d71 1299
59224555 1300 return !!budget;
9a799d71
AK		 1301}
1302
5dd2d332 1303#ifdef CONFIG_IXGBE_DCA
bdda1a61
AD		 1304static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
1305 struct ixgbe_ring *tx_ring,
33cf09c9 1306 int cpu)
bd0362dd 1307{
33cf09c9 1308 struct ixgbe_hw *hw = &adapter->hw;
9de7605e 1309 u32 txctrl = 0;
bdda1a61 1310 u16 reg_offset;
33cf09c9 1311
9de7605e
MR		 1312 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1313 txctrl = dca3_get_tag(tx_ring->dev, cpu);
1314
33cf09c9
AD		 1315 switch (hw->mac.type) {
1316 case ixgbe_mac_82598EB:
bdda1a61 1317 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
33cf09c9
AD		 1318 break;
1319 case ixgbe_mac_82599EB:
b93a2226 1320 case ixgbe_mac_X540:
bdda1a61
AD		 1321 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
1322 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
33cf09c9
AD		 1323 break;
1324 default:
bdda1a61
AD		 1325 /* for unknown hardware do not write register */
1326 return;
bd0362dd 1327 }
bdda1a61
AD		 1328
1329 /*
1330 * We can enable relaxed ordering for reads, but not writes when
1331 * DCA is enabled. This is due to a known issue in some chipsets
1332 * which will cause the DCA tag to be cleared.
1333 */
1334 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1335 IXGBE_DCA_TXCTRL_DATA_RRO_EN |
1336 IXGBE_DCA_TXCTRL_DESC_DCA_EN;
1337
1338 IXGBE_WRITE_REG(hw, reg_offset, txctrl);
bd0362dd
JC		 1339}
1340
bdda1a61
AD		 1341static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
1342 struct ixgbe_ring *rx_ring,
33cf09c9 1343 int cpu)
bd0362dd 1344{
33cf09c9 1345 struct ixgbe_hw *hw = &adapter->hw;
9de7605e 1346 u32 rxctrl = 0;
bdda1a61
AD		 1347 u8 reg_idx = rx_ring->reg_idx;
1348
9de7605e
MR		 1349 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1350 rxctrl = dca3_get_tag(rx_ring->dev, cpu);
33cf09c9
AD		 1351
1352 switch (hw->mac.type) {
33cf09c9 1353 case ixgbe_mac_82599EB:
b93a2226 1354 case ixgbe_mac_X540:
bdda1a61 1355 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
33cf09c9
AD		 1356 break;
1357 default:
1358 break;
1359 }
bdda1a61
AD		 1360
1361 /*
1362 * We can enable relaxed ordering for reads, but not writes when
1363 * DCA is enabled. This is due to a known issue in some chipsets
1364 * which will cause the DCA tag to be cleared.
1365 */
1366 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
9de7605e 1367 IXGBE_DCA_RXCTRL_DATA_DCA_EN |
bdda1a61
AD		 1368 IXGBE_DCA_RXCTRL_DESC_DCA_EN;
1369
1370 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
33cf09c9
AD		 1371}
1372
1373static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
1374{
1375 struct ixgbe_adapter *adapter = q_vector->adapter;
efe3d3c8 1376 struct ixgbe_ring *ring;
bd0362dd 1377 int cpu = get_cpu();
bd0362dd 1378
33cf09c9
AD		 1379 if (q_vector->cpu == cpu)
1380 goto out_no_update;
1381
a557928e 1382 ixgbe_for_each_ring(ring, q_vector->tx)
efe3d3c8 1383 ixgbe_update_tx_dca(adapter, ring, cpu);
33cf09c9 1384
a557928e 1385 ixgbe_for_each_ring(ring, q_vector->rx)
efe3d3c8 1386 ixgbe_update_rx_dca(adapter, ring, cpu);
33cf09c9
AD		 1387
1388 q_vector->cpu = cpu;
1389out_no_update:
bd0362dd
JC		 1390 put_cpu();
1391}
1392
1393static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
1394{
1395 int i;
1396
e35ec126 1397 /* always use CB2 mode, difference is masked in the CB driver */
9de7605e
MR		 1398 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1399 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1400 IXGBE_DCA_CTRL_DCA_MODE_CB2);
1401 else
1402 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1403 IXGBE_DCA_CTRL_DCA_DISABLE);
e35ec126 1404
49c7ffbe 1405 for (i = 0; i < adapter->num_q_vectors; i++) {
33cf09c9
AD		 1406 adapter->q_vector[i]->cpu = -1;
1407 ixgbe_update_dca(adapter->q_vector[i]);
bd0362dd
JC		 1408 }
1409}
1410
1411static int __ixgbe_notify_dca(struct device *dev, void *data)
1412{
c60fbb00 1413 struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
bd0362dd
JC		 1414 unsigned long event = *(unsigned long *)data;
1415
2a72c31e 1416 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
33cf09c9
AD		 1417 return 0;
1418
bd0362dd
JC		 1419 switch (event) {
1420 case DCA_PROVIDER_ADD:
96b0e0f6
JB		 1421 /* if we're already enabled, don't do it again */
1422 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1423 break;
652f093f 1424 if (dca_add_requester(dev) == 0) {
96b0e0f6 1425 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
9de7605e
MR		 1426 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1427 IXGBE_DCA_CTRL_DCA_MODE_CB2);
bd0362dd
JC		 1428 break;
1429 }
1430 /* Fall Through since DCA is disabled. */
1431 case DCA_PROVIDER_REMOVE:
1432 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1433 dca_remove_requester(dev);
1434 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
9de7605e
MR		 1435 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1436 IXGBE_DCA_CTRL_DCA_DISABLE);
bd0362dd
JC		 1437 }
1438 break;
1439 }
1440
652f093f 1441 return 0;
bd0362dd 1442}
67a74ee2 1443
bdda1a61 1444#endif /* CONFIG_IXGBE_DCA */
7edda4b8
FD		 1445
1446#define IXGBE_RSS_L4_TYPES_MASK \
1447 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
1448 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
1449 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
1450 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
1451
8a0da21b
AD		 1452static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
1453 union ixgbe_adv_rx_desc *rx_desc,
67a74ee2
ET		 1454 struct sk_buff *skb)
1455{
7edda4b8
FD		 1456 u16 rss_type;
1457
1458 if (!(ring->netdev->features & NETIF_F_RXHASH))
1459 return;
1460
1461 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
1462 IXGBE_RXDADV_RSSTYPE_MASK;
1463
1464 if (!rss_type)
1465 return;
1466
1467 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1468 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
1469 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
67a74ee2
ET		 1470}
1471
f800326d 1472#ifdef IXGBE_FCOE
ff886dfc
AD		 1473/**
1474 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
57efd44c 1475 * @ring: structure containing ring specific data
ff886dfc
AD		 1476 * @rx_desc: advanced rx descriptor
1477 *
1478 * Returns : true if it is FCoE pkt
1479 */
57efd44c 1480static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
ff886dfc
AD		 1481 union ixgbe_adv_rx_desc *rx_desc)
1482{
1483 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1484
57efd44c 1485 return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
ff886dfc
AD		 1486 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1487 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1488 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1489}
1490
f800326d 1491#endif /* IXGBE_FCOE */
e59bd25d
AV		 1492/**
1493 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
8a0da21b
AD		 1494 * @ring: structure containing ring specific data
1495 * @rx_desc: current Rx descriptor being processed
e59bd25d
AV		 1496 * @skb: skb currently being received and modified
1497 **/
8a0da21b 1498static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
8bae1b2b 1499 union ixgbe_adv_rx_desc *rx_desc,
f56e0cb1 1500 struct sk_buff *skb)
9a799d71 1501{
3f207800 1502 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
3f207800
DS		 1503 bool encap_pkt = false;
1504
8a0da21b 1505 skb_checksum_none_assert(skb);
9a799d71 1506
712744be 1507 /* Rx csum disabled */
8a0da21b 1508 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9a799d71 1509 return;
e59bd25d 1510
a21d0822
ET		 1511 /* check for VXLAN and Geneve packets */
1512 if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {
3f207800
DS		 1513 encap_pkt = true;
1514 skb->encapsulation = 1;
3f207800
DS		 1515 }
1516
e59bd25d 1517 /* if IP and error */
f56e0cb1
AD		 1518 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
1519 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
8a0da21b 1520 ring->rx_stats.csum_err++;
9a799d71
AK		 1521 return;
1522 }
e59bd25d 1523
f56e0cb1 1524 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
e59bd25d
AV		 1525 return;
1526
f56e0cb1 1527 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
8bae1b2b
DS		 1528 /*
1529 * 82599 errata, UDP frames with a 0 checksum can be marked as
1530 * checksum errors.
1531 */
8a0da21b
AD		 1532 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
1533 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
8bae1b2b
DS		 1534 return;
1535
8a0da21b 1536 ring->rx_stats.csum_err++;
e59bd25d
AV		 1537 return;
1538 }
1539
9a799d71 1540 /* It must be a TCP or UDP packet with a valid checksum */
e59bd25d 1541 skb->ip_summed = CHECKSUM_UNNECESSARY;
3f207800
DS		 1542 if (encap_pkt) {
1543 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))
1544 return;
1545
1546 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {
d469251b 1547 skb->ip_summed = CHECKSUM_NONE;
3f207800
DS		 1548 return;
1549 }
1550 /* If we checked the outer header let the stack know */
1551 skb->csum_level = 1;
1552 }
9a799d71
AK		 1553}
1554
f990b79b
AD		 1555static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
1556 struct ixgbe_rx_buffer *bi)
1557{
1558 struct page *page = bi->page;
18cb652a 1559 dma_addr_t dma;
f990b79b 1560
f800326d 1561 /* since we are recycling buffers we should seldom need to alloc */
18cb652a 1562 if (likely(page))
f990b79b
AD		 1563 return true;
1564
f800326d 1565 /* alloc new page for storage */
18cb652a
AD		 1566 page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));
1567 if (unlikely(!page)) {
1568 rx_ring->rx_stats.alloc_rx_page_failed++;
1569 return false;
f990b79b
AD		 1570 }
1571
f800326d 1572 /* map page for use */
f3213d93
AD		 1573 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1574 ixgbe_rx_pg_size(rx_ring),
1575 DMA_FROM_DEVICE,
1576 IXGBE_RX_DMA_ATTR);
f800326d
AD		 1577
1578 /*
1579 * if mapping failed free memory back to system since
1580 * there isn't much point in holding memory we can't use
1581 */
1582 if (dma_mapping_error(rx_ring->dev, dma)) {
dd411ec4 1583 __free_pages(page, ixgbe_rx_pg_order(rx_ring));
f990b79b 1584
f990b79b
AD		 1585 rx_ring->rx_stats.alloc_rx_page_failed++;
1586 return false;
1587 }
1588
f800326d 1589 bi->dma = dma;
18cb652a 1590 bi->page = page;
afaa9459 1591 bi->page_offset = 0;
1b56cf49 1592 bi->pagecnt_bias = 1;
f800326d 1593
f990b79b
AD		 1594 return true;
1595}
1596
9a799d71 1597/**
f990b79b 1598 * ixgbe_alloc_rx_buffers - Replace used receive buffers
fc77dc3c
AD		 1599 * @rx_ring: ring to place buffers on
1600 * @cleaned_count: number of buffers to replace
9a799d71 1601 **/
fc77dc3c 1602void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
9a799d71 1603{
9a799d71 1604 union ixgbe_adv_rx_desc *rx_desc;
3a581073 1605 struct ixgbe_rx_buffer *bi;
d5f398ed 1606 u16 i = rx_ring->next_to_use;
4f4542bf 1607 u16 bufsz;
9a799d71 1608
f800326d
AD		 1609 /* nothing to do */
1610 if (!cleaned_count)
fc77dc3c
AD		 1611 return;
1612
e4f74028 1613 rx_desc = IXGBE_RX_DESC(rx_ring, i);
f990b79b
AD		 1614 bi = &rx_ring->rx_buffer_info[i];
1615 i -= rx_ring->count;
9a799d71 1616
4f4542bf
AD		 1617 bufsz = ixgbe_rx_bufsz(rx_ring);
1618
f800326d
AD		 1619 do {
1620 if (!ixgbe_alloc_mapped_page(rx_ring, bi))
f990b79b 1621 break;
d5f398ed 1622
f3213d93
AD		 1623 /* sync the buffer for use by the device */
1624 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
4f4542bf 1625 bi->page_offset, bufsz,
f3213d93
AD		 1626 DMA_FROM_DEVICE);
1627
f800326d
AD		 1628 /*
1629 * Refresh the desc even if buffer_addrs didn't change
1630 * because each write-back erases this info.
1631 */
1632 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
9a799d71 1633
f990b79b
AD		 1634 rx_desc++;
1635 bi++;
9a799d71 1636 i++;
f990b79b 1637 if (unlikely(!i)) {
e4f74028 1638 rx_desc = IXGBE_RX_DESC(rx_ring, 0);
f990b79b
AD		 1639 bi = rx_ring->rx_buffer_info;
1640 i -= rx_ring->count;
1641 }
1642
c3630cc4
AD		 1643 /* clear the length for the next_to_use descriptor */
1644 rx_desc->wb.upper.length = 0;
f800326d
AD		 1645
1646 cleaned_count--;
1647 } while (cleaned_count);
7c6e0a43 1648
f990b79b
AD		 1649 i += rx_ring->count;
1650
ad435ec6
AD		 1651 if (rx_ring->next_to_use != i) {
1652 rx_ring->next_to_use = i;
1653
1654 /* update next to alloc since we have filled the ring */
1655 rx_ring->next_to_alloc = i;
1656
1657 /* Force memory writes to complete before letting h/w
1658 * know there are new descriptors to fetch. (Only
1659 * applicable for weak-ordered memory model archs,
1660 * such as IA-64).
1661 */
1662 wmb();
1663 writel(i, rx_ring->tail);
1664 }
9a799d71
AK		 1665}
1666
1d2024f6
AD		 1667static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1668 struct sk_buff *skb)
1669{
f800326d 1670 u16 hdr_len = skb_headlen(skb);
1d2024f6
AD		 1671
1672 /* set gso_size to avoid messing up TCP MSS */
1673 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1674 IXGBE_CB(skb)->append_cnt);
96be80ab 1675 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1d2024f6
AD		 1676}
1677
1678static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1679 struct sk_buff *skb)
1680{
1681 /* if append_cnt is 0 then frame is not RSC */
1682 if (!IXGBE_CB(skb)->append_cnt)
1683 return;
1684
1685 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1686 rx_ring->rx_stats.rsc_flush++;
1687
1688 ixgbe_set_rsc_gso_size(rx_ring, skb);
1689
1690 /* gso_size is computed using append_cnt so always clear it last */
1691 IXGBE_CB(skb)->append_cnt = 0;
1692}
1693
8a0da21b
AD		 1694/**
1695 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
1696 * @rx_ring: rx descriptor ring packet is being transacted on
1697 * @rx_desc: pointer to the EOP Rx descriptor
1698 * @skb: pointer to current skb being populated
f8212f97 1699 *
8a0da21b
AD		 1700 * This function checks the ring, descriptor, and packet information in
1701 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1702 * other fields within the skb.
f8212f97 1703 **/
8a0da21b
AD		 1704static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
1705 union ixgbe_adv_rx_desc *rx_desc,
1706 struct sk_buff *skb)
f8212f97 1707{
43e95f11 1708 struct net_device *dev = rx_ring->netdev;
a9763f3c 1709 u32 flags = rx_ring->q_vector->adapter->flags;
43e95f11 1710
8a0da21b
AD		 1711 ixgbe_update_rsc_stats(rx_ring, skb);
1712
1713 ixgbe_rx_hash(rx_ring, rx_desc, skb);
f8212f97 1714
8a0da21b
AD		 1715 ixgbe_rx_checksum(rx_ring, rx_desc, skb);
1716
a9763f3c
MR		 1717 if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))
1718 ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);
3a6a4eda 1719
f646968f 1720 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
43e95f11 1721 ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
8a0da21b 1722 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
86a9bad3 1723 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
f8212f97
AD		 1724 }
1725
8a0da21b 1726 skb_record_rx_queue(skb, rx_ring->queue_index);
aa80175a 1727
43e95f11 1728 skb->protocol = eth_type_trans(skb, dev);
f8212f97
AD		 1729}
1730
8a0da21b
AD		 1731static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
1732 struct sk_buff *skb)
aa80175a 1733{
3ffc1af5 1734 napi_gro_receive(&q_vector->napi, skb);
aa80175a 1735}
43634e82 1736
f800326d
AD		 1737/**
1738 * ixgbe_is_non_eop - process handling of non-EOP buffers
1739 * @rx_ring: Rx ring being processed
1740 * @rx_desc: Rx descriptor for current buffer
1741 * @skb: Current socket buffer containing buffer in progress
1742 *
1743 * This function updates next to clean. If the buffer is an EOP buffer
1744 * this function exits returning false, otherwise it will place the
1745 * sk_buff in the next buffer to be chained and return true indicating
1746 * that this is in fact a non-EOP buffer.
1747 **/
1748static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
1749 union ixgbe_adv_rx_desc *rx_desc,
1750 struct sk_buff *skb)
1751{
1752 u32 ntc = rx_ring->next_to_clean + 1;
1753
1754 /* fetch, update, and store next to clean */
1755 ntc = (ntc < rx_ring->count) ? ntc : 0;
1756 rx_ring->next_to_clean = ntc;
1757
1758 prefetch(IXGBE_RX_DESC(rx_ring, ntc));
1759
5a02cbd1
AD		 1760 /* update RSC append count if present */
1761 if (ring_is_rsc_enabled(rx_ring)) {
1762 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1763 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
1764
1765 if (unlikely(rsc_enabled)) {
1766 u32 rsc_cnt = le32_to_cpu(rsc_enabled);
1767
1768 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1769 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
f800326d 1770
5a02cbd1
AD		 1771 /* update ntc based on RSC value */
1772 ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
1773 ntc &= IXGBE_RXDADV_NEXTP_MASK;
1774 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
1775 }
f800326d
AD		 1776 }
1777
5a02cbd1
AD		 1778 /* if we are the last buffer then there is nothing else to do */
1779 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1780 return false;
1781
f800326d
AD		 1782 /* place skb in next buffer to be received */
1783 rx_ring->rx_buffer_info[ntc].skb = skb;
1784 rx_ring->rx_stats.non_eop_descs++;
1785
1786 return true;
1787}
1788
19861ce2
AD		 1789/**
1790 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
1791 * @rx_ring: rx descriptor ring packet is being transacted on
1792 * @skb: pointer to current skb being adjusted
1793 *
1794 * This function is an ixgbe specific version of __pskb_pull_tail. The
1795 * main difference between this version and the original function is that
1796 * this function can make several assumptions about the state of things
1797 * that allow for significant optimizations versus the standard function.
1798 * As a result we can do things like drop a frag and maintain an accurate
1799 * truesize for the skb.
1800 */
1801static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
1802 struct sk_buff *skb)
1803{
1804 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1805 unsigned char *va;
1806 unsigned int pull_len;
1807
1808 /*
1809 * it is valid to use page_address instead of kmap since we are
1810 * working with pages allocated out of the lomem pool per
1811 * alloc_page(GFP_ATOMIC)
1812 */
1813 va = skb_frag_address(frag);
1814
1815 /*
1816 * we need the header to contain the greater of either ETH_HLEN or
1817 * 60 bytes if the skb->len is less than 60 for skb_pad.
1818 */
8496e338 1819 pull_len = eth_get_headlen(va, IXGBE_RX_HDR_SIZE);
19861ce2
AD		 1820
1821 /* align pull length to size of long to optimize memcpy performance */
1822 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
1823
1824 /* update all of the pointers */
1825 skb_frag_size_sub(frag, pull_len);
1826 frag->page_offset += pull_len;
1827 skb->data_len -= pull_len;
1828 skb->tail += pull_len;
19861ce2
AD		 1829}
1830
42073d91
AD		 1831/**
1832 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
1833 * @rx_ring: rx descriptor ring packet is being transacted on
1834 * @skb: pointer to current skb being updated
1835 *
1836 * This function provides a basic DMA sync up for the first fragment of an
1837 * skb. The reason for doing this is that the first fragment cannot be
1838 * unmapped until we have reached the end of packet descriptor for a buffer
1839 * chain.
1840 */
1841static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
1842 struct sk_buff *skb)
1843{
1844 /* if the page was released unmap it, else just sync our portion */
1845 if (unlikely(IXGBE_CB(skb)->page_released)) {
f3213d93
AD		 1846 dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma,
1847 ixgbe_rx_pg_size(rx_ring),
1848 DMA_FROM_DEVICE,
1849 IXGBE_RX_DMA_ATTR);
42073d91
AD		 1850 } else {
1851 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1852
1853 dma_sync_single_range_for_cpu(rx_ring->dev,
1854 IXGBE_CB(skb)->dma,
1855 frag->page_offset,
f215af8c 1856 skb_frag_size(frag),
42073d91
AD		 1857 DMA_FROM_DEVICE);
1858 }
42073d91
AD		 1859}
1860
f800326d
AD		 1861/**
1862 * ixgbe_cleanup_headers - Correct corrupted or empty headers
1863 * @rx_ring: rx descriptor ring packet is being transacted on
1864 * @rx_desc: pointer to the EOP Rx descriptor
1865 * @skb: pointer to current skb being fixed
1866 *
1867 * Check for corrupted packet headers caused by senders on the local L2
1868 * embedded NIC switch not setting up their Tx Descriptors right. These
1869 * should be very rare.
1870 *
1871 * Also address the case where we are pulling data in on pages only
1872 * and as such no data is present in the skb header.
1873 *
1874 * In addition if skb is not at least 60 bytes we need to pad it so that
1875 * it is large enough to qualify as a valid Ethernet frame.
1876 *
1877 * Returns true if an error was encountered and skb was freed.
1878 **/
1879static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
1880 union ixgbe_adv_rx_desc *rx_desc,
1881 struct sk_buff *skb)
1882{
f800326d 1883 struct net_device *netdev = rx_ring->netdev;
f800326d
AD		 1884
1885 /* verify that the packet does not have any known errors */
1886 if (unlikely(ixgbe_test_staterr(rx_desc,
1887 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
1888 !(netdev->features & NETIF_F_RXALL))) {
1889 dev_kfree_skb_any(skb);
1890 return true;
1891 }
1892
19861ce2 1893 /* place header in linear portion of buffer */
cf3fe7ac
AD		 1894 if (skb_is_nonlinear(skb))
1895 ixgbe_pull_tail(rx_ring, skb);
f800326d 1896
57efd44c
AD		 1897#ifdef IXGBE_FCOE
1898 /* do not attempt to pad FCoE Frames as this will disrupt DDP */
1899 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
1900 return false;
1901
1902#endif
a94d9e22
AD		 1903 /* if eth_skb_pad returns an error the skb was freed */
1904 if (eth_skb_pad(skb))
1905 return true;
f800326d
AD		 1906
1907 return false;
1908}
1909
f800326d
AD		 1910/**
1911 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
1912 * @rx_ring: rx descriptor ring to store buffers on
1913 * @old_buff: donor buffer to have page reused
1914 *
0549ae20 1915 * Synchronizes page for reuse by the adapter
f800326d
AD		 1916 **/
1917static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
1918 struct ixgbe_rx_buffer *old_buff)
1919{
1920 struct ixgbe_rx_buffer *new_buff;
1921 u16 nta = rx_ring->next_to_alloc;
f800326d
AD		 1922
1923 new_buff = &rx_ring->rx_buffer_info[nta];
1924
1925 /* update, and store next to alloc */
1926 nta++;
1927 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1928
3fd21876
AD		 1929 /* Transfer page from old buffer to new buffer.
1930 * Move each member individually to avoid possible store
1931 * forwarding stalls and unnecessary copy of skb.
1932 */
1933 new_buff->dma = old_buff->dma;
1934 new_buff->page = old_buff->page;
1935 new_buff->page_offset = old_buff->page_offset;
1936 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
f800326d
AD		 1937}
1938
18cb652a
AD		 1939static inline bool ixgbe_page_is_reserved(struct page *page)
1940{
2f064f34 1941 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
18cb652a
AD		 1942}
1943
3fd21876 1944static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer)
af43da0d 1945{
3fd21876
AD		 1946 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1947 struct page *page = rx_buffer->page;
1b56cf49 1948
af43da0d
AD		 1949 /* avoid re-using remote pages */
1950 if (unlikely(ixgbe_page_is_reserved(page)))
1951 return false;
1952
1953#if (PAGE_SIZE < 8192)
1954 /* if we are only owner of page we can reuse it */
3fd21876 1955 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
af43da0d 1956 return false;
af43da0d 1957#else
3fd21876
AD		 1958 /* The last offset is a bit aggressive in that we assume the
1959 * worst case of FCoE being enabled and using a 3K buffer.
1960 * However this should have minimal impact as the 1K extra is
1961 * still less than one buffer in size.
1962 */
1963#define IXGBE_LAST_OFFSET \
1964 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)
1965 if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)
af43da0d
AD		 1966 return false;
1967#endif
1968
1b56cf49
AD		 1969 /* If we have drained the page fragment pool we need to update
1970 * the pagecnt_bias and page count so that we fully restock the
1971 * number of references the driver holds.
af43da0d 1972 */
3fd21876 1973 if (unlikely(!pagecnt_bias)) {
1b56cf49
AD		 1974 page_ref_add(page, USHRT_MAX);
1975 rx_buffer->pagecnt_bias = USHRT_MAX;
1976 }
af43da0d
AD		 1977
1978 return true;
1979}
1980
f800326d
AD		 1981/**
1982 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
1983 * @rx_ring: rx descriptor ring to transact packets on
1984 * @rx_buffer: buffer containing page to add
1985 * @rx_desc: descriptor containing length of buffer written by hardware
1986 * @skb: sk_buff to place the data into
1987 *
0549ae20
AD		 1988 * This function will add the data contained in rx_buffer->page to the skb.
1989 * This is done either through a direct copy if the data in the buffer is
1990 * less than the skb header size, otherwise it will just attach the page as
1991 * a frag to the skb.
1992 *
1993 * The function will then update the page offset if necessary and return
1994 * true if the buffer can be reused by the adapter.
f800326d 1995 **/
3fd21876 1996static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
f800326d 1997 struct ixgbe_rx_buffer *rx_buffer,
3fd21876
AD		 1998 struct sk_buff *skb,
1999 unsigned int size)
f800326d 2000{
09816fbe 2001#if (PAGE_SIZE < 8192)
4f4542bf 2002 unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
09816fbe 2003#else
4f4542bf 2004 unsigned int truesize = SKB_DATA_ALIGN(size);
09816fbe 2005#endif
3fd21876 2006 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
0549ae20 2007 rx_buffer->page_offset, size, truesize);
3fd21876
AD		 2008#if (PAGE_SIZE < 8192)
2009 rx_buffer->page_offset ^= truesize;
2010#else
2011 rx_buffer->page_offset += truesize;
2012#endif
f800326d
AD		 2013}
2014
3fd21876
AD		 2015static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,
2016 union ixgbe_adv_rx_desc *rx_desc,
2017 struct sk_buff **skb,
2018 const unsigned int size)
18806c9e
AD		 2019{
2020 struct ixgbe_rx_buffer *rx_buffer;
18806c9e
AD		 2021
2022 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
3fd21876
AD		 2023 prefetchw(rx_buffer->page);
2024 *skb = rx_buffer->skb;
18806c9e 2025
3fd21876
AD		 2026 /* Delay unmapping of the first packet. It carries the header
2027 * information, HW may still access the header after the writeback.
2028 * Only unmap it when EOP is reached
2029 */
2030 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {
2031 if (!*skb)
2032 goto skip_sync;
18806c9e 2033 } else {
3fd21876
AD		 2034 if (*skb)
2035 ixgbe_dma_sync_frag(rx_ring, *skb);
2036 }
18806c9e 2037
3fd21876
AD		 2038 /* we are reusing so sync this buffer for CPU use */
2039 dma_sync_single_range_for_cpu(rx_ring->dev,
2040 rx_buffer->dma,
2041 rx_buffer->page_offset,
2042 size,
2043 DMA_FROM_DEVICE);
2044skip_sync:
2045 rx_buffer->pagecnt_bias--;
18cb652a 2046
3fd21876
AD		 2047 return rx_buffer;
2048}
18806c9e 2049
3fd21876
AD		 2050static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,
2051 struct ixgbe_rx_buffer *rx_buffer,
2052 struct sk_buff *skb)
2053{
2054 if (ixgbe_can_reuse_rx_page(rx_buffer)) {
18806c9e
AD		 2055 /* hand second half of page back to the ring */
2056 ixgbe_reuse_rx_page(rx_ring, rx_buffer);
18806c9e 2057 } else {
1b56cf49
AD		 2058 if (IXGBE_CB(skb)->dma == rx_buffer->dma) {
2059 /* the page has been released from the ring */
2060 IXGBE_CB(skb)->page_released = true;
2061 } else {
2062 /* we are not reusing the buffer so unmap it */
2063 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
2064 ixgbe_rx_pg_size(rx_ring),
2065 DMA_FROM_DEVICE,
2066 IXGBE_RX_DMA_ATTR);
2067 }
3fd21876 2068 __page_frag_cache_drain(rx_buffer->page,
1b56cf49 2069 rx_buffer->pagecnt_bias);
18806c9e
AD		 2070 }
2071
3fd21876 2072 /* clear contents of rx_buffer */
18806c9e 2073 rx_buffer->page = NULL;
3fd21876
AD		 2074 rx_buffer->skb = NULL;
2075}
2076
2077static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
2078 struct ixgbe_rx_buffer *rx_buffer,
2079 union ixgbe_adv_rx_desc *rx_desc,
2080 unsigned int size)
2081{
2082 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
2083#if (PAGE_SIZE < 8192)
2084 unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2085#else
2086 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
2087#endif
2088 struct sk_buff *skb;
2089
2090 /* prefetch first cache line of first page */
2091 prefetch(va);
2092#if L1_CACHE_BYTES < 128
2093 prefetch(va + L1_CACHE_BYTES);
2094#endif
2095
2096 /* allocate a skb to store the frags */
2097 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);
2098 if (unlikely(!skb))
2099 return NULL;
2100
2101 if (size > IXGBE_RX_HDR_SIZE) {
2102 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
2103 IXGBE_CB(skb)->dma = rx_buffer->dma;
2104
2105 skb_add_rx_frag(skb, 0, rx_buffer->page,
2106 rx_buffer->page_offset,
2107 size, truesize);
2108#if (PAGE_SIZE < 8192)
2109 rx_buffer->page_offset ^= truesize;
2110#else
2111 rx_buffer->page_offset += truesize;
2112#endif
2113 } else {
2114 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
2115 rx_buffer->pagecnt_bias++;
2116 }
18806c9e
AD		 2117
2118 return skb;
f800326d
AD		 2119}
2120
2121/**
2122 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
2123 * @q_vector: structure containing interrupt and ring information
2124 * @rx_ring: rx descriptor ring to transact packets on
2125 * @budget: Total limit on number of packets to process
2126 *
2127 * This function provides a "bounce buffer" approach to Rx interrupt
2128 * processing. The advantage to this is that on systems that have
2129 * expensive overhead for IOMMU access this provides a means of avoiding
2130 * it by maintaining the mapping of the page to the syste.
2131 *
5a85e737 2132 * Returns amount of work completed
f800326d 2133 **/
5a85e737 2134static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
e8e9f696 2135 struct ixgbe_ring *rx_ring,
f4de00ed 2136 const int budget)
9a799d71 2137{
d2f4fbe2 2138 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
3f2d1c0f 2139#ifdef IXGBE_FCOE
f800326d 2140 struct ixgbe_adapter *adapter = q_vector->adapter;
4ffdf91a
MR		 2141 int ddp_bytes;
2142 unsigned int mss = 0;
3d8fd385 2143#endif /* IXGBE_FCOE */
f800326d 2144 u16 cleaned_count = ixgbe_desc_unused(rx_ring);
9a799d71 2145
fdabfc8a 2146 while (likely(total_rx_packets < budget)) {
f800326d 2147 union ixgbe_adv_rx_desc *rx_desc;
3fd21876 2148 struct ixgbe_rx_buffer *rx_buffer;
f800326d 2149 struct sk_buff *skb;
3fd21876 2150 unsigned int size;
f800326d
AD		 2151
2152 /* return some buffers to hardware, one at a time is too slow */
2153 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
2154 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
2155 cleaned_count = 0;
2156 }
2157
18806c9e 2158 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
3fd21876
AD		 2159 size = le16_to_cpu(rx_desc->wb.upper.length);
2160 if (!size)
f800326d 2161 break;
9a799d71 2162
124b74c1 2163 /* This memory barrier is needed to keep us from reading
f800326d 2164 * any other fields out of the rx_desc until we know the
124b74c1 2165 * descriptor has been written back
f800326d 2166 */
124b74c1 2167 dma_rmb();
9a799d71 2168
3fd21876
AD		 2169 rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size);
2170
18806c9e 2171 /* retrieve a buffer from the ring */
3fd21876
AD		 2172 if (skb)
2173 ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);
2174 else
2175 skb = ixgbe_construct_skb(rx_ring, rx_buffer,
2176 rx_desc, size);
f800326d 2177
18806c9e 2178 /* exit if we failed to retrieve a buffer */
3fd21876
AD		 2179 if (!skb) {
2180 rx_ring->rx_stats.alloc_rx_buff_failed++;
2181 rx_buffer->pagecnt_bias++;
18806c9e 2182 break;
3fd21876 2183 }
9a799d71 2184
3fd21876 2185 ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb);
9a799d71 2186 cleaned_count++;
f8212f97 2187
f800326d
AD		 2188 /* place incomplete frames back on ring for completion */
2189 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
2190 continue;
c267fc16 2191
f800326d
AD		 2192 /* verify the packet layout is correct */
2193 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
2194 continue;
9a799d71 2195
d2f4fbe2
AV		 2196 /* probably a little skewed due to removing CRC */
2197 total_rx_bytes += skb->len;
d2f4fbe2 2198
8a0da21b
AD		 2199 /* populate checksum, timestamp, VLAN, and protocol */
2200 ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
2201
332d4a7d
YZ		 2202#ifdef IXGBE_FCOE
2203 /* if ddp, not passing to ULD unless for FCP_RSP or error */
57efd44c 2204 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
f56e0cb1 2205 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
4ffdf91a
MR		 2206 /* include DDPed FCoE data */
2207 if (ddp_bytes > 0) {
2208 if (!mss) {
2209 mss = rx_ring->netdev->mtu -
2210 sizeof(struct fcoe_hdr) -
2211 sizeof(struct fc_frame_header) -
2212 sizeof(struct fcoe_crc_eof);
2213 if (mss > 512)
2214 mss &= ~511;
2215 }
2216 total_rx_bytes += ddp_bytes;
2217 total_rx_packets += DIV_ROUND_UP(ddp_bytes,
2218 mss);
2219 }
63d635b2
AD		 2220 if (!ddp_bytes) {
2221 dev_kfree_skb_any(skb);
f800326d 2222 continue;
63d635b2 2223 }
3d8fd385 2224 }
f800326d 2225
332d4a7d 2226#endif /* IXGBE_FCOE */
8a0da21b 2227 ixgbe_rx_skb(q_vector, skb);
9a799d71 2228
f800326d 2229 /* update budget accounting */
f4de00ed 2230 total_rx_packets++;
fdabfc8a 2231 }
9a799d71 2232
c267fc16
AD		 2233 u64_stats_update_begin(&rx_ring->syncp);
2234 rx_ring->stats.packets += total_rx_packets;
2235 rx_ring->stats.bytes += total_rx_bytes;
2236 u64_stats_update_end(&rx_ring->syncp);
bd198058
AD		 2237 q_vector->rx.total_packets += total_rx_packets;
2238 q_vector->rx.total_bytes += total_rx_bytes;
4ff7fb12 2239
5a85e737 2240 return total_rx_packets;
9a799d71
AK		 2241}
2242
9a799d71
AK		 2243/**
2244 * ixgbe_configure_msix - Configure MSI-X hardware
2245 * @adapter: board private structure
2246 *
2247 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
2248 * interrupts.
2249 **/
2250static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
2251{
021230d4 2252 struct ixgbe_q_vector *q_vector;
49c7ffbe 2253 int v_idx;
021230d4 2254 u32 mask;
9a799d71 2255
8e34d1aa
AD		 2256 /* Populate MSIX to EITR Select */
2257 if (adapter->num_vfs > 32) {
b4f47a48 2258 u32 eitrsel = BIT(adapter->num_vfs - 32) - 1;
8e34d1aa
AD		 2259 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
2260 }
2261
4df10466
JB		 2262 /*
2263 * Populate the IVAR table and set the ITR values to the
021230d4
AV		 2264 * corresponding register.
2265 */
49c7ffbe 2266 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
efe3d3c8 2267 struct ixgbe_ring *ring;
7a921c93 2268 q_vector = adapter->q_vector[v_idx];
021230d4 2269
a557928e 2270 ixgbe_for_each_ring(ring, q_vector->rx)
efe3d3c8
AD		 2271 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
2272
a557928e 2273 ixgbe_for_each_ring(ring, q_vector->tx)
efe3d3c8
AD		 2274 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
2275
fe49f04a 2276 ixgbe_write_eitr(q_vector);
9a799d71
AK		 2277 }
2278
bd508178
AD		 2279 switch (adapter->hw.mac.type) {
2280 case ixgbe_mac_82598EB:
e8e26350 2281 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
e8e9f696 2282 v_idx);
bd508178
AD		 2283 break;
2284 case ixgbe_mac_82599EB:
b93a2226 2285 case ixgbe_mac_X540:
9a75a1ac
DS		 2286 case ixgbe_mac_X550:
2287 case ixgbe_mac_X550EM_x:
49425dfc 2288 case ixgbe_mac_x550em_a:
e8e26350 2289 ixgbe_set_ivar(adapter, -1, 1, v_idx);
bd508178 2290 break;
bd508178
AD		 2291 default:
2292 break;
2293 }
021230d4
AV		 2294 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
2295
41fb9248 2296 /* set up to autoclear timer, and the vectors */
021230d4 2297 mask = IXGBE_EIMS_ENABLE_MASK;
d5bf4f67
ET		 2298 mask &= ~(IXGBE_EIMS_OTHER |
2299 IXGBE_EIMS_MAILBOX |
2300 IXGBE_EIMS_LSC);
2301
021230d4 2302 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
9a799d71
AK		 2303}
2304
f494e8fa
AV		 2305enum latency_range {
2306 lowest_latency = 0,
2307 low_latency = 1,
2308 bulk_latency = 2,
2309 latency_invalid = 255
2310};
2311
2312/**
2313 * ixgbe_update_itr - update the dynamic ITR value based on statistics
bd198058
AD		 2314 * @q_vector: structure containing interrupt and ring information
2315 * @ring_container: structure containing ring performance data
f494e8fa
AV		 2316 *
2317 * Stores a new ITR value based on packets and byte
2318 * counts during the last interrupt. The advantage of per interrupt
2319 * computation is faster updates and more accurate ITR for the current
2320 * traffic pattern. Constants in this function were computed
2321 * based on theoretical maximum wire speed and thresholds were set based
2322 * on testing data as well as attempting to minimize response time
2323 * while increasing bulk throughput.
2324 * this functionality is controlled by the InterruptThrottleRate module
2325 * parameter (see ixgbe_param.c)
2326 **/
bd198058
AD		 2327static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
2328 struct ixgbe_ring_container *ring_container)
f494e8fa 2329{
bd198058
AD		 2330 int bytes = ring_container->total_bytes;
2331 int packets = ring_container->total_packets;
2332 u32 timepassed_us;
621bd70e 2333 u64 bytes_perint;
bd198058 2334 u8 itr_setting = ring_container->itr;
f494e8fa
AV		 2335
2336 if (packets == 0)
bd198058 2337 return;
f494e8fa
AV		 2338
2339 /* simple throttlerate management
621bd70e
AD		 2340 * 0-10MB/s lowest (100000 ints/s)
2341 * 10-20MB/s low (20000 ints/s)
8ac34f10 2342 * 20-1249MB/s bulk (12000 ints/s)
f494e8fa
AV		 2343 */
2344 /* what was last interrupt timeslice? */
d5bf4f67 2345 timepassed_us = q_vector->itr >> 2;
bdbeefe8
DS		 2346 if (timepassed_us == 0)
2347 return;
2348
f494e8fa
AV		 2349 bytes_perint = bytes / timepassed_us; /* bytes/usec */
2350
2351 switch (itr_setting) {
2352 case lowest_latency:
621bd70e 2353 if (bytes_perint > 10)
bd198058 2354 itr_setting = low_latency;
f494e8fa
AV		 2355 break;
2356 case low_latency:
621bd70e 2357 if (bytes_perint > 20)
bd198058 2358 itr_setting = bulk_latency;
621bd70e 2359 else if (bytes_perint <= 10)
bd198058 2360 itr_setting = lowest_latency;
f494e8fa
AV		 2361 break;
2362 case bulk_latency:
621bd70e 2363 if (bytes_perint <= 20)
bd198058 2364 itr_setting = low_latency;
f494e8fa
AV		 2365 break;
2366 }
2367
bd198058
AD		 2368 /* clear work counters since we have the values we need */
2369 ring_container->total_bytes = 0;
2370 ring_container->total_packets = 0;
2371
2372 /* write updated itr to ring container */
2373 ring_container->itr = itr_setting;
f494e8fa
AV		 2374}
2375
509ee935
JB		 2376/**
2377 * ixgbe_write_eitr - write EITR register in hardware specific way
fe49f04a 2378 * @q_vector: structure containing interrupt and ring information
509ee935
JB		 2379 *
2380 * This function is made to be called by ethtool and by the driver
2381 * when it needs to update EITR registers at runtime. Hardware
2382 * specific quirks/differences are taken care of here.
2383 */
fe49f04a 2384void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
509ee935 2385{
fe49f04a 2386 struct ixgbe_adapter *adapter = q_vector->adapter;
509ee935 2387 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 2388 int v_idx = q_vector->v_idx;
5d967eb7 2389 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
fe49f04a 2390
bd508178
AD		 2391 switch (adapter->hw.mac.type) {
2392 case ixgbe_mac_82598EB:
509ee935
JB		 2393 /* must write high and low 16 bits to reset counter */
2394 itr_reg |= (itr_reg << 16);
bd508178
AD		 2395 break;
2396 case ixgbe_mac_82599EB:
b93a2226 2397 case ixgbe_mac_X540:
9a75a1ac
DS		 2398 case ixgbe_mac_X550:
2399 case ixgbe_mac_X550EM_x:
49425dfc 2400 case ixgbe_mac_x550em_a:
509ee935
JB		 2401 /*
2402 * set the WDIS bit to not clear the timer bits and cause an
2403 * immediate assertion of the interrupt
2404 */
2405 itr_reg |= IXGBE_EITR_CNT_WDIS;
bd508178
AD		 2406 break;
2407 default:
2408 break;
509ee935
JB		 2409 }
2410 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
2411}
2412
bd198058 2413static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
f494e8fa 2414{
d5bf4f67 2415 u32 new_itr = q_vector->itr;
bd198058 2416 u8 current_itr;
f494e8fa 2417
bd198058
AD		 2418 ixgbe_update_itr(q_vector, &q_vector->tx);
2419 ixgbe_update_itr(q_vector, &q_vector->rx);
f494e8fa 2420
08c8833b 2421 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
f494e8fa
AV		 2422
2423 switch (current_itr) {
2424 /* counts and packets in update_itr are dependent on these numbers */
2425 case lowest_latency:
d5bf4f67 2426 new_itr = IXGBE_100K_ITR;
f494e8fa
AV		 2427 break;
2428 case low_latency:
d5bf4f67 2429 new_itr = IXGBE_20K_ITR;
f494e8fa
AV		 2430 break;
2431 case bulk_latency:
8ac34f10 2432 new_itr = IXGBE_12K_ITR;
f494e8fa 2433 break;
bd198058
AD		 2434 default:
2435 break;
f494e8fa
AV		 2436 }
2437
d5bf4f67 2438 if (new_itr != q_vector->itr) {
fe49f04a 2439 /* do an exponential smoothing */
d5bf4f67
ET		 2440 new_itr = (10 * new_itr * q_vector->itr) /
2441 ((9 * new_itr) + q_vector->itr);
509ee935 2442
bd198058 2443 /* save the algorithm value here */
5d967eb7 2444 q_vector->itr = new_itr;
fe49f04a
AD		 2445
2446 ixgbe_write_eitr(q_vector);
f494e8fa 2447 }
f494e8fa
AV		 2448}
2449
119fc60a 2450/**
de88eeeb 2451 * ixgbe_check_overtemp_subtask - check for over temperature
f0f9778d 2452 * @adapter: pointer to adapter
119fc60a 2453 **/
f0f9778d 2454static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
119fc60a 2455{
119fc60a
MC		 2456 struct ixgbe_hw *hw = &adapter->hw;
2457 u32 eicr = adapter->interrupt_event;
b3eb4e18 2458 s32 rc;
119fc60a 2459
f0f9778d 2460 if (test_bit(__IXGBE_DOWN, &adapter->state))
7ca647bd
JP		 2461 return;
2462
f0f9778d
AD		 2463 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2464 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
2465 return;
2466
2467 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2468
7ca647bd 2469 switch (hw->device_id) {
f0f9778d
AD		 2470 case IXGBE_DEV_ID_82599_T3_LOM:
2471 /*
2472 * Since the warning interrupt is for both ports
2473 * we don't have to check if:
2474 * - This interrupt wasn't for our port.
2475 * - We may have missed the interrupt so always have to
2476 * check if we got a LSC
2477 */
9a900eca 2478 if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&
f0f9778d
AD		 2479 !(eicr & IXGBE_EICR_LSC))
2480 return;
2481
2482 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
3d292265 2483 u32 speed;
f0f9778d 2484 bool link_up = false;
7ca647bd 2485
3d292265 2486 hw->mac.ops.check_link(hw, &speed, &link_up, false);
7ca647bd 2487
f0f9778d
AD		 2488 if (link_up)
2489 return;
2490 }
2491
2492 /* Check if this is not due to overtemp */
2493 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
2494 return;
2495
2496 break;
b3eb4e18
MR		 2497 case IXGBE_DEV_ID_X550EM_A_1G_T:
2498 case IXGBE_DEV_ID_X550EM_A_1G_T_L:
2499 rc = hw->phy.ops.check_overtemp(hw);
2500 if (rc != IXGBE_ERR_OVERTEMP)
2501 return;
2502 break;
7ca647bd 2503 default:
597f22d6
DS		 2504 if (adapter->hw.mac.type >= ixgbe_mac_X540)
2505 return;
9a900eca 2506 if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))
119fc60a 2507 return;
7ca647bd 2508 break;
119fc60a 2509 }
f44e751b 2510 e_crit(drv, "%s\n", ixgbe_overheat_msg);
f0f9778d
AD		 2511
2512 adapter->interrupt_event = 0;
119fc60a
MC		 2513}
2514
0befdb3e
JB		 2515static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
2516{
2517 struct ixgbe_hw *hw = &adapter->hw;
2518
2519 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
9a900eca 2520 (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
396e799c 2521 e_crit(probe, "Fan has stopped, replace the adapter\n");
0befdb3e 2522 /* write to clear the interrupt */
9a900eca 2523 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
0befdb3e
JB		 2524 }
2525}
cf8280ee 2526
4f51bf70
JK		 2527static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
2528{
9a900eca
DS		 2529 struct ixgbe_hw *hw = &adapter->hw;
2530
4f51bf70
JK		 2531 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
2532 return;
2533
2534 switch (adapter->hw.mac.type) {
2535 case ixgbe_mac_82599EB:
2536 /*
2537 * Need to check link state so complete overtemp check
2538 * on service task
2539 */
9a900eca
DS		 2540 if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||
2541 (eicr & IXGBE_EICR_LSC)) &&
4f51bf70
JK		 2542 (!test_bit(__IXGBE_DOWN, &adapter->state))) {
2543 adapter->interrupt_event = eicr;
2544 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2545 ixgbe_service_event_schedule(adapter);
2546 return;
2547 }
2548 return;
b3eb4e18
MR		 2549 case ixgbe_mac_x550em_a:
2550 if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) {
2551 adapter->interrupt_event = eicr;
2552 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2553 ixgbe_service_event_schedule(adapter);
2554 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC,
2555 IXGBE_EICR_GPI_SDP0_X550EM_a);
2556 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR,
2557 IXGBE_EICR_GPI_SDP0_X550EM_a);
2558 }
2559 return;
2560 case ixgbe_mac_X550:
4f51bf70
JK		 2561 case ixgbe_mac_X540:
2562 if (!(eicr & IXGBE_EICR_TS))
2563 return;
2564 break;
2565 default:
2566 return;
2567 }
2568
f44e751b 2569 e_crit(drv, "%s\n", ixgbe_overheat_msg);
4f51bf70
JK		 2570}
2571
45788d2a
DS		 2572static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
2573{
2574 switch (hw->mac.type) {
2575 case ixgbe_mac_82598EB:
2576 if (hw->phy.type == ixgbe_phy_nl)
2577 return true;
2578 return false;
2579 case ixgbe_mac_82599EB:
2580 case ixgbe_mac_X550EM_x:
49425dfc 2581 case ixgbe_mac_x550em_a:
45788d2a
DS		 2582 switch (hw->mac.ops.get_media_type(hw)) {
2583 case ixgbe_media_type_fiber:
2584 case ixgbe_media_type_fiber_qsfp:
2585 return true;
2586 default:
2587 return false;
2588 }
2589 default:
2590 return false;
2591 }
2592}
2593
e8e26350
PW		 2594static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
2595{
2596 struct ixgbe_hw *hw = &adapter->hw;
4ccc650c 2597 u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);
e8e26350 2598
4ccc650c
DS		 2599 if (!ixgbe_is_sfp(hw))
2600 return;
2601
2602 /* Later MAC's use different SDP */
2603 if (hw->mac.type >= ixgbe_mac_X540)
2604 eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
2605
2606 if (eicr & eicr_mask) {
73c4b7cd 2607 /* Clear the interrupt */
4ccc650c 2608 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
7086400d
AD		 2609 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2610 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
58e7cd24 2611 adapter->sfp_poll_time = 0;
7086400d
AD		 2612 ixgbe_service_event_schedule(adapter);
2613 }
73c4b7cd
AD		 2614 }
2615
4ccc650c
DS		 2616 if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
2617 (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
e8e26350 2618 /* Clear the interrupt */
9a900eca 2619 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
7086400d
AD		 2620 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2621 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
2622 ixgbe_service_event_schedule(adapter);
2623 }
e8e26350
PW		 2624 }
2625}
2626
cf8280ee
JB		 2627static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
2628{
2629 struct ixgbe_hw *hw = &adapter->hw;
2630
2631 adapter->lsc_int++;
2632 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2633 adapter->link_check_timeout = jiffies;
2634 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2635 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
8a0717f3 2636 IXGBE_WRITE_FLUSH(hw);
93c52dd0 2637 ixgbe_service_event_schedule(adapter);
cf8280ee
JB		 2638 }
2639}
2640
fe49f04a
AD		 2641static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
2642 u64 qmask)
2643{
2644 u32 mask;
bd508178 2645 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 2646
bd508178
AD		 2647 switch (hw->mac.type) {
2648 case ixgbe_mac_82598EB:
fe49f04a 2649 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
bd508178
AD		 2650 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
2651 break;
2652 case ixgbe_mac_82599EB:
b93a2226 2653 case ixgbe_mac_X540:
9a75a1ac
DS		 2654 case ixgbe_mac_X550:
2655 case ixgbe_mac_X550EM_x:
49425dfc 2656 case ixgbe_mac_x550em_a:
fe49f04a 2657 mask = (qmask & 0xFFFFFFFF);
bd508178
AD		 2658 if (mask)
2659 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
fe49f04a 2660 mask = (qmask >> 32);
bd508178
AD		 2661 if (mask)
2662 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
2663 break;
2664 default:
2665 break;
fe49f04a
AD		 2666 }
2667 /* skip the flush */
2668}
2669
2670static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
e8e9f696 2671 u64 qmask)
fe49f04a
AD		 2672{
2673 u32 mask;
bd508178 2674 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 2675
bd508178
AD		 2676 switch (hw->mac.type) {
2677 case ixgbe_mac_82598EB:
fe49f04a 2678 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
bd508178
AD		 2679 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
2680 break;
2681 case ixgbe_mac_82599EB:
b93a2226 2682 case ixgbe_mac_X540:
9a75a1ac
DS		 2683 case ixgbe_mac_X550:
2684 case ixgbe_mac_X550EM_x:
49425dfc 2685 case ixgbe_mac_x550em_a:
fe49f04a 2686 mask = (qmask & 0xFFFFFFFF);
bd508178
AD		 2687 if (mask)
2688 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
fe49f04a 2689 mask = (qmask >> 32);
bd508178
AD		 2690 if (mask)
2691 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
2692 break;
2693 default:
2694 break;
fe49f04a
AD		 2695 }
2696 /* skip the flush */
2697}
2698
021230d4 2699/**
2c4af694
AD		 2700 * ixgbe_irq_enable - Enable default interrupt generation settings
2701 * @adapter: board private structure
021230d4 2702 **/
2c4af694
AD		 2703static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2704 bool flush)
9a799d71 2705{
9a900eca 2706 struct ixgbe_hw *hw = &adapter->hw;
2c4af694 2707 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
9a799d71 2708
2c4af694
AD		 2709 /* don't reenable LSC while waiting for link */
2710 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
2711 mask &= ~IXGBE_EIMS_LSC;
9a799d71 2712
2c4af694 2713 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
4f51bf70
JK		 2714 switch (adapter->hw.mac.type) {
2715 case ixgbe_mac_82599EB:
9a900eca 2716 mask |= IXGBE_EIMS_GPI_SDP0(hw);
4f51bf70
JK		 2717 break;
2718 case ixgbe_mac_X540:
9a75a1ac
DS		 2719 case ixgbe_mac_X550:
2720 case ixgbe_mac_X550EM_x:
49425dfc 2721 case ixgbe_mac_x550em_a:
4f51bf70
JK		 2722 mask |= IXGBE_EIMS_TS;
2723 break;
2724 default:
2725 break;
2726 }
2c4af694 2727 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
9a900eca 2728 mask |= IXGBE_EIMS_GPI_SDP1(hw);
2c4af694
AD		 2729 switch (adapter->hw.mac.type) {
2730 case ixgbe_mac_82599EB:
9a900eca
DS		 2731 mask |= IXGBE_EIMS_GPI_SDP1(hw);
2732 mask |= IXGBE_EIMS_GPI_SDP2(hw);
9a75a1ac 2733 /* fall through */
858bc081 2734 case ixgbe_mac_X540:
9a75a1ac
DS		 2735 case ixgbe_mac_X550:
2736 case ixgbe_mac_X550EM_x:
49425dfc
MR		 2737 case ixgbe_mac_x550em_a:
2738 if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
2d40cd17 2739 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
49425dfc 2740 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)
cbd45ec7 2741 mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);
597f22d6
DS		 2742 if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)
2743 mask |= IXGBE_EICR_GPI_SDP0_X540;
858bc081 2744 mask |= IXGBE_EIMS_ECC;
2c4af694
AD		 2745 mask |= IXGBE_EIMS_MAILBOX;
2746 break;
2747 default:
2748 break;
9a799d71 2749 }
db0677fa 2750
2c4af694
AD		 2751 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2752 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
2753 mask |= IXGBE_EIMS_FLOW_DIR;
9a799d71 2754
2c4af694
AD		 2755 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2756 if (queues)
2757 ixgbe_irq_enable_queues(adapter, ~0);
2758 if (flush)
2759 IXGBE_WRITE_FLUSH(&adapter->hw);
9a799d71
AK		 2760}
2761
2c4af694 2762static irqreturn_t ixgbe_msix_other(int irq, void *data)
f0848276 2763{
a65151ba 2764 struct ixgbe_adapter *adapter = data;
9a799d71 2765 struct ixgbe_hw *hw = &adapter->hw;
54037505 2766 u32 eicr;
91281fd3 2767
54037505
DS		 2768 /*
2769 * Workaround for Silicon errata. Use clear-by-write instead
2770 * of clear-by-read. Reading with EICS will return the
2771 * interrupt causes without clearing, which later be done
2772 * with the write to EICR.
2773 */
2774 eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
d87d8307
JK		 2775
2776 /* The lower 16bits of the EICR register are for the queue interrupts
dbedd44e 2777 * which should be masked here in order to not accidentally clear them if
d87d8307
JK		 2778 * the bits are high when ixgbe_msix_other is called. There is a race
2779 * condition otherwise which results in possible performance loss
2780 * especially if the ixgbe_msix_other interrupt is triggering
2781 * consistently (as it would when PPS is turned on for the X540 device)
2782 */
2783 eicr &= 0xFFFF0000;
2784
54037505 2785 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
33cf09c9 2786
cf8280ee
JB		 2787 if (eicr & IXGBE_EICR_LSC)
2788 ixgbe_check_lsc(adapter);
f0848276 2789
1cdd1ec8
GR		 2790 if (eicr & IXGBE_EICR_MAILBOX)
2791 ixgbe_msg_task(adapter);
efe3d3c8 2792
bd508178
AD		 2793 switch (hw->mac.type) {
2794 case ixgbe_mac_82599EB:
b93a2226 2795 case ixgbe_mac_X540:
9a75a1ac
DS		 2796 case ixgbe_mac_X550:
2797 case ixgbe_mac_X550EM_x:
49425dfc 2798 case ixgbe_mac_x550em_a:
597f22d6
DS		 2799 if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
2800 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
2801 adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;
2802 ixgbe_service_event_schedule(adapter);
2803 IXGBE_WRITE_REG(hw, IXGBE_EICR,
2804 IXGBE_EICR_GPI_SDP0_X540);
2805 }
d773ce2d
DS		 2806 if (eicr & IXGBE_EICR_ECC) {
2807 e_info(link, "Received ECC Err, initiating reset\n");
57ca2a4f 2808 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
d773ce2d
DS		 2809 ixgbe_service_event_schedule(adapter);
2810 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
2811 }
c4cf55e5
PWJ		 2812 /* Handle Flow Director Full threshold interrupt */
2813 if (eicr & IXGBE_EICR_FLOW_DIR) {
d034acf1 2814 int reinit_count = 0;
c4cf55e5 2815 int i;
c4cf55e5 2816 for (i = 0; i < adapter->num_tx_queues; i++) {
d034acf1 2817 struct ixgbe_ring *ring = adapter->tx_ring[i];
7d637bcc 2818 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
d034acf1
AD		 2819 &ring->state))
2820 reinit_count++;
2821 }
2822 if (reinit_count) {
2823 /* no more flow director interrupts until after init */
2824 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
d034acf1
AD		 2825 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
2826 ixgbe_service_event_schedule(adapter);
c4cf55e5
PWJ		 2827 }
2828 }
f0f9778d 2829 ixgbe_check_sfp_event(adapter, eicr);
4f51bf70 2830 ixgbe_check_overtemp_event(adapter, eicr);
bd508178
AD		 2831 break;
2832 default:
2833 break;
c4cf55e5 2834 }
f0848276 2835
bd508178 2836 ixgbe_check_fan_failure(adapter, eicr);
db0677fa 2837
db0677fa 2838 if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
a9763f3c 2839 ixgbe_ptp_check_pps_event(adapter);
efe3d3c8 2840
7086400d 2841 /* re-enable the original interrupt state, no lsc, no queues */
d4f80882 2842 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2c4af694 2843 ixgbe_irq_enable(adapter, false, false);
f0848276 2844
9a799d71 2845 return IRQ_HANDLED;
f0848276 2846}
91281fd3 2847
4ff7fb12 2848static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
91281fd3 2849{
021230d4 2850 struct ixgbe_q_vector *q_vector = data;
91281fd3 2851
9b471446 2852 /* EIAM disabled interrupts (on this vector) for us */
91281fd3 2853
4ff7fb12 2854 if (q_vector->rx.ring || q_vector->tx.ring)
ef2662b2 2855 napi_schedule_irqoff(&q_vector->napi);
91281fd3 2856
9a799d71 2857 return IRQ_HANDLED;
91281fd3
AD		 2858}
2859
eb01b975
AD		 2860/**
2861 * ixgbe_poll - NAPI Rx polling callback
2862 * @napi: structure for representing this polling device
2863 * @budget: how many packets driver is allowed to clean
2864 *
2865 * This function is used for legacy and MSI, NAPI mode
2866 **/
8af3c33f 2867int ixgbe_poll(struct napi_struct *napi, int budget)
eb01b975
AD		 2868{
2869 struct ixgbe_q_vector *q_vector =
2870 container_of(napi, struct ixgbe_q_vector, napi);
2871 struct ixgbe_adapter *adapter = q_vector->adapter;
2872 struct ixgbe_ring *ring;
32b3e08f 2873 int per_ring_budget, work_done = 0;
eb01b975
AD		 2874 bool clean_complete = true;
2875
2876#ifdef CONFIG_IXGBE_DCA
2877 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2878 ixgbe_update_dca(q_vector);
2879#endif
2880
8220bbc1
AD		 2881 ixgbe_for_each_ring(ring, q_vector->tx) {
2882 if (!ixgbe_clean_tx_irq(q_vector, ring, budget))
2883 clean_complete = false;
2884 }
eb01b975 2885
3ffc1af5
ED		 2886 /* Exit if we are called by netpoll */
2887 if (budget <= 0)
5a85e737
ET		 2888 return budget;
2889
eb01b975
AD		 2890 /* attempt to distribute budget to each queue fairly, but don't allow
2891 * the budget to go below 1 because we'll exit polling */
2892 if (q_vector->rx.count > 1)
2893 per_ring_budget = max(budget/q_vector->rx.count, 1);
2894 else
2895 per_ring_budget = budget;
2896
32b3e08f
JB		 2897 ixgbe_for_each_ring(ring, q_vector->rx) {
2898 int cleaned = ixgbe_clean_rx_irq(q_vector, ring,
2899 per_ring_budget);
2900
2901 work_done += cleaned;
8220bbc1
AD		 2902 if (cleaned >= per_ring_budget)
2903 clean_complete = false;
32b3e08f 2904 }
eb01b975
AD		 2905
2906 /* If all work not completed, return budget and keep polling */
2907 if (!clean_complete)
2908 return budget;
2909
2910 /* all work done, exit the polling mode */
32b3e08f 2911 napi_complete_done(napi, work_done);
eb01b975
AD		 2912 if (adapter->rx_itr_setting & 1)
2913 ixgbe_set_itr(q_vector);
2914 if (!test_bit(__IXGBE_DOWN, &adapter->state))
b4f47a48 2915 ixgbe_irq_enable_queues(adapter, BIT_ULL(q_vector->v_idx));
eb01b975 2916
4b732cd4 2917 return min(work_done, budget - 1);
eb01b975
AD		 2918}
2919
021230d4
AV		 2920/**
2921 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2922 * @adapter: board private structure
2923 *
2924 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2925 * interrupts from the kernel.
2926 **/
2927static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2928{
2929 struct net_device *netdev = adapter->netdev;
207867f5 2930 int vector, err;
e8e9f696 2931 int ri = 0, ti = 0;
021230d4 2932
49c7ffbe 2933 for (vector = 0; vector < adapter->num_q_vectors; vector++) {
d0759ebb 2934 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
207867f5 2935 struct msix_entry *entry = &adapter->msix_entries[vector];
cb13fc20 2936
4ff7fb12 2937 if (q_vector->tx.ring && q_vector->rx.ring) {
9fe93afd 2938 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4ff7fb12
AD		 2939 "%s-%s-%d", netdev->name, "TxRx", ri++);
2940 ti++;
2941 } else if (q_vector->rx.ring) {
9fe93afd 2942 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4ff7fb12
AD		 2943 "%s-%s-%d", netdev->name, "rx", ri++);
2944 } else if (q_vector->tx.ring) {
9fe93afd 2945 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4ff7fb12 2946 "%s-%s-%d", netdev->name, "tx", ti++);
d0759ebb
AD		 2947 } else {
2948 /* skip this unused q_vector */
2949 continue;
32aa77a4 2950 }
207867f5
AD		 2951 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2952 q_vector->name, q_vector);
9a799d71 2953 if (err) {
396e799c 2954 e_err(probe, "request_irq failed for MSIX interrupt "
849c4542 2955 "Error: %d\n", err);
021230d4 2956 goto free_queue_irqs;
9a799d71 2957 }
207867f5
AD		 2958 /* If Flow Director is enabled, set interrupt affinity */
2959 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2960 /* assign the mask for this irq */
2961 irq_set_affinity_hint(entry->vector,
de88eeeb 2962 &q_vector->affinity_mask);
207867f5 2963 }
9a799d71
AK		 2964 }
2965
021230d4 2966 err = request_irq(adapter->msix_entries[vector].vector,
2c4af694 2967 ixgbe_msix_other, 0, netdev->name, adapter);
9a799d71 2968 if (err) {
de88eeeb 2969 e_err(probe, "request_irq for msix_other failed: %d\n", err);
021230d4 2970 goto free_queue_irqs;
9a799d71
AK		 2971 }
2972
9a799d71
AK		 2973 return 0;
2974
021230d4 2975free_queue_irqs:
207867f5
AD		 2976 while (vector) {
2977 vector--;
2978 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2979 NULL);
2980 free_irq(adapter->msix_entries[vector].vector,
2981 adapter->q_vector[vector]);
2982 }
021230d4
AV		 2983 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2984 pci_disable_msix(adapter->pdev);
9a799d71
AK		 2985 kfree(adapter->msix_entries);
2986 adapter->msix_entries = NULL;
9a799d71
AK		 2987 return err;
2988}
2989
2990/**
021230d4 2991 * ixgbe_intr - legacy mode Interrupt Handler
9a799d71
AK		 2992 * @irq: interrupt number
2993 * @data: pointer to a network interface device structure
9a799d71
AK		 2994 **/
2995static irqreturn_t ixgbe_intr(int irq, void *data)
2996{
a65151ba 2997 struct ixgbe_adapter *adapter = data;
9a799d71 2998 struct ixgbe_hw *hw = &adapter->hw;
7a921c93 2999 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
9a799d71
AK		 3000 u32 eicr;
3001
54037505 3002 /*
24ddd967 3003 * Workaround for silicon errata #26 on 82598. Mask the interrupt
54037505
DS		 3004 * before the read of EICR.
3005 */
3006 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
3007
021230d4 3008 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
52f33af8 3009 * therefore no explicit interrupt disable is necessary */
021230d4 3010 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
f47cf66e 3011 if (!eicr) {
6af3b9eb
ET		 3012 /*
3013 * shared interrupt alert!
f47cf66e 3014 * make sure interrupts are enabled because the read will
6af3b9eb
ET		 3015 * have disabled interrupts due to EIAM
3016 * finish the workaround of silicon errata on 82598. Unmask
3017 * the interrupt that we masked before the EICR read.
3018 */
3019 if (!test_bit(__IXGBE_DOWN, &adapter->state))
3020 ixgbe_irq_enable(adapter, true, true);
9a799d71 3021 return IRQ_NONE; /* Not our interrupt */
f47cf66e 3022 }
9a799d71 3023
cf8280ee
JB		 3024 if (eicr & IXGBE_EICR_LSC)
3025 ixgbe_check_lsc(adapter);
021230d4 3026
bd508178
AD		 3027 switch (hw->mac.type) {
3028 case ixgbe_mac_82599EB:
e8e26350 3029 ixgbe_check_sfp_event(adapter, eicr);
0ccb974d
DS		 3030 /* Fall through */
3031 case ixgbe_mac_X540:
9a75a1ac
DS		 3032 case ixgbe_mac_X550:
3033 case ixgbe_mac_X550EM_x:
49425dfc 3034 case ixgbe_mac_x550em_a:
d773ce2d
DS		 3035 if (eicr & IXGBE_EICR_ECC) {
3036 e_info(link, "Received ECC Err, initiating reset\n");
57ca2a4f 3037 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
d773ce2d
DS		 3038 ixgbe_service_event_schedule(adapter);
3039 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
3040 }
4f51bf70 3041 ixgbe_check_overtemp_event(adapter, eicr);
bd508178
AD		 3042 break;
3043 default:
3044 break;
3045 }
e8e26350 3046
0befdb3e 3047 ixgbe_check_fan_failure(adapter, eicr);
db0677fa 3048 if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
a9763f3c 3049 ixgbe_ptp_check_pps_event(adapter);
0befdb3e 3050
b9f6ed2b 3051 /* would disable interrupts here but EIAM disabled it */
ef2662b2 3052 napi_schedule_irqoff(&q_vector->napi);
9a799d71 3053
6af3b9eb
ET		 3054 /*
3055 * re-enable link(maybe) and non-queue interrupts, no flush.
3056 * ixgbe_poll will re-enable the queue interrupts
3057 */
6af3b9eb
ET		 3058 if (!test_bit(__IXGBE_DOWN, &adapter->state))
3059 ixgbe_irq_enable(adapter, false, false);
3060
9a799d71
AK		 3061 return IRQ_HANDLED;
3062}
3063
3064/**
3065 * ixgbe_request_irq - initialize interrupts
3066 * @adapter: board private structure
3067 *
3068 * Attempts to configure interrupts using the best available
3069 * capabilities of the hardware and kernel.
3070 **/
021230d4 3071static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
9a799d71
AK		 3072{
3073 struct net_device *netdev = adapter->netdev;
021230d4 3074 int err;
9a799d71 3075
4cc6df29 3076 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
021230d4 3077 err = ixgbe_request_msix_irqs(adapter);
4cc6df29 3078 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
a0607fd3 3079 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
a65151ba 3080 netdev->name, adapter);
4cc6df29 3081 else
a0607fd3 3082 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
a65151ba 3083 netdev->name, adapter);
9a799d71 3084
de88eeeb 3085 if (err)
396e799c 3086 e_err(probe, "request_irq failed, Error %d\n", err);
9a799d71 3087
9a799d71
AK		 3088 return err;
3089}
3090
3091static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
3092{
49c7ffbe 3093 int vector;
9a799d71 3094
49c7ffbe
AD		 3095 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
3096 free_irq(adapter->pdev->irq, adapter);
3097 return;
3098 }
4cc6df29 3099
1fa71252
MR		 3100 if (!adapter->msix_entries)
3101 return;
3102
49c7ffbe
AD		 3103 for (vector = 0; vector < adapter->num_q_vectors; vector++) {
3104 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
3105 struct msix_entry *entry = &adapter->msix_entries[vector];
894ff7cf 3106
49c7ffbe
AD		 3107 /* free only the irqs that were actually requested */
3108 if (!q_vector->rx.ring && !q_vector->tx.ring)
3109 continue;
207867f5 3110
49c7ffbe
AD		 3111 /* clear the affinity_mask in the IRQ descriptor */
3112 irq_set_affinity_hint(entry->vector, NULL);
3113
3114 free_irq(entry->vector, q_vector);
9a799d71 3115 }
49c7ffbe 3116
90c6f877 3117 free_irq(adapter->msix_entries[vector].vector, adapter);
9a799d71
AK		 3118}
3119
22d5a71b
JB		 3120/**
3121 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
3122 * @adapter: board private structure
3123 **/
3124static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
3125{
bd508178
AD		 3126 switch (adapter->hw.mac.type) {
3127 case ixgbe_mac_82598EB:
835462fc 3128 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
bd508178
AD		 3129 break;
3130 case ixgbe_mac_82599EB:
b93a2226 3131 case ixgbe_mac_X540:
9a75a1ac
DS		 3132 case ixgbe_mac_X550:
3133 case ixgbe_mac_X550EM_x:
49425dfc 3134 case ixgbe_mac_x550em_a:
835462fc
NS		 3135 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
3136 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
22d5a71b 3137 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
bd508178
AD		 3138 break;
3139 default:
3140 break;
22d5a71b
JB		 3141 }
3142 IXGBE_WRITE_FLUSH(&adapter->hw);
3143 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
49c7ffbe
AD		 3144 int vector;
3145
3146 for (vector = 0; vector < adapter->num_q_vectors; vector++)
3147 synchronize_irq(adapter->msix_entries[vector].vector);
3148
3149 synchronize_irq(adapter->msix_entries[vector++].vector);
22d5a71b
JB		 3150 } else {
3151 synchronize_irq(adapter->pdev->irq);
3152 }
3153}
3154
9a799d71
AK		 3155/**
3156 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
3157 *
3158 **/
3159static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
3160{
d5bf4f67 3161 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
9a799d71 3162
d5bf4f67 3163 ixgbe_write_eitr(q_vector);
9a799d71 3164
e8e26350
PW		 3165 ixgbe_set_ivar(adapter, 0, 0, 0);
3166 ixgbe_set_ivar(adapter, 1, 0, 0);
021230d4 3167
396e799c 3168 e_info(hw, "Legacy interrupt IVAR setup done\n");
9a799d71
AK		 3169}
3170
43e69bf0
AD		 3171/**
3172 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
3173 * @adapter: board private structure
3174 * @ring: structure containing ring specific data
3175 *
3176 * Configure the Tx descriptor ring after a reset.
3177 **/
84418e3b
AD		 3178void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
3179 struct ixgbe_ring *ring)
43e69bf0
AD		 3180{
3181 struct ixgbe_hw *hw = &adapter->hw;
3182 u64 tdba = ring->dma;
2f1860b8 3183 int wait_loop = 10;
b88c6de2 3184 u32 txdctl = IXGBE_TXDCTL_ENABLE;
bf29ee6c 3185 u8 reg_idx = ring->reg_idx;
43e69bf0 3186
2f1860b8 3187 /* disable queue to avoid issues while updating state */
b88c6de2 3188 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2f1860b8
AD		 3189 IXGBE_WRITE_FLUSH(hw);
3190
43e69bf0 3191 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
e8e9f696 3192 (tdba & DMA_BIT_MASK(32)));
43e69bf0
AD		 3193 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
3194 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
3195 ring->count * sizeof(union ixgbe_adv_tx_desc));
3196 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
3197 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2a1a091c 3198 ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);
43e69bf0 3199
b88c6de2
AD		 3200 /*
3201 * set WTHRESH to encourage burst writeback, it should not be set
67da097e
ET		 3202 * higher than 1 when:
3203 * - ITR is 0 as it could cause false TX hangs
3204 * - ITR is set to > 100k int/sec and BQL is enabled
b88c6de2
AD		 3205 *
3206 * In order to avoid issues WTHRESH + PTHRESH should always be equal
3207 * to or less than the number of on chip descriptors, which is
3208 * currently 40.
3209 */
67da097e 3210 if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))
b4f47a48 3211 txdctl |= 1u << 16; /* WTHRESH = 1 */
b88c6de2 3212 else
b4f47a48 3213 txdctl |= 8u << 16; /* WTHRESH = 8 */
b88c6de2 3214
e954b374
AD		 3215 /*
3216 * Setting PTHRESH to 32 both improves performance
3217 * and avoids a TX hang with DFP enabled
3218 */
b4f47a48 3219 txdctl |= (1u << 8) | /* HTHRESH = 1 */
b88c6de2 3220 32; /* PTHRESH = 32 */
2f1860b8
AD		 3221
3222 /* reinitialize flowdirector state */
39cb681b 3223 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
ee9e0f0b
AD		 3224 ring->atr_sample_rate = adapter->atr_sample_rate;
3225 ring->atr_count = 0;
3226 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
3227 } else {
3228 ring->atr_sample_rate = 0;
3229 }
2f1860b8 3230
fd786b7b
AD		 3231 /* initialize XPS */
3232 if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) {
3233 struct ixgbe_q_vector *q_vector = ring->q_vector;
3234
3235 if (q_vector)
2a47fa45 3236 netif_set_xps_queue(ring->netdev,
fd786b7b
AD		 3237 &q_vector->affinity_mask,
3238 ring->queue_index);
3239 }
3240
c84d324c
JF		 3241 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
3242
2f1860b8 3243 /* enable queue */
2f1860b8
AD		 3244 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
3245
3246 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3247 if (hw->mac.type == ixgbe_mac_82598EB &&
3248 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3249 return;
3250
3251 /* poll to verify queue is enabled */
3252 do {
032b4325 3253 usleep_range(1000, 2000);
2f1860b8
AD		 3254 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
3255 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
3256 if (!wait_loop)
a55defd8 3257 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
43e69bf0
AD		 3258}
3259
120ff942
AD		 3260static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
3261{
3262 struct ixgbe_hw *hw = &adapter->hw;
671c0adb 3263 u32 rttdcs, mtqc;
8b1c0b24 3264 u8 tcs = netdev_get_num_tc(adapter->netdev);
120ff942
AD		 3265
3266 if (hw->mac.type == ixgbe_mac_82598EB)
3267 return;
3268
3269 /* disable the arbiter while setting MTQC */
3270 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3271 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3272 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3273
3274 /* set transmit pool layout */
671c0adb
AD		 3275 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3276 mtqc = IXGBE_MTQC_VT_ENA;
3277 if (tcs > 4)
3278 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3279 else if (tcs > 1)
3280 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
e24fcf28
AD		 3281 else if (adapter->ring_feature[RING_F_VMDQ].mask ==
3282 IXGBE_82599_VMDQ_4Q_MASK)
671c0adb
AD		 3283 mtqc |= IXGBE_MTQC_32VF;
3284 else
3285 mtqc |= IXGBE_MTQC_64VF;
3286 } else {
3287 if (tcs > 4)
3288 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3289 else if (tcs > 1)
3290 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
8b1c0b24 3291 else
671c0adb
AD		 3292 mtqc = IXGBE_MTQC_64Q_1PB;
3293 }
120ff942 3294
671c0adb 3295 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
120ff942 3296
671c0adb
AD		 3297 /* Enable Security TX Buffer IFG for multiple pb */
3298 if (tcs) {
3299 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3300 sectx |= IXGBE_SECTX_DCB;
3301 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
120ff942
AD		 3302 }
3303
3304 /* re-enable the arbiter */
3305 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3306 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3307}
3308
9a799d71 3309/**
3a581073 3310 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
9a799d71
AK		 3311 * @adapter: board private structure
3312 *
3313 * Configure the Tx unit of the MAC after a reset.
3314 **/
3315static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
3316{
2f1860b8
AD		 3317 struct ixgbe_hw *hw = &adapter->hw;
3318 u32 dmatxctl;
43e69bf0 3319 u32 i;
9a799d71 3320
2f1860b8
AD		 3321 ixgbe_setup_mtqc(adapter);
3322
3323 if (hw->mac.type != ixgbe_mac_82598EB) {
3324 /* DMATXCTL.EN must be before Tx queues are enabled */
3325 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3326 dmatxctl |= IXGBE_DMATXCTL_TE;
3327 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3328 }
3329
9a799d71 3330 /* Setup the HW Tx Head and Tail descriptor pointers */
43e69bf0
AD		 3331 for (i = 0; i < adapter->num_tx_queues; i++)
3332 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
9a799d71
AK		 3333}
3334
3ebe8fde
AD		 3335static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
3336 struct ixgbe_ring *ring)
3337{
3338 struct ixgbe_hw *hw = &adapter->hw;
3339 u8 reg_idx = ring->reg_idx;
3340 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
3341
3342 srrctl |= IXGBE_SRRCTL_DROP_EN;
3343
3344 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3345}
3346
3347static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
3348 struct ixgbe_ring *ring)
3349{
3350 struct ixgbe_hw *hw = &adapter->hw;
3351 u8 reg_idx = ring->reg_idx;
3352 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
3353
3354 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
3355
3356 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3357}
3358
3359#ifdef CONFIG_IXGBE_DCB
3360void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
3361#else
3362static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
3363#endif
3364{
3365 int i;
3366 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
3367
3368 if (adapter->ixgbe_ieee_pfc)
3369 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
3370
3371 /*
3372 * We should set the drop enable bit if:
3373 * SR-IOV is enabled
3374 * or
3375 * Number of Rx queues > 1 and flow control is disabled
3376 *
3377 * This allows us to avoid head of line blocking for security
3378 * and performance reasons.
3379 */
3380 if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
3381 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
3382 for (i = 0; i < adapter->num_rx_queues; i++)
3383 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
3384 } else {
3385 for (i = 0; i < adapter->num_rx_queues; i++)
3386 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
3387 }
3388}
3389
e8e26350 3390#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
cc41ac7c 3391
a6616b42 3392static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
e8e9f696 3393 struct ixgbe_ring *rx_ring)
cc41ac7c 3394{
45e9baa5 3395 struct ixgbe_hw *hw = &adapter->hw;
cc41ac7c 3396 u32 srrctl;
bf29ee6c 3397 u8 reg_idx = rx_ring->reg_idx;
3be1adfb 3398
45e9baa5
AD		 3399 if (hw->mac.type == ixgbe_mac_82598EB) {
3400 u16 mask = adapter->ring_feature[RING_F_RSS].mask;
cc41ac7c 3401
45e9baa5
AD		 3402 /*
3403 * if VMDq is not active we must program one srrctl register
3404 * per RSS queue since we have enabled RDRXCTL.MVMEN
3405 */
3406 reg_idx &= mask;
3407 }
cc41ac7c 3408
45e9baa5
AD		 3409 /* configure header buffer length, needed for RSC */
3410 srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
afafd5b0 3411
45e9baa5 3412 /* configure the packet buffer length */
f800326d 3413 srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
45e9baa5
AD		 3414
3415 /* configure descriptor type */
f800326d 3416 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
e8e26350 3417
45e9baa5 3418 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
cc41ac7c 3419}
9a799d71 3420
dfaf891d 3421/**
a897a2ad 3422 * ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries
dfaf891d
VZ		 3423 * @adapter: device handle
3424 *
3425 * - 82598/82599/X540: 128
3426 * - X550(non-SRIOV mode): 512
3427 * - X550(SRIOV mode): 64
3428 */
7f276efb 3429u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)
dfaf891d
VZ		 3430{
3431 if (adapter->hw.mac.type < ixgbe_mac_X550)
3432 return 128;
3433 else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3434 return 64;
3435 else
3436 return 512;
3437}
3438
3439/**
a897a2ad 3440 * ixgbe_store_reta - Write the RETA table to HW
dfaf891d
VZ		 3441 * @adapter: device handle
3442 *
3443 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
3444 */
1c7cf078 3445void ixgbe_store_reta(struct ixgbe_adapter *adapter)
0cefafad 3446{
dfaf891d 3447 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
05abb126 3448 struct ixgbe_hw *hw = &adapter->hw;
d1b849b9 3449 u32 reta = 0;
dfaf891d
VZ		 3450 u32 indices_multi;
3451 u8 *indir_tbl = adapter->rss_indir_tbl;
05abb126 3452
0f9b232b 3453 /* Fill out the redirection table as follows:
dfaf891d
VZ		 3454 * - 82598: 8 bit wide entries containing pair of 4 bit RSS
3455 * indices.
3456 * - 82599/X540: 8 bit wide entries containing 4 bit RSS index
3457 * - X550: 8 bit wide entries containing 6 bit RSS index
0f9b232b
DS		 3458 */
3459 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
3460 indices_multi = 0x11;
3461 else
3462 indices_multi = 0x1;
3463
dfaf891d
VZ		 3464 /* Write redirection table to HW */
3465 for (i = 0; i < reta_entries; i++) {
3466 reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;
0f9b232b
DS		 3467 if ((i & 3) == 3) {
3468 if (i < 128)
3469 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3470 else
3471 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3472 reta);
dfaf891d 3473 reta = 0;
0f9b232b
DS		 3474 }
3475 }
3476}
3477
dfaf891d 3478/**
a897a2ad 3479 * ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)
dfaf891d
VZ		 3480 * @adapter: device handle
3481 *
3482 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
3483 */
3484static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
0f9b232b 3485{
dfaf891d 3486 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
0f9b232b
DS		 3487 struct ixgbe_hw *hw = &adapter->hw;
3488 u32 vfreta = 0;
dfaf891d
VZ		 3489 unsigned int pf_pool = adapter->num_vfs;
3490
3491 /* Write redirection table to HW */
3492 for (i = 0; i < reta_entries; i++) {
3493 vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
3494 if ((i & 3) == 3) {
3495 IXGBE_WRITE_REG(hw, IXGBE_PFVFRETA(i >> 2, pf_pool),
3496 vfreta);
3497 vfreta = 0;
3498 }
3499 }
3500}
3501
3502static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
3503{
3504 struct ixgbe_hw *hw = &adapter->hw;
3505 u32 i, j;
3506 u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
3507 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
3508
e24fcf28 3509 /* Program table for at least 4 queues w/ SR-IOV so that VFs can
dfaf891d
VZ		 3510 * make full use of any rings they may have. We will use the
3511 * PSRTYPE register to control how many rings we use within the PF.
3512 */
e24fcf28
AD		 3513 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
3514 rss_i = 4;
dfaf891d
VZ		 3515
3516 /* Fill out hash function seeds */
3517 for (i = 0; i < 10; i++)
3518 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
3519
3520 /* Fill out redirection table */
3521 memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));
3522
3523 for (i = 0, j = 0; i < reta_entries; i++, j++) {
3524 if (j == rss_i)
3525 j = 0;
3526
3527 adapter->rss_indir_tbl[i] = j;
3528 }
3529
3530 ixgbe_store_reta(adapter);
3531}
3532
3533static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
3534{
3535 struct ixgbe_hw *hw = &adapter->hw;
0f9b232b
DS		 3536 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
3537 unsigned int pf_pool = adapter->num_vfs;
3538 int i, j;
3539
3540 /* Fill out hash function seeds */
3541 for (i = 0; i < 10; i++)
dfaf891d
VZ		 3542 IXGBE_WRITE_REG(hw, IXGBE_PFVFRSSRK(i, pf_pool),
3543 adapter->rss_key[i]);
0f9b232b
DS		 3544
3545 /* Fill out the redirection table */
3546 for (i = 0, j = 0; i < 64; i++, j++) {
671c0adb 3547 if (j == rss_i)
05abb126 3548 j = 0;
dfaf891d
VZ		 3549
3550 adapter->rss_indir_tbl[i] = j;
05abb126 3551 }
dfaf891d
VZ		 3552
3553 ixgbe_store_vfreta(adapter);
d1b849b9
DS		 3554}
3555
3556static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
3557{
3558 struct ixgbe_hw *hw = &adapter->hw;
0f9b232b 3559 u32 mrqc = 0, rss_field = 0, vfmrqc = 0;
d1b849b9 3560 u32 rxcsum;
0cefafad 3561
05abb126
AD		 3562 /* Disable indicating checksum in descriptor, enables RSS hash */
3563 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3564 rxcsum |= IXGBE_RXCSUM_PCSD;
3565 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3566
671c0adb 3567 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
fbe7ca7f 3568 if (adapter->ring_feature[RING_F_RSS].mask)
671c0adb 3569 mrqc = IXGBE_MRQC_RSSEN;
8b1c0b24 3570 } else {
671c0adb
AD		 3571 u8 tcs = netdev_get_num_tc(adapter->netdev);
3572
3573 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3574 if (tcs > 4)
3575 mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */
3576 else if (tcs > 1)
3577 mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */
e24fcf28
AD		 3578 else if (adapter->ring_feature[RING_F_VMDQ].mask ==
3579 IXGBE_82599_VMDQ_4Q_MASK)
671c0adb 3580 mrqc = IXGBE_MRQC_VMDQRSS32EN;
8b1c0b24 3581 else
671c0adb
AD		 3582 mrqc = IXGBE_MRQC_VMDQRSS64EN;
3583 } else {
3584 if (tcs > 4)
8b1c0b24 3585 mrqc = IXGBE_MRQC_RTRSS8TCEN;
671c0adb
AD		 3586 else if (tcs > 1)
3587 mrqc = IXGBE_MRQC_RTRSS4TCEN;
3588 else
3589 mrqc = IXGBE_MRQC_RSSEN;
8b1c0b24 3590 }
0cefafad
JB		 3591 }
3592
05abb126 3593 /* Perform hash on these packet types */
d1b849b9
DS		 3594 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |
3595 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
3596 IXGBE_MRQC_RSS_FIELD_IPV6 |
3597 IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
05abb126 3598
ef6afc0c 3599 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
d1b849b9 3600 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
ef6afc0c 3601 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
d1b849b9 3602 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
ef6afc0c 3603
dfaf891d 3604 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key));
0f9b232b
DS		 3605 if ((hw->mac.type >= ixgbe_mac_X550) &&
3606 (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
3607 unsigned int pf_pool = adapter->num_vfs;
3608
3609 /* Enable VF RSS mode */
3610 mrqc |= IXGBE_MRQC_MULTIPLE_RSS;
3611 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3612
3613 /* Setup RSS through the VF registers */
dfaf891d 3614 ixgbe_setup_vfreta(adapter);
0f9b232b
DS		 3615 vfmrqc = IXGBE_MRQC_RSSEN;
3616 vfmrqc |= rss_field;
3617 IXGBE_WRITE_REG(hw, IXGBE_PFVFMRQC(pf_pool), vfmrqc);
3618 } else {
dfaf891d 3619 ixgbe_setup_reta(adapter);
0f9b232b
DS		 3620 mrqc |= rss_field;
3621 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3622 }
0cefafad
JB		 3623}
3624
bb5a9ad2
NS		 3625/**
3626 * ixgbe_configure_rscctl - enable RSC for the indicated ring
3627 * @adapter: address of board private structure
3628 * @index: index of ring to set
bb5a9ad2 3629 **/
082757af 3630static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
7367096a 3631 struct ixgbe_ring *ring)
bb5a9ad2 3632{
bb5a9ad2 3633 struct ixgbe_hw *hw = &adapter->hw;
bb5a9ad2 3634 u32 rscctrl;
bf29ee6c 3635 u8 reg_idx = ring->reg_idx;
7367096a 3636
7d637bcc 3637 if (!ring_is_rsc_enabled(ring))
7367096a 3638 return;
bb5a9ad2 3639
7367096a 3640 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
bb5a9ad2
NS		 3641 rscctrl |= IXGBE_RSCCTL_RSCEN;
3642 /*
3643 * we must limit the number of descriptors so that the
3644 * total size of max desc * buf_len is not greater
642c680e 3645 * than 65536
bb5a9ad2 3646 */
f800326d 3647 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
7367096a 3648 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
bb5a9ad2
NS		 3649}
3650
9e10e045
AD		 3651#define IXGBE_MAX_RX_DESC_POLL 10
3652static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
3653 struct ixgbe_ring *ring)
3654{
3655 struct ixgbe_hw *hw = &adapter->hw;
9e10e045
AD		 3656 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3657 u32 rxdctl;
bf29ee6c 3658 u8 reg_idx = ring->reg_idx;
9e10e045 3659
b0483c8f
MR		 3660 if (ixgbe_removed(hw->hw_addr))
3661 return;
9e10e045
AD		 3662 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3663 if (hw->mac.type == ixgbe_mac_82598EB &&
3664 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3665 return;
3666
3667 do {
032b4325 3668 usleep_range(1000, 2000);
9e10e045
AD		 3669 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3670 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
3671
3672 if (!wait_loop) {
3673 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
3674 "the polling period\n", reg_idx);
3675 }
3676}
3677
2d39d576
YZ		 3678void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
3679 struct ixgbe_ring *ring)
3680{
3681 struct ixgbe_hw *hw = &adapter->hw;
3682 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3683 u32 rxdctl;
3684 u8 reg_idx = ring->reg_idx;
3685
b0483c8f
MR		 3686 if (ixgbe_removed(hw->hw_addr))
3687 return;
2d39d576
YZ		 3688 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3689 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
3690
3691 /* write value back with RXDCTL.ENABLE bit cleared */
3692 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3693
3694 if (hw->mac.type == ixgbe_mac_82598EB &&
3695 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3696 return;
3697
3698 /* the hardware may take up to 100us to really disable the rx queue */
3699 do {
3700 udelay(10);
3701 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3702 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
3703
3704 if (!wait_loop) {
3705 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
3706 "the polling period\n", reg_idx);
3707 }
3708}
3709
84418e3b
AD		 3710void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
3711 struct ixgbe_ring *ring)
acd37177
AD		 3712{
3713 struct ixgbe_hw *hw = &adapter->hw;
c3630cc4 3714 union ixgbe_adv_rx_desc *rx_desc;
acd37177 3715 u64 rdba = ring->dma;
9e10e045 3716 u32 rxdctl;
bf29ee6c 3717 u8 reg_idx = ring->reg_idx;
acd37177 3718
9e10e045
AD		 3719 /* disable queue to avoid issues while updating state */
3720 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2d39d576 3721 ixgbe_disable_rx_queue(adapter, ring);
9e10e045 3722
acd37177
AD		 3723 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
3724 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
3725 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
3726 ring->count * sizeof(union ixgbe_adv_rx_desc));
8b75451b
NP		 3727 /* Force flushing of IXGBE_RDLEN to prevent MDD */
3728 IXGBE_WRITE_FLUSH(hw);
3729
acd37177
AD		 3730 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
3731 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
2a1a091c 3732 ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx);
9e10e045
AD		 3733
3734 ixgbe_configure_srrctl(adapter, ring);
3735 ixgbe_configure_rscctl(adapter, ring);
3736
3737 if (hw->mac.type == ixgbe_mac_82598EB) {
3738 /*
3739 * enable cache line friendly hardware writes:
3740 * PTHRESH=32 descriptors (half the internal cache),
3741 * this also removes ugly rx_no_buffer_count increment
3742 * HTHRESH=4 descriptors (to minimize latency on fetch)
3743 * WTHRESH=8 burst writeback up to two cache lines
3744 */
3745 rxdctl &= ~0x3FFFFF;
3746 rxdctl |= 0x080420;
3747 }
3748
c3630cc4
AD		 3749 /* initialize Rx descriptor 0 */
3750 rx_desc = IXGBE_RX_DESC(ring, 0);
3751 rx_desc->wb.upper.length = 0;
3752
9e10e045
AD		 3753 /* enable receive descriptor ring */
3754 rxdctl |= IXGBE_RXDCTL_ENABLE;
3755 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3756
3757 ixgbe_rx_desc_queue_enable(adapter, ring);
7d4987de 3758 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
acd37177
AD		 3759}
3760
48654521
AD		 3761static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
3762{
3763 struct ixgbe_hw *hw = &adapter->hw;
fbe7ca7f 3764 int rss_i = adapter->ring_feature[RING_F_RSS].indices;
2a47fa45 3765 u16 pool;
48654521
AD		 3766
3767 /* PSRTYPE must be initialized in non 82598 adapters */
3768 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
e8e9f696
JP		 3769 IXGBE_PSRTYPE_UDPHDR |
3770 IXGBE_PSRTYPE_IPV4HDR |
48654521 3771 IXGBE_PSRTYPE_L2HDR |
e8e9f696 3772 IXGBE_PSRTYPE_IPV6HDR;
48654521
AD		 3773
3774 if (hw->mac.type == ixgbe_mac_82598EB)
3775 return;
3776
fbe7ca7f 3777 if (rss_i > 3)
b4f47a48 3778 psrtype |= 2u << 29;
fbe7ca7f 3779 else if (rss_i > 1)
b4f47a48 3780 psrtype |= 1u << 29;
48654521 3781
2a47fa45
JF		 3782 for_each_set_bit(pool, &adapter->fwd_bitmask, 32)
3783 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
48654521
AD		 3784}
3785
f5b4a52e
AD		 3786static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
3787{
3788 struct ixgbe_hw *hw = &adapter->hw;
f5b4a52e 3789 u32 reg_offset, vf_shift;
435b19f6 3790 u32 gcr_ext, vmdctl;
de4c7f65 3791 int i;
f5b4a52e
AD		 3792
3793 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
3794 return;
3795
3796 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
435b19f6
AD		 3797 vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
3798 vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
1d9c0bfd 3799 vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
435b19f6
AD		 3800 vmdctl |= IXGBE_VT_CTL_REPLEN;
3801 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
f5b4a52e 3802
1d9c0bfd
AD		 3803 vf_shift = VMDQ_P(0) % 32;
3804 reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
f5b4a52e
AD		 3805
3806 /* Enable only the PF's pool for Tx/Rx */
11f2b494 3807 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), GENMASK(31, vf_shift));
435b19f6 3808 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
11f2b494 3809 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), GENMASK(31, vf_shift));
435b19f6 3810 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
aa2bacb6 3811 if (adapter->bridge_mode == BRIDGE_MODE_VEB)
9b735984 3812 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
f5b4a52e
AD		 3813
3814 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
1d9c0bfd 3815 hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));
f5b4a52e 3816
16369564
AD		 3817 /* clear VLAN promisc flag so VFTA will be updated if necessary */
3818 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
3819
f5b4a52e
AD		 3820 /*
3821 * Set up VF register offsets for selected VT Mode,
3822 * i.e. 32 or 64 VFs for SR-IOV
3823 */
73079ea0
AD		 3824 switch (adapter->ring_feature[RING_F_VMDQ].mask) {
3825 case IXGBE_82599_VMDQ_8Q_MASK:
3826 gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
3827 break;
3828 case IXGBE_82599_VMDQ_4Q_MASK:
3829 gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
3830 break;
3831 default:
3832 gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
3833 break;
3834 }
3835
f5b4a52e
AD		 3836 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
3837
de4c7f65 3838 for (i = 0; i < adapter->num_vfs; i++) {
77f192af
ET		 3839 /* configure spoof checking */
3840 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i,
3841 adapter->vfinfo[i].spoofchk_enabled);
e65ce0d3
VZ		 3842
3843 /* Enable/Disable RSS query feature */
3844 ixgbe_ndo_set_vf_rss_query_en(adapter->netdev, i,
3845 adapter->vfinfo[i].rss_query_enabled);
de4c7f65 3846 }
f5b4a52e
AD		 3847}
3848
477de6ed 3849static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
9a799d71 3850{
9a799d71
AK		 3851 struct ixgbe_hw *hw = &adapter->hw;
3852 struct net_device *netdev = adapter->netdev;
3853 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
477de6ed
AD		 3854 struct ixgbe_ring *rx_ring;
3855 int i;
3856 u32 mhadd, hlreg0;
48654521 3857
63f39bd1 3858#ifdef IXGBE_FCOE
477de6ed
AD		 3859 /* adjust max frame to be able to do baby jumbo for FCoE */
3860 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
3861 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
3862 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
9a799d71 3863
477de6ed 3864#endif /* IXGBE_FCOE */
872844dd
AD		 3865
3866 /* adjust max frame to be at least the size of a standard frame */
3867 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
3868 max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);
3869
477de6ed
AD		 3870 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3871 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
3872 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3873 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
3874
3875 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3876 }
3877
3878 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3879 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
3880 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3881 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
9a799d71 3882
0cefafad
JB		 3883 /*
3884 * Setup the HW Rx Head and Tail Descriptor Pointers and
3885 * the Base and Length of the Rx Descriptor Ring
3886 */
9a799d71 3887 for (i = 0; i < adapter->num_rx_queues; i++) {
4a0b9ca0 3888 rx_ring = adapter->rx_ring[i];
4f4542bf
AD		 3889
3890 clear_ring_rsc_enabled(rx_ring);
3891 clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
3892
7d637bcc
AD		 3893 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
3894 set_ring_rsc_enabled(rx_ring);
4f4542bf
AD		 3895
3896 if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state))
3897 set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);
477de6ed 3898 }
477de6ed
AD		 3899}
3900
7367096a
AD		 3901static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
3902{
3903 struct ixgbe_hw *hw = &adapter->hw;
3904 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3905
3906 switch (hw->mac.type) {
3907 case ixgbe_mac_82598EB:
3908 /*
3909 * For VMDq support of different descriptor types or
3910 * buffer sizes through the use of multiple SRRCTL
3911 * registers, RDRXCTL.MVMEN must be set to 1
3912 *
3913 * also, the manual doesn't mention it clearly but DCA hints
3914 * will only use queue 0's tags unless this bit is set. Side
3915 * effects of setting this bit are only that SRRCTL must be
3916 * fully programmed [0..15]
3917 */
3918 rdrxctl |= IXGBE_RDRXCTL_MVMEN;
3919 break;
052a1a72
MR		 3920 case ixgbe_mac_X550:
3921 case ixgbe_mac_X550EM_x:
49425dfc 3922 case ixgbe_mac_x550em_a:
f961ddae
MR		 3923 if (adapter->num_vfs)
3924 rdrxctl |= IXGBE_RDRXCTL_PSP;
3925 /* fall through for older HW */
7367096a 3926 case ixgbe_mac_82599EB:
b93a2226 3927 case ixgbe_mac_X540:
7367096a
AD		 3928 /* Disable RSC for ACK packets */
3929 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3930 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3931 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3932 /* hardware requires some bits to be set by default */
3933 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
3934 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3935 break;
3936 default:
3937 /* We should do nothing since we don't know this hardware */
3938 return;
3939 }
3940
3941 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3942}
3943
477de6ed
AD		 3944/**
3945 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
3946 * @adapter: board private structure
3947 *
3948 * Configure the Rx unit of the MAC after a reset.
3949 **/
3950static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
3951{
3952 struct ixgbe_hw *hw = &adapter->hw;
477de6ed 3953 int i;
6dcc28b9 3954 u32 rxctrl, rfctl;
477de6ed
AD		 3955
3956 /* disable receives while setting up the descriptors */
1f9ac57c 3957 hw->mac.ops.disable_rx(hw);
477de6ed
AD		 3958
3959 ixgbe_setup_psrtype(adapter);
7367096a 3960 ixgbe_setup_rdrxctl(adapter);
477de6ed 3961
6dcc28b9
JK		 3962 /* RSC Setup */
3963 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
3964 rfctl &= ~IXGBE_RFCTL_RSC_DIS;
3965 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
3966 rfctl |= IXGBE_RFCTL_RSC_DIS;
a21d0822
ET		 3967
3968 /* disable NFS filtering */
3969 rfctl |= (IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS);
6dcc28b9
JK		 3970 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
3971
9e10e045 3972 /* Program registers for the distribution of queues */
f5b4a52e 3973 ixgbe_setup_mrqc(adapter);
f5b4a52e 3974
477de6ed
AD		 3975 /* set_rx_buffer_len must be called before ring initialization */
3976 ixgbe_set_rx_buffer_len(adapter);
3977
3978 /*
3979 * Setup the HW Rx Head and Tail Descriptor Pointers and
3980 * the Base and Length of the Rx Descriptor Ring
3981 */
9e10e045
AD		 3982 for (i = 0; i < adapter->num_rx_queues; i++)
3983 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
177db6ff 3984
1f9ac57c 3985 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
9e10e045
AD		 3986 /* disable drop enable for 82598 parts */
3987 if (hw->mac.type == ixgbe_mac_82598EB)
3988 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3989
3990 /* enable all receives */
3991 rxctrl |= IXGBE_RXCTRL_RXEN;
3992 hw->mac.ops.enable_rx_dma(hw, rxctrl);
9a799d71
AK		 3993}
3994
80d5c368
PM		 3995static int ixgbe_vlan_rx_add_vid(struct net_device *netdev,
3996 __be16 proto, u16 vid)
068c89b0
DS		 3997{
3998 struct ixgbe_adapter *adapter = netdev_priv(netdev);
3999 struct ixgbe_hw *hw = &adapter->hw;
4000
4001 /* add VID to filter table */
18be4fce
AD		 4002 if (!vid || !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4003 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true, !!vid);
4004
f62bbb5e 4005 set_bit(vid, adapter->active_vlans);
8e586137
JP		 4006
4007 return 0;
068c89b0
DS		 4008}
4009
e1d0a2af
AD		 4010static int ixgbe_find_vlvf_entry(struct ixgbe_hw *hw, u32 vlan)
4011{
4012 u32 vlvf;
4013 int idx;
4014
4015 /* short cut the special case */
4016 if (vlan == 0)
4017 return 0;
4018
4019 /* Search for the vlan id in the VLVF entries */
4020 for (idx = IXGBE_VLVF_ENTRIES; --idx;) {
4021 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(idx));
4022 if ((vlvf & VLAN_VID_MASK) == vlan)
4023 break;
4024 }
4025
4026 return idx;
4027}
4028
4029void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid)
4030{
4031 struct ixgbe_hw *hw = &adapter->hw;
4032 u32 bits, word;
4033 int idx;
4034
4035 idx = ixgbe_find_vlvf_entry(hw, vid);
4036 if (!idx)
4037 return;
4038
4039 /* See if any other pools are set for this VLAN filter
4040 * entry other than the PF.
4041 */
4042 word = idx * 2 + (VMDQ_P(0) / 32);
b4f47a48 4043 bits = ~BIT(VMDQ_P(0) % 32);
e1d0a2af
AD		 4044 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4045
4046 /* Disable the filter so this falls into the default pool. */
4047 if (!bits && !IXGBE_READ_REG(hw, IXGBE_VLVFB(word ^ 1))) {
4048 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4049 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), 0);
4050 IXGBE_WRITE_REG(hw, IXGBE_VLVF(idx), 0);
4051 }
4052}
4053
80d5c368
PM		 4054static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev,
4055 __be16 proto, u16 vid)
068c89b0
DS		 4056{
4057 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4058 struct ixgbe_hw *hw = &adapter->hw;
4059
068c89b0 4060 /* remove VID from filter table */
18be4fce 4061 if (vid && !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
e1d0a2af
AD		 4062 hw->mac.ops.set_vfta(hw, vid, VMDQ_P(0), false, true);
4063
f62bbb5e 4064 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 4065
4066 return 0;
068c89b0
DS		 4067}
4068
f62bbb5e
JG		 4069/**
4070 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
4071 * @adapter: driver data
4072 */
4073static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
4074{
4075 struct ixgbe_hw *hw = &adapter->hw;
4076 u32 vlnctrl;
5f6c0181
JB		 4077 int i, j;
4078
4079 switch (hw->mac.type) {
4080 case ixgbe_mac_82598EB:
f62bbb5e
JG		 4081 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4082 vlnctrl &= ~IXGBE_VLNCTRL_VME;
5f6c0181
JB		 4083 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4084 break;
4085 case ixgbe_mac_82599EB:
b93a2226 4086 case ixgbe_mac_X540:
9a75a1ac
DS		 4087 case ixgbe_mac_X550:
4088 case ixgbe_mac_X550EM_x:
49425dfc 4089 case ixgbe_mac_x550em_a:
5f6c0181 4090 for (i = 0; i < adapter->num_rx_queues; i++) {
2a47fa45
JF		 4091 struct ixgbe_ring *ring = adapter->rx_ring[i];
4092
4093 if (ring->l2_accel_priv)
4094 continue;
4095 j = ring->reg_idx;
5f6c0181
JB		 4096 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4097 vlnctrl &= ~IXGBE_RXDCTL_VME;
4098 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4099 }
4100 break;
4101 default:
4102 break;
4103 }
4104}
4105
4106/**
f62bbb5e 4107 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
5f6c0181
JB		 4108 * @adapter: driver data
4109 */
f62bbb5e 4110static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
5f6c0181
JB		 4111{
4112 struct ixgbe_hw *hw = &adapter->hw;
f62bbb5e 4113 u32 vlnctrl;
5f6c0181
JB		 4114 int i, j;
4115
4116 switch (hw->mac.type) {
4117 case ixgbe_mac_82598EB:
f62bbb5e
JG		 4118 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4119 vlnctrl |= IXGBE_VLNCTRL_VME;
5f6c0181
JB		 4120 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4121 break;
4122 case ixgbe_mac_82599EB:
b93a2226 4123 case ixgbe_mac_X540:
9a75a1ac
DS		 4124 case ixgbe_mac_X550:
4125 case ixgbe_mac_X550EM_x:
49425dfc 4126 case ixgbe_mac_x550em_a:
5f6c0181 4127 for (i = 0; i < adapter->num_rx_queues; i++) {
2a47fa45
JF		 4128 struct ixgbe_ring *ring = adapter->rx_ring[i];
4129
4130 if (ring->l2_accel_priv)
4131 continue;
4132 j = ring->reg_idx;
5f6c0181
JB		 4133 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4134 vlnctrl |= IXGBE_RXDCTL_VME;
4135 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4136 }
4137 break;
4138 default:
4139 break;
4140 }
4141}
4142
16369564
AD		 4143static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter)
4144{
4145 struct ixgbe_hw *hw = &adapter->hw;
4146 u32 vlnctrl, i;
4147
f60439bc
AD		 4148 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4149
691e4121
ET		 4150 if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) {
4151 /* For VMDq and SR-IOV we must leave VLAN filtering enabled */
4152 vlnctrl |= IXGBE_VLNCTRL_VFE;
4153 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4154 } else {
f60439bc 4155 vlnctrl &= ~IXGBE_VLNCTRL_VFE;
16369564
AD		 4156 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4157 return;
4158 }
4159
691e4121
ET		 4160 /* Nothing to do for 82598 */
4161 if (hw->mac.type == ixgbe_mac_82598EB)
4162 return;
4163
16369564
AD		 4164 /* We are already in VLAN promisc, nothing to do */
4165 if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)
4166 return;
4167
4168 /* Set flag so we don't redo unnecessary work */
4169 adapter->flags2 |= IXGBE_FLAG2_VLAN_PROMISC;
4170
4171 /* Add PF to all active pools */
4172 for (i = IXGBE_VLVF_ENTRIES; --i;) {
4173 u32 reg_offset = IXGBE_VLVFB(i * 2 + VMDQ_P(0) / 32);
4174 u32 vlvfb = IXGBE_READ_REG(hw, reg_offset);
4175
b4f47a48 4176 vlvfb |= BIT(VMDQ_P(0) % 32);
16369564
AD		 4177 IXGBE_WRITE_REG(hw, reg_offset, vlvfb);
4178 }
4179
4180 /* Set all bits in the VLAN filter table array */
4181 for (i = hw->mac.vft_size; i--;)
4182 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ~0U);
4183}
4184
4185#define VFTA_BLOCK_SIZE 8
4186static void ixgbe_scrub_vfta(struct ixgbe_adapter *adapter, u32 vfta_offset)
4187{
4188 struct ixgbe_hw *hw = &adapter->hw;
4189 u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
4190 u32 vid_start = vfta_offset * 32;
4191 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
4192 u32 i, vid, word, bits;
4193
4194 for (i = IXGBE_VLVF_ENTRIES; --i;) {
4195 u32 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(i));
4196
4197 /* pull VLAN ID from VLVF */
4198 vid = vlvf & VLAN_VID_MASK;
4199
4200 /* only concern outselves with a certain range */
4201 if (vid < vid_start || vid >= vid_end)
4202 continue;
4203
4204 if (vlvf) {
4205 /* record VLAN ID in VFTA */
b4f47a48 4206 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
16369564
AD		 4207
4208 /* if PF is part of this then continue */
4209 if (test_bit(vid, adapter->active_vlans))
4210 continue;
4211 }
4212
4213 /* remove PF from the pool */
4214 word = i * 2 + VMDQ_P(0) / 32;
b4f47a48 4215 bits = ~BIT(VMDQ_P(0) % 32);
16369564
AD		 4216 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4217 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), bits);
4218 }
4219
4220 /* extract values from active_vlans and write back to VFTA */
4221 for (i = VFTA_BLOCK_SIZE; i--;) {
4222 vid = (vfta_offset + i) * 32;
4223 word = vid / BITS_PER_LONG;
4224 bits = vid % BITS_PER_LONG;
4225
4226 vfta[i] |= adapter->active_vlans[word] >> bits;
4227
4228 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vfta_offset + i), vfta[i]);
4229 }
4230}
4231
4232static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter)
4233{
4234 struct ixgbe_hw *hw = &adapter->hw;
4235 u32 vlnctrl, i;
4236
f60439bc
AD		 4237 /* Set VLAN filtering to enabled */
4238 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4239 vlnctrl |= IXGBE_VLNCTRL_VFE;
4240 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4241
691e4121
ET		 4242 if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) ||
4243 hw->mac.type == ixgbe_mac_82598EB)
16369564 4244 return;
16369564
AD		 4245
4246 /* We are not in VLAN promisc, nothing to do */
4247 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4248 return;
4249
4250 /* Set flag so we don't redo unnecessary work */
4251 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
4252
4253 for (i = 0; i < hw->mac.vft_size; i += VFTA_BLOCK_SIZE)
4254 ixgbe_scrub_vfta(adapter, i);
4255}
4256
9a799d71
AK		 4257static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
4258{
06bb1c39 4259 u16 vid = 1;
9a799d71 4260
80d5c368 4261 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
f62bbb5e 4262
06bb1c39 4263 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
80d5c368 4264 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
9a799d71
AK		 4265}
4266
b335e75b
JK		 4267/**
4268 * ixgbe_write_mc_addr_list - write multicast addresses to MTA
4269 * @netdev: network interface device structure
4270 *
4271 * Writes multicast address list to the MTA hash table.
4272 * Returns: -ENOMEM on failure
4273 * 0 on no addresses written
4274 * X on writing X addresses to MTA
4275 **/
4276static int ixgbe_write_mc_addr_list(struct net_device *netdev)
4277{
4278 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4279 struct ixgbe_hw *hw = &adapter->hw;
4280
4281 if (!netif_running(netdev))
4282 return 0;
4283
4284 if (hw->mac.ops.update_mc_addr_list)
4285 hw->mac.ops.update_mc_addr_list(hw, netdev);
4286 else
4287 return -ENOMEM;
4288
4289#ifdef CONFIG_PCI_IOV
5d7daa35 4290 ixgbe_restore_vf_multicasts(adapter);
b335e75b
JK		 4291#endif
4292
4293 return netdev_mc_count(netdev);
4294}
4295
5d7daa35
JK		 4296#ifdef CONFIG_PCI_IOV
4297void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter)
4298{
c9f53e63 4299 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35
JK		 4300 struct ixgbe_hw *hw = &adapter->hw;
4301 int i;
c9f53e63
AD		 4302
4303 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4304 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
4305
4306 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
4307 hw->mac.ops.set_rar(hw, i,
4308 mac_table->addr,
4309 mac_table->pool,
5d7daa35
JK		 4310 IXGBE_RAH_AV);
4311 else
4312 hw->mac.ops.clear_rar(hw, i);
5d7daa35
JK		 4313 }
4314}
5d7daa35 4315
c9f53e63 4316#endif
5d7daa35
JK		 4317static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter)
4318{
c9f53e63 4319 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35
JK		 4320 struct ixgbe_hw *hw = &adapter->hw;
4321 int i;
5d7daa35 4322
c9f53e63
AD		 4323 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4324 if (!(mac_table->state & IXGBE_MAC_STATE_MODIFIED))
4325 continue;
4326
4327 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
4328
4329 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
4330 hw->mac.ops.set_rar(hw, i,
4331 mac_table->addr,
4332 mac_table->pool,
4333 IXGBE_RAH_AV);
4334 else
4335 hw->mac.ops.clear_rar(hw, i);
5d7daa35
JK		 4336 }
4337}
4338
4339static void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter)
4340{
c9f53e63 4341 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35 4342 struct ixgbe_hw *hw = &adapter->hw;
c9f53e63 4343 int i;
5d7daa35 4344
c9f53e63
AD		 4345 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4346 mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
4347 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
5d7daa35 4348 }
c9f53e63 4349
5d7daa35
JK		 4350 ixgbe_sync_mac_table(adapter);
4351}
4352
c9f53e63 4353static int ixgbe_available_rars(struct ixgbe_adapter *adapter, u16 pool)
5d7daa35 4354{
c9f53e63 4355 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35
JK		 4356 struct ixgbe_hw *hw = &adapter->hw;
4357 int i, count = 0;
4358
c9f53e63
AD		 4359 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4360 /* do not count default RAR as available */
4361 if (mac_table->state & IXGBE_MAC_STATE_DEFAULT)
4362 continue;
4363
4364 /* only count unused and addresses that belong to us */
4365 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) {
4366 if (mac_table->pool != pool)
4367 continue;
4368 }
4369
4370 count++;
5d7daa35 4371 }
c9f53e63 4372
5d7daa35
JK		 4373 return count;
4374}
4375
4376/* this function destroys the first RAR entry */
c9f53e63 4377static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter)
5d7daa35 4378{
c9f53e63 4379 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35
JK		 4380 struct ixgbe_hw *hw = &adapter->hw;
4381
c9f53e63
AD		 4382 memcpy(&mac_table->addr, hw->mac.addr, ETH_ALEN);
4383 mac_table->pool = VMDQ_P(0);
4384
4385 mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE;
4386
4387 hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool,
5d7daa35
JK		 4388 IXGBE_RAH_AV);
4389}
4390
c9f53e63
AD		 4391int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter,
4392 const u8 *addr, u16 pool)
5d7daa35 4393{
c9f53e63 4394 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35
JK		 4395 struct ixgbe_hw *hw = &adapter->hw;
4396 int i;
4397
4398 if (is_zero_ether_addr(addr))
4399 return -EINVAL;
4400
c9f53e63
AD		 4401 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4402 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
5d7daa35 4403 continue;
c9f53e63
AD		 4404
4405 ether_addr_copy(mac_table->addr, addr);
4406 mac_table->pool = pool;
4407
4408 mac_table->state |= IXGBE_MAC_STATE_MODIFIED |
4409 IXGBE_MAC_STATE_IN_USE;
4410
5d7daa35 4411 ixgbe_sync_mac_table(adapter);
c9f53e63 4412
5d7daa35
JK		 4413 return i;
4414 }
c9f53e63 4415
5d7daa35
JK		 4416 return -ENOMEM;
4417}
4418
c9f53e63
AD		 4419int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,
4420 const u8 *addr, u16 pool)
5d7daa35 4421{
c9f53e63 4422 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5d7daa35 4423 struct ixgbe_hw *hw = &adapter->hw;
c9f53e63 4424 int i;
5d7daa35
JK		 4425
4426 if (is_zero_ether_addr(addr))
4427 return -EINVAL;
4428
c9f53e63
AD		 4429 /* search table for addr, if found clear IN_USE flag and sync */
4430 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4431 /* we can only delete an entry if it is in use */
4432 if (!(mac_table->state & IXGBE_MAC_STATE_IN_USE))
4433 continue;
4434 /* we only care about entries that belong to the given pool */
4435 if (mac_table->pool != pool)
4436 continue;
4437 /* we only care about a specific MAC address */
4438 if (!ether_addr_equal(addr, mac_table->addr))
4439 continue;
4440
4441 mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
4442 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
4443
4444 ixgbe_sync_mac_table(adapter);
4445
4446 return 0;
5d7daa35 4447 }
c9f53e63 4448
5d7daa35
JK		 4449 return -ENOMEM;
4450}
2850062a
AD		 4451/**
4452 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
4453 * @netdev: network interface device structure
4454 *
4455 * Writes unicast address list to the RAR table.
4456 * Returns: -ENOMEM on failure/insufficient address space
4457 * 0 on no addresses written
4458 * X on writing X addresses to the RAR table
4459 **/
5d7daa35 4460static int ixgbe_write_uc_addr_list(struct net_device *netdev, int vfn)
2850062a
AD		 4461{
4462 struct ixgbe_adapter *adapter = netdev_priv(netdev);
2850062a
AD		 4463 int count = 0;
4464
4465 /* return ENOMEM indicating insufficient memory for addresses */
c9f53e63 4466 if (netdev_uc_count(netdev) > ixgbe_available_rars(adapter, vfn))
2850062a
AD		 4467 return -ENOMEM;
4468
95447461 4469 if (!netdev_uc_empty(netdev)) {
2850062a 4470 struct netdev_hw_addr *ha;
2850062a 4471 netdev_for_each_uc_addr(ha, netdev) {
5d7daa35
JK		 4472 ixgbe_del_mac_filter(adapter, ha->addr, vfn);
4473 ixgbe_add_mac_filter(adapter, ha->addr, vfn);
2850062a
AD		 4474 count++;
4475 }
4476 }
2850062a
AD		 4477 return count;
4478}
4479
0f079d22
AD		 4480static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr)
4481{
4482 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4483 int ret;
4484
4485 ret = ixgbe_add_mac_filter(adapter, addr, VMDQ_P(0));
4486
4487 return min_t(int, ret, 0);
4488}
4489
4490static int ixgbe_uc_unsync(struct net_device *netdev, const unsigned char *addr)
4491{
4492 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4493
4494 ixgbe_del_mac_filter(adapter, addr, VMDQ_P(0));
4495
4496 return 0;
4497}
4498
9a799d71 4499/**
2c5645cf 4500 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
9a799d71
AK		 4501 * @netdev: network interface device structure
4502 *
2c5645cf
CL		 4503 * The set_rx_method entry point is called whenever the unicast/multicast
4504 * address list or the network interface flags are updated. This routine is
4505 * responsible for configuring the hardware for proper unicast, multicast and
4506 * promiscuous mode.
9a799d71 4507 **/
7f870475 4508void ixgbe_set_rx_mode(struct net_device *netdev)
9a799d71
AK		 4509{
4510 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4511 struct ixgbe_hw *hw = &adapter->hw;
2850062a 4512 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
0c5a6166 4513 netdev_features_t features = netdev->features;
2850062a 4514 int count;
9a799d71
AK		 4515
4516 /* Check for Promiscuous and All Multicast modes */
9a799d71
AK		 4517 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4518
f5dc442b 4519 /* set all bits that we expect to always be set */
3f2d1c0f 4520 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
f5dc442b
AD		 4521 fctrl |= IXGBE_FCTRL_BAM;
4522 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
4523 fctrl |= IXGBE_FCTRL_PMCF;
4524
2850062a
AD		 4525 /* clear the bits we are changing the status of */
4526 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
9a799d71 4527 if (netdev->flags & IFF_PROMISC) {
e433ea1f 4528 hw->addr_ctrl.user_set_promisc = true;
9a799d71 4529 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
b335e75b 4530 vmolr |= IXGBE_VMOLR_MPE;
0c5a6166 4531 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
9a799d71 4532 } else {
746b9f02
PM		 4533 if (netdev->flags & IFF_ALLMULTI) {
4534 fctrl |= IXGBE_FCTRL_MPE;
2850062a 4535 vmolr |= IXGBE_VMOLR_MPE;
746b9f02 4536 }
e433ea1f 4537 hw->addr_ctrl.user_set_promisc = false;
9dcb373c
JF		 4538 }
4539
4540 /*
4541 * Write addresses to available RAR registers, if there is not
4542 * sufficient space to store all the addresses then enable
4543 * unicast promiscuous mode
4544 */
0f079d22 4545 if (__dev_uc_sync(netdev, ixgbe_uc_sync, ixgbe_uc_unsync)) {
9dcb373c
JF		 4546 fctrl |= IXGBE_FCTRL_UPE;
4547 vmolr |= IXGBE_VMOLR_ROPE;
9a799d71
AK		 4548 }
4549
cf78959c
ET		 4550 /* Write addresses to the MTA, if the attempt fails
4551 * then we should just turn on promiscuous mode so
4552 * that we can at least receive multicast traffic
4553 */
b335e75b
JK		 4554 count = ixgbe_write_mc_addr_list(netdev);
4555 if (count < 0) {
4556 fctrl |= IXGBE_FCTRL_MPE;
4557 vmolr |= IXGBE_VMOLR_MPE;
4558 } else if (count) {
4559 vmolr |= IXGBE_VMOLR_ROMPE;
4560 }
1d9c0bfd
AD		 4561
4562 if (hw->mac.type != ixgbe_mac_82598EB) {
4563 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
2850062a
AD		 4564 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
4565 IXGBE_VMOLR_ROPE);
1d9c0bfd 4566 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
2850062a
AD		 4567 }
4568
3f2d1c0f 4569 /* This is useful for sniffing bad packets. */
0c5a6166 4570 if (features & NETIF_F_RXALL) {
3f2d1c0f
BG		 4571 /* UPE and MPE will be handled by normal PROMISC logic
4572 * in e1000e_set_rx_mode */
4573 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
4574 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
4575 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */
4576
4577 fctrl &= ~(IXGBE_FCTRL_DPF);
4578 /* NOTE: VLAN filtering is disabled by setting PROMISC */
4579 }
4580
2850062a 4581 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
f62bbb5e 4582
0c5a6166 4583 if (features & NETIF_F_HW_VLAN_CTAG_RX)
f62bbb5e
JG		 4584 ixgbe_vlan_strip_enable(adapter);
4585 else
4586 ixgbe_vlan_strip_disable(adapter);
0c5a6166
AD		 4587
4588 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
4589 ixgbe_vlan_promisc_disable(adapter);
4590 else
4591 ixgbe_vlan_promisc_enable(adapter);
9a799d71
AK		 4592}
4593
021230d4
AV		 4594static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
4595{
4596 int q_idx;
021230d4 4597
3ffc1af5 4598 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
49c7ffbe 4599 napi_enable(&adapter->q_vector[q_idx]->napi);
021230d4
AV		 4600}
4601
4602static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
4603{
4604 int q_idx;
021230d4 4605
3ffc1af5 4606 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
49c7ffbe 4607 napi_disable(&adapter->q_vector[q_idx]->napi);
021230d4
AV		 4608}
4609
a21d0822 4610static void ixgbe_clear_udp_tunnel_port(struct ixgbe_adapter *adapter, u32 mask)
67359c3c 4611{
a21d0822
ET		 4612 struct ixgbe_hw *hw = &adapter->hw;
4613 u32 vxlanctrl;
4614
4615 if (!(adapter->flags & (IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE |
4616 IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)))
4617 return;
4618
4619 vxlanctrl = IXGBE_READ_REG(hw, IXGBE_VXLANCTRL) && ~mask;
4620 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, vxlanctrl);
4621
4622 if (mask & IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK)
67359c3c 4623 adapter->vxlan_port = 0;
a21d0822
ET		 4624
4625 if (mask & IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK)
4626 adapter->geneve_port = 0;
67359c3c
MR		 4627}
4628
7a6b6f51 4629#ifdef CONFIG_IXGBE_DCB
49ce9c2c 4630/**
2f90b865
AD		 4631 * ixgbe_configure_dcb - Configure DCB hardware
4632 * @adapter: ixgbe adapter struct
4633 *
4634 * This is called by the driver on open to configure the DCB hardware.
4635 * This is also called by the gennetlink interface when reconfiguring
4636 * the DCB state.
4637 */
4638static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
4639{
4640 struct ixgbe_hw *hw = &adapter->hw;
9806307a 4641 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2f90b865 4642
67ebd791
AD		 4643 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
4644 if (hw->mac.type == ixgbe_mac_82598EB)
4645 netif_set_gso_max_size(adapter->netdev, 65536);
4646 return;
4647 }
4648
4649 if (hw->mac.type == ixgbe_mac_82598EB)
4650 netif_set_gso_max_size(adapter->netdev, 32768);
4651
971060b1 4652#ifdef IXGBE_FCOE
b120818e
JF		 4653 if (adapter->netdev->features & NETIF_F_FCOE_MTU)
4654 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
c27931da 4655#endif
b120818e
JF		 4656
4657 /* reconfigure the hardware */
4658 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
c27931da
JF		 4659 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
4660 DCB_TX_CONFIG);
4661 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
4662 DCB_RX_CONFIG);
4663 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
b120818e
JF		 4664 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
4665 ixgbe_dcb_hw_ets(&adapter->hw,
4666 adapter->ixgbe_ieee_ets,
4667 max_frame);
4668 ixgbe_dcb_hw_pfc_config(&adapter->hw,
4669 adapter->ixgbe_ieee_pfc->pfc_en,
4670 adapter->ixgbe_ieee_ets->prio_tc);
c27931da 4671 }
8187cd48
JF		 4672
4673 /* Enable RSS Hash per TC */
4674 if (hw->mac.type != ixgbe_mac_82598EB) {
4ae63730
AD		 4675 u32 msb = 0;
4676 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;
8187cd48 4677
d411a936
AD		 4678 while (rss_i) {
4679 msb++;
4680 rss_i >>= 1;
4681 }
8187cd48 4682
4ae63730
AD		 4683 /* write msb to all 8 TCs in one write */
4684 IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
8187cd48 4685 }
2f90b865 4686}
9da712d2
JF		 4687#endif
4688
4689/* Additional bittime to account for IXGBE framing */
4690#define IXGBE_ETH_FRAMING 20
4691
49ce9c2c 4692/**
9da712d2
JF		 4693 * ixgbe_hpbthresh - calculate high water mark for flow control
4694 *
4695 * @adapter: board private structure to calculate for
49ce9c2c 4696 * @pb: packet buffer to calculate
9da712d2
JF		 4697 */
4698static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
4699{
4700 struct ixgbe_hw *hw = &adapter->hw;
4701 struct net_device *dev = adapter->netdev;
4702 int link, tc, kb, marker;
4703 u32 dv_id, rx_pba;
4704
4705 /* Calculate max LAN frame size */
4706 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
4707
4708#ifdef IXGBE_FCOE
4709 /* FCoE traffic class uses FCOE jumbo frames */
800bd607
AD		 4710 if ((dev->features & NETIF_F_FCOE_MTU) &&
4711 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
4712 (pb == ixgbe_fcoe_get_tc(adapter)))
4713 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
9da712d2 4714#endif
e5776620 4715
9da712d2
JF		 4716 /* Calculate delay value for device */
4717 switch (hw->mac.type) {
4718 case ixgbe_mac_X540:
9a75a1ac
DS		 4719 case ixgbe_mac_X550:
4720 case ixgbe_mac_X550EM_x:
49425dfc 4721 case ixgbe_mac_x550em_a:
9da712d2
JF		 4722 dv_id = IXGBE_DV_X540(link, tc);
4723 break;
4724 default:
4725 dv_id = IXGBE_DV(link, tc);
4726 break;
4727 }
4728
4729 /* Loopback switch introduces additional latency */
4730 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4731 dv_id += IXGBE_B2BT(tc);
4732
4733 /* Delay value is calculated in bit times convert to KB */
4734 kb = IXGBE_BT2KB(dv_id);
4735 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
4736
4737 marker = rx_pba - kb;
4738
4739 /* It is possible that the packet buffer is not large enough
4740 * to provide required headroom. In this case throw an error
4741 * to user and a do the best we can.
4742 */
4743 if (marker < 0) {
4744 e_warn(drv, "Packet Buffer(%i) can not provide enough"
4745 "headroom to support flow control."
4746 "Decrease MTU or number of traffic classes\n", pb);
4747 marker = tc + 1;
4748 }
4749
4750 return marker;
4751}
4752
49ce9c2c 4753/**
9da712d2
JF		 4754 * ixgbe_lpbthresh - calculate low water mark for for flow control
4755 *
4756 * @adapter: board private structure to calculate for
49ce9c2c 4757 * @pb: packet buffer to calculate
9da712d2 4758 */
e5776620 4759static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb)
9da712d2
JF		 4760{
4761 struct ixgbe_hw *hw = &adapter->hw;
4762 struct net_device *dev = adapter->netdev;
4763 int tc;
4764 u32 dv_id;
4765
4766 /* Calculate max LAN frame size */
4767 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
4768
e5776620
JK		 4769#ifdef IXGBE_FCOE
4770 /* FCoE traffic class uses FCOE jumbo frames */
4771 if ((dev->features & NETIF_F_FCOE_MTU) &&
4772 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
4773 (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up)))
4774 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
4775#endif
4776
9da712d2
JF		 4777 /* Calculate delay value for device */
4778 switch (hw->mac.type) {
4779 case ixgbe_mac_X540:
9a75a1ac
DS		 4780 case ixgbe_mac_X550:
4781 case ixgbe_mac_X550EM_x:
49425dfc 4782 case ixgbe_mac_x550em_a:
9da712d2
JF		 4783 dv_id = IXGBE_LOW_DV_X540(tc);
4784 break;
4785 default:
4786 dv_id = IXGBE_LOW_DV(tc);
4787 break;
4788 }
4789
4790 /* Delay value is calculated in bit times convert to KB */
4791 return IXGBE_BT2KB(dv_id);
4792}
4793
4794/*
4795 * ixgbe_pbthresh_setup - calculate and setup high low water marks
4796 */
4797static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
4798{
4799 struct ixgbe_hw *hw = &adapter->hw;
4800 int num_tc = netdev_get_num_tc(adapter->netdev);
4801 int i;
4802
4803 if (!num_tc)
4804 num_tc = 1;
4805
9da712d2
JF		 4806 for (i = 0; i < num_tc; i++) {
4807 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);
e5776620 4808 hw->fc.low_water[i] = ixgbe_lpbthresh(adapter, i);
9da712d2
JF		 4809
4810 /* Low water marks must not be larger than high water marks */
e5776620
JK		 4811 if (hw->fc.low_water[i] > hw->fc.high_water[i])
4812 hw->fc.low_water[i] = 0;
9da712d2 4813 }
e5776620
JK		 4814
4815 for (; i < MAX_TRAFFIC_CLASS; i++)
4816 hw->fc.high_water[i] = 0;
9da712d2
JF		 4817}
4818
80605c65
JF		 4819static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
4820{
80605c65 4821 struct ixgbe_hw *hw = &adapter->hw;
f7e1027f
AD		 4822 int hdrm;
4823 u8 tc = netdev_get_num_tc(adapter->netdev);
80605c65
JF		 4824
4825 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
4826 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
f7e1027f
AD		 4827 hdrm = 32 << adapter->fdir_pballoc;
4828 else
4829 hdrm = 0;
80605c65 4830
f7e1027f 4831 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
9da712d2 4832 ixgbe_pbthresh_setup(adapter);
80605c65
JF		 4833}
4834
e4911d57
AD		 4835static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
4836{
4837 struct ixgbe_hw *hw = &adapter->hw;
b67bfe0d 4838 struct hlist_node *node2;
e4911d57
AD		 4839 struct ixgbe_fdir_filter *filter;
4840
4841 spin_lock(&adapter->fdir_perfect_lock);
4842
4843 if (!hlist_empty(&adapter->fdir_filter_list))
4844 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);
4845
b67bfe0d 4846 hlist_for_each_entry_safe(filter, node2,
e4911d57
AD		 4847 &adapter->fdir_filter_list, fdir_node) {
4848 ixgbe_fdir_write_perfect_filter_82599(hw,
1f4d5183
AD		 4849 &filter->filter,
4850 filter->sw_idx,
4851 (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
4852 IXGBE_FDIR_DROP_QUEUE :
4853 adapter->rx_ring[filter->action]->reg_idx);
e4911d57
AD		 4854 }
4855
4856 spin_unlock(&adapter->fdir_perfect_lock);
4857}
4858
2a47fa45
JF		 4859static void ixgbe_macvlan_set_rx_mode(struct net_device *dev, unsigned int pool,
4860 struct ixgbe_adapter *adapter)
4861{
4862 struct ixgbe_hw *hw = &adapter->hw;
4863 u32 vmolr;
4864
4865 /* No unicast promiscuous support for VMDQ devices. */
4866 vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
4867 vmolr |= (IXGBE_VMOLR_ROMPE | IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE);
4868
4869 /* clear the affected bit */
4870 vmolr &= ~IXGBE_VMOLR_MPE;
4871
4872 if (dev->flags & IFF_ALLMULTI) {
4873 vmolr |= IXGBE_VMOLR_MPE;
4874 } else {
4875 vmolr |= IXGBE_VMOLR_ROMPE;
4876 hw->mac.ops.update_mc_addr_list(hw, dev);
4877 }
5d7daa35 4878 ixgbe_write_uc_addr_list(adapter->netdev, pool);
2a47fa45
JF		 4879 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
4880}
4881
2a47fa45
JF		 4882static void ixgbe_fwd_psrtype(struct ixgbe_fwd_adapter *vadapter)
4883{
4884 struct ixgbe_adapter *adapter = vadapter->real_adapter;
219354d4 4885 int rss_i = adapter->num_rx_queues_per_pool;
2a47fa45
JF		 4886 struct ixgbe_hw *hw = &adapter->hw;
4887 u16 pool = vadapter->pool;
4888 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4889 IXGBE_PSRTYPE_UDPHDR |
4890 IXGBE_PSRTYPE_IPV4HDR |
4891 IXGBE_PSRTYPE_L2HDR |
4892 IXGBE_PSRTYPE_IPV6HDR;
4893
4894 if (hw->mac.type == ixgbe_mac_82598EB)
4895 return;
4896
4897 if (rss_i > 3)
b4f47a48 4898 psrtype |= 2u << 29;
2a47fa45 4899 else if (rss_i > 1)
b4f47a48 4900 psrtype |= 1u << 29;
2a47fa45
JF		 4901
4902 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
4903}
4904
4905/**
4906 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
4907 * @rx_ring: ring to free buffers from
4908 **/
4909static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
4910{
4911 struct device *dev = rx_ring->dev;
4912 unsigned long size;
4913 u16 i;
4914
4915 /* ring already cleared, nothing to do */
4916 if (!rx_ring->rx_buffer_info)
4917 return;
4918
4919 /* Free all the Rx ring sk_buffs */
4920 for (i = 0; i < rx_ring->count; i++) {
18cb652a 4921 struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];
2a47fa45 4922
2a47fa45
JF		 4923 if (rx_buffer->skb) {
4924 struct sk_buff *skb = rx_buffer->skb;
18cb652a 4925 if (IXGBE_CB(skb)->page_released)
f3213d93
AD		 4926 dma_unmap_page_attrs(dev,
4927 IXGBE_CB(skb)->dma,
4928 ixgbe_rx_pg_size(rx_ring),
4929 DMA_FROM_DEVICE,
4930 IXGBE_RX_DMA_ATTR);
2a47fa45 4931 dev_kfree_skb(skb);
4d2fcfbc 4932 rx_buffer->skb = NULL;
2a47fa45 4933 }
18cb652a
AD		 4934
4935 if (!rx_buffer->page)
4936 continue;
4937
f3213d93
AD		 4938 /* Invalidate cache lines that may have been written to by
4939 * device so that we avoid corrupting memory.
4940 */
4941 dma_sync_single_range_for_cpu(rx_ring->dev,
4942 rx_buffer->dma,
4943 rx_buffer->page_offset,
4944 ixgbe_rx_bufsz(rx_ring),
4945 DMA_FROM_DEVICE);
4946
4947 /* free resources associated with mapping */
4948 dma_unmap_page_attrs(dev, rx_buffer->dma,
4949 ixgbe_rx_pg_size(rx_ring),
4950 DMA_FROM_DEVICE,
4951 IXGBE_RX_DMA_ATTR);
1b56cf49
AD		 4952 __page_frag_cache_drain(rx_buffer->page,
4953 rx_buffer->pagecnt_bias);
18cb652a 4954
2a47fa45
JF		 4955 rx_buffer->page = NULL;
4956 }
4957
4958 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4959 memset(rx_ring->rx_buffer_info, 0, size);
4960
2a47fa45
JF		 4961 rx_ring->next_to_alloc = 0;
4962 rx_ring->next_to_clean = 0;
4963 rx_ring->next_to_use = 0;
4964}
4965
4966static void ixgbe_disable_fwd_ring(struct ixgbe_fwd_adapter *vadapter,
4967 struct ixgbe_ring *rx_ring)
4968{
4969 struct ixgbe_adapter *adapter = vadapter->real_adapter;
4970 int index = rx_ring->queue_index + vadapter->rx_base_queue;
4971
4972 /* shutdown specific queue receive and wait for dma to settle */
4973 ixgbe_disable_rx_queue(adapter, rx_ring);
4974 usleep_range(10000, 20000);
b4f47a48 4975 ixgbe_irq_disable_queues(adapter, BIT_ULL(index));
2a47fa45
JF		 4976 ixgbe_clean_rx_ring(rx_ring);
4977 rx_ring->l2_accel_priv = NULL;
4978}
4979
ae72c8d0
JF		 4980static int ixgbe_fwd_ring_down(struct net_device *vdev,
4981 struct ixgbe_fwd_adapter *accel)
2a47fa45
JF		 4982{
4983 struct ixgbe_adapter *adapter = accel->real_adapter;
4984 unsigned int rxbase = accel->rx_base_queue;
4985 unsigned int txbase = accel->tx_base_queue;
4986 int i;
4987
4988 netif_tx_stop_all_queues(vdev);
4989
4990 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
4991 ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]);
4992 adapter->rx_ring[rxbase + i]->netdev = adapter->netdev;
4993 }
4994
4995 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
4996 adapter->tx_ring[txbase + i]->l2_accel_priv = NULL;
4997 adapter->tx_ring[txbase + i]->netdev = adapter->netdev;
4998 }
4999
5000
5001 return 0;
5002}
5003
5004static int ixgbe_fwd_ring_up(struct net_device *vdev,
5005 struct ixgbe_fwd_adapter *accel)
5006{
5007 struct ixgbe_adapter *adapter = accel->real_adapter;
5008 unsigned int rxbase, txbase, queues;
5009 int i, baseq, err = 0;
5010
5011 if (!test_bit(accel->pool, &adapter->fwd_bitmask))
5012 return 0;
5013
5014 baseq = accel->pool * adapter->num_rx_queues_per_pool;
5015 netdev_dbg(vdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n",
5016 accel->pool, adapter->num_rx_pools,
5017 baseq, baseq + adapter->num_rx_queues_per_pool,
5018 adapter->fwd_bitmask);
5019
5020 accel->netdev = vdev;
5021 accel->rx_base_queue = rxbase = baseq;
5022 accel->tx_base_queue = txbase = baseq;
5023
5024 for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
5025 ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]);
5026
5027 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
5028 adapter->rx_ring[rxbase + i]->netdev = vdev;
5029 adapter->rx_ring[rxbase + i]->l2_accel_priv = accel;
5030 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[rxbase + i]);
5031 }
5032
5033 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
5034 adapter->tx_ring[txbase + i]->netdev = vdev;
5035 adapter->tx_ring[txbase + i]->l2_accel_priv = accel;
5036 }
5037
5038 queues = min_t(unsigned int,
5039 adapter->num_rx_queues_per_pool, vdev->num_tx_queues);
5040 err = netif_set_real_num_tx_queues(vdev, queues);
5041 if (err)
5042 goto fwd_queue_err;
5043
2a47fa45
JF		 5044 err = netif_set_real_num_rx_queues(vdev, queues);
5045 if (err)
5046 goto fwd_queue_err;
5047
5048 if (is_valid_ether_addr(vdev->dev_addr))
5049 ixgbe_add_mac_filter(adapter, vdev->dev_addr, accel->pool);
5050
5051 ixgbe_fwd_psrtype(accel);
5052 ixgbe_macvlan_set_rx_mode(vdev, accel->pool, adapter);
5053 return err;
5054fwd_queue_err:
5055 ixgbe_fwd_ring_down(vdev, accel);
5056 return err;
5057}
5058
1cd127fc 5059static int ixgbe_upper_dev_walk(struct net_device *upper, void *data)
2a47fa45 5060{
1cd127fc
DA		 5061 if (netif_is_macvlan(upper)) {
5062 struct macvlan_dev *dfwd = netdev_priv(upper);
5063 struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv;
2a47fa45 5064
1cd127fc
DA		 5065 if (dfwd->fwd_priv)
5066 ixgbe_fwd_ring_up(upper, vadapter);
2a47fa45 5067 }
1cd127fc
DA		 5068
5069 return 0;
5070}
5071
5072static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)
5073{
5074 netdev_walk_all_upper_dev_rcu(adapter->netdev,
5075 ixgbe_upper_dev_walk, NULL);
2a47fa45
JF		 5076}
5077
9a799d71
AK		 5078static void ixgbe_configure(struct ixgbe_adapter *adapter)
5079{
d2f5e7f3
AS		 5080 struct ixgbe_hw *hw = &adapter->hw;
5081
80605c65 5082 ixgbe_configure_pb(adapter);
7a6b6f51 5083#ifdef CONFIG_IXGBE_DCB
67ebd791 5084 ixgbe_configure_dcb(adapter);
2f90b865 5085#endif
b35d4d42
AD		 5086 /*
5087 * We must restore virtualization before VLANs or else
5088 * the VLVF registers will not be populated
5089 */
5090 ixgbe_configure_virtualization(adapter);
9a799d71 5091
4c1d7b4b 5092 ixgbe_set_rx_mode(adapter->netdev);
f62bbb5e
JG		 5093 ixgbe_restore_vlan(adapter);
5094
d2f5e7f3
AS		 5095 switch (hw->mac.type) {
5096 case ixgbe_mac_82599EB:
5097 case ixgbe_mac_X540:
5098 hw->mac.ops.disable_rx_buff(hw);
5099 break;
5100 default:
5101 break;
5102 }
5103
c4cf55e5 5104 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
4c1d7b4b
AD		 5105 ixgbe_init_fdir_signature_82599(&adapter->hw,
5106 adapter->fdir_pballoc);
e4911d57
AD		 5107 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
5108 ixgbe_init_fdir_perfect_82599(&adapter->hw,
5109 adapter->fdir_pballoc);
5110 ixgbe_fdir_filter_restore(adapter);
c4cf55e5 5111 }
4c1d7b4b 5112
d2f5e7f3
AS		 5113 switch (hw->mac.type) {
5114 case ixgbe_mac_82599EB:
5115 case ixgbe_mac_X540:
5116 hw->mac.ops.enable_rx_buff(hw);
5117 break;
5118 default:
5119 break;
5120 }
5121
9de7605e
MR		 5122#ifdef CONFIG_IXGBE_DCA
5123 /* configure DCA */
5124 if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE)
5125 ixgbe_setup_dca(adapter);
5126#endif /* CONFIG_IXGBE_DCA */
5127
7c8ae65a
AD		 5128#ifdef IXGBE_FCOE
5129 /* configure FCoE L2 filters, redirection table, and Rx control */
5130 ixgbe_configure_fcoe(adapter);
5131
5132#endif /* IXGBE_FCOE */
9a799d71
AK		 5133 ixgbe_configure_tx(adapter);
5134 ixgbe_configure_rx(adapter);
2a47fa45 5135 ixgbe_configure_dfwd(adapter);
9a799d71
AK		 5136}
5137
0ecc061d 5138/**
e8e26350
PW		 5139 * ixgbe_sfp_link_config - set up SFP+ link
5140 * @adapter: pointer to private adapter struct
5141 **/
5142static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
5143{
7086400d 5144 /*
52f33af8 5145 * We are assuming the worst case scenario here, and that
7086400d
AD		 5146 * is that an SFP was inserted/removed after the reset
5147 * but before SFP detection was enabled. As such the best
5148 * solution is to just start searching as soon as we start
5149 */
5150 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
5151 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
e8e26350 5152
7086400d 5153 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
58e7cd24 5154 adapter->sfp_poll_time = 0;
e8e26350
PW		 5155}
5156
5157/**
5158 * ixgbe_non_sfp_link_config - set up non-SFP+ link
0ecc061d
PWJ		 5159 * @hw: pointer to private hardware struct
5160 *
5161 * Returns 0 on success, negative on failure
5162 **/
e8e26350 5163static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
0ecc061d 5164{
3d292265
JH		 5165 u32 speed;
5166 bool autoneg, link_up = false;
a1e869de 5167 int ret = IXGBE_ERR_LINK_SETUP;
0ecc061d
PWJ		 5168
5169 if (hw->mac.ops.check_link)
3d292265 5170 ret = hw->mac.ops.check_link(hw, &speed, &link_up, false);
0ecc061d
PWJ		 5171
5172 if (ret)
e90dd264 5173 return ret;
0ecc061d 5174
3d292265
JH		 5175 speed = hw->phy.autoneg_advertised;
5176 if ((!speed) && (hw->mac.ops.get_link_capabilities))
5177 ret = hw->mac.ops.get_link_capabilities(hw, &speed,
5178 &autoneg);
0ecc061d 5179 if (ret)
e90dd264 5180 return ret;
0ecc061d 5181
8620a103 5182 if (hw->mac.ops.setup_link)
fd0326f2 5183 ret = hw->mac.ops.setup_link(hw, speed, link_up);
e90dd264 5184
0ecc061d
PWJ		 5185 return ret;
5186}
5187
a34bcfff 5188static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
9a799d71 5189{
9a799d71 5190 struct ixgbe_hw *hw = &adapter->hw;
a34bcfff 5191 u32 gpie = 0;
9a799d71 5192
9b471446 5193 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
a34bcfff
AD		 5194 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
5195 IXGBE_GPIE_OCD;
5196 gpie |= IXGBE_GPIE_EIAME;
9b471446
JB		 5197 /*
5198 * use EIAM to auto-mask when MSI-X interrupt is asserted
5199 * this saves a register write for every interrupt
5200 */
5201 switch (hw->mac.type) {
5202 case ixgbe_mac_82598EB:
5203 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5204 break;
9b471446 5205 case ixgbe_mac_82599EB:
b93a2226 5206 case ixgbe_mac_X540:
9a75a1ac
DS		 5207 case ixgbe_mac_X550:
5208 case ixgbe_mac_X550EM_x:
49425dfc 5209 case ixgbe_mac_x550em_a:
b93a2226 5210 default:
9b471446
JB		 5211 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
5212 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
5213 break;
5214 }
5215 } else {
021230d4
AV		 5216 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
5217 * specifically only auto mask tx and rx interrupts */
5218 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5219 }
9a799d71 5220
a34bcfff
AD		 5221 /* XXX: to interrupt immediately for EICS writes, enable this */
5222 /* gpie |= IXGBE_GPIE_EIMEN; */
5223
5224 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
5225 gpie &= ~IXGBE_GPIE_VTMODE_MASK;
73079ea0
AD		 5226
5227 switch (adapter->ring_feature[RING_F_VMDQ].mask) {
5228 case IXGBE_82599_VMDQ_8Q_MASK:
5229 gpie |= IXGBE_GPIE_VTMODE_16;
5230 break;
5231 case IXGBE_82599_VMDQ_4Q_MASK:
5232 gpie |= IXGBE_GPIE_VTMODE_32;
5233 break;
5234 default:
5235 gpie |= IXGBE_GPIE_VTMODE_64;
5236 break;
5237 }
119fc60a
MC		 5238 }
5239
5fdd31f9 5240 /* Enable Thermal over heat sensor interrupt */
f3df98ec
DS		 5241 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
5242 switch (adapter->hw.mac.type) {
5243 case ixgbe_mac_82599EB:
9a900eca 5244 gpie |= IXGBE_SDP0_GPIEN_8259X;
f3df98ec 5245 break;
f3df98ec
DS		 5246 default:
5247 break;
5248 }
5249 }
5fdd31f9 5250
a34bcfff
AD		 5251 /* Enable fan failure interrupt */
5252 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
9a900eca 5253 gpie |= IXGBE_SDP1_GPIEN(hw);
0befdb3e 5254
a023bbd0
DS		 5255 switch (hw->mac.type) {
5256 case ixgbe_mac_82599EB:
5257 gpie |= IXGBE_SDP1_GPIEN_8259X | IXGBE_SDP2_GPIEN_8259X;
5258 break;
5259 case ixgbe_mac_X550EM_x:
49425dfc 5260 case ixgbe_mac_x550em_a:
a023bbd0
DS		 5261 gpie |= IXGBE_SDP0_GPIEN_X540;
5262 break;
5263 default:
5264 break;
2698b208 5265 }
a34bcfff
AD		 5266
5267 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
5268}
5269
c7ccde0f 5270static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
a34bcfff
AD		 5271{
5272 struct ixgbe_hw *hw = &adapter->hw;
a34bcfff 5273 int err;
a34bcfff
AD		 5274 u32 ctrl_ext;
5275
5276 ixgbe_get_hw_control(adapter);
5277 ixgbe_setup_gpie(adapter);
e8e26350 5278
9a799d71
AK		 5279 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
5280 ixgbe_configure_msix(adapter);
5281 else
5282 ixgbe_configure_msi_and_legacy(adapter);
5283
ec74a471
ET		 5284 /* enable the optics for 82599 SFP+ fiber */
5285 if (hw->mac.ops.enable_tx_laser)
61fac744
PW		 5286 hw->mac.ops.enable_tx_laser(hw);
5287
961fac88
DS		 5288 if (hw->phy.ops.set_phy_power)
5289 hw->phy.ops.set_phy_power(hw, true);
5290
4e857c58 5291 smp_mb__before_atomic();
9a799d71 5292 clear_bit(__IXGBE_DOWN, &adapter->state);
021230d4
AV		 5293 ixgbe_napi_enable_all(adapter);
5294
73c4b7cd
AD		 5295 if (ixgbe_is_sfp(hw)) {
5296 ixgbe_sfp_link_config(adapter);
5297 } else {
5298 err = ixgbe_non_sfp_link_config(hw);
5299 if (err)
5300 e_err(probe, "link_config FAILED %d\n", err);
5301 }
5302
021230d4
AV		 5303 /* clear any pending interrupts, may auto mask */
5304 IXGBE_READ_REG(hw, IXGBE_EICR);
6af3b9eb 5305 ixgbe_irq_enable(adapter, true, true);
9a799d71 5306
bf069c97
DS		 5307 /*
5308 * If this adapter has a fan, check to see if we had a failure
5309 * before we enabled the interrupt.
5310 */
5311 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
5312 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
5313 if (esdp & IXGBE_ESDP_SDP1)
396e799c 5314 e_crit(drv, "Fan has stopped, replace the adapter\n");
bf069c97
DS		 5315 }
5316
9a799d71
AK		 5317 /* bring the link up in the watchdog, this could race with our first
5318 * link up interrupt but shouldn't be a problem */
cf8280ee
JB		 5319 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
5320 adapter->link_check_timeout = jiffies;
7086400d 5321 mod_timer(&adapter->service_timer, jiffies);
c9205697
GR		 5322
5323 /* Set PF Reset Done bit so PF/VF Mail Ops can work */
5324 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5325 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
5326 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
9a799d71
AK		 5327}
5328
d4f80882
AV		 5329void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
5330{
5331 WARN_ON(in_interrupt());
7086400d 5332 /* put off any impending NetWatchDogTimeout */
860e9538 5333 netif_trans_update(adapter->netdev);
7086400d 5334
d4f80882 5335 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
032b4325 5336 usleep_range(1000, 2000);
b3eb4e18
MR		 5337 if (adapter->hw.phy.type == ixgbe_phy_fw)
5338 ixgbe_watchdog_link_is_down(adapter);
d4f80882 5339 ixgbe_down(adapter);
5809a1ae
GR		 5340 /*
5341 * If SR-IOV enabled then wait a bit before bringing the adapter
5342 * back up to give the VFs time to respond to the reset. The
5343 * two second wait is based upon the watchdog timer cycle in
5344 * the VF driver.
5345 */
5346 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
5347 msleep(2000);
d4f80882
AV		 5348 ixgbe_up(adapter);
5349 clear_bit(__IXGBE_RESETTING, &adapter->state);
5350}
5351
c7ccde0f 5352void ixgbe_up(struct ixgbe_adapter *adapter)
9a799d71
AK		 5353{
5354 /* hardware has been reset, we need to reload some things */
5355 ixgbe_configure(adapter);
5356
c7ccde0f 5357 ixgbe_up_complete(adapter);
9a799d71
AK		 5358}
5359
5360void ixgbe_reset(struct ixgbe_adapter *adapter)
5361{
c44ade9e 5362 struct ixgbe_hw *hw = &adapter->hw;
5d7daa35 5363 struct net_device *netdev = adapter->netdev;
8ca783ab
DS		 5364 int err;
5365
b0483c8f
MR		 5366 if (ixgbe_removed(hw->hw_addr))
5367 return;
7086400d
AD		 5368 /* lock SFP init bit to prevent race conditions with the watchdog */
5369 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
5370 usleep_range(1000, 2000);
5371
5372 /* clear all SFP and link config related flags while holding SFP_INIT */
5373 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
5374 IXGBE_FLAG2_SFP_NEEDS_RESET);
5375 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
5376
8ca783ab 5377 err = hw->mac.ops.init_hw(hw);
da4dd0f7
PWJ		 5378 switch (err) {
5379 case 0:
5380 case IXGBE_ERR_SFP_NOT_PRESENT:
7086400d 5381 case IXGBE_ERR_SFP_NOT_SUPPORTED:
da4dd0f7
PWJ		 5382 break;
5383 case IXGBE_ERR_MASTER_REQUESTS_PENDING:
849c4542 5384 e_dev_err("master disable timed out\n");
da4dd0f7 5385 break;
794caeb2
PWJ		 5386 case IXGBE_ERR_EEPROM_VERSION:
5387 /* We are running on a pre-production device, log a warning */
849c4542 5388 e_dev_warn("This device is a pre-production adapter/LOM. "
52f33af8 5389 "Please be aware there may be issues associated with "
849c4542
ET		 5390 "your hardware. If you are experiencing problems "
5391 "please contact your Intel or hardware "
5392 "representative who provided you with this "
5393 "hardware.\n");
794caeb2 5394 break;
da4dd0f7 5395 default:
849c4542 5396 e_dev_err("Hardware Error: %d\n", err);
da4dd0f7 5397 }
9a799d71 5398
7086400d 5399 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
0f079d22
AD		 5400
5401 /* flush entries out of MAC table */
5d7daa35 5402 ixgbe_flush_sw_mac_table(adapter);
0f079d22
AD		 5403 __dev_uc_unsync(netdev, NULL);
5404
5405 /* do not flush user set addresses */
c9f53e63 5406 ixgbe_mac_set_default_filter(adapter);
7fa7c9dc
AD		 5407
5408 /* update SAN MAC vmdq pool selection */
5409 if (hw->mac.san_mac_rar_index)
5410 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
1a71ab24 5411
8fecf67c 5412 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
1a71ab24 5413 ixgbe_ptp_reset(adapter);
961fac88
DS		 5414
5415 if (hw->phy.ops.set_phy_power) {
5416 if (!netif_running(adapter->netdev) && !adapter->wol)
5417 hw->phy.ops.set_phy_power(hw, false);
5418 else
5419 hw->phy.ops.set_phy_power(hw, true);
5420 }
9a799d71
AK		 5421}
5422
9a799d71
AK		 5423/**
5424 * ixgbe_clean_tx_ring - Free Tx Buffers
9a799d71
AK		 5425 * @tx_ring: ring to be cleaned
5426 **/
b6ec895e 5427static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
9a799d71
AK		 5428{
5429 struct ixgbe_tx_buffer *tx_buffer_info;
5430 unsigned long size;
b6ec895e 5431 u16 i;
9a799d71 5432
84418e3b
AD		 5433 /* ring already cleared, nothing to do */
5434 if (!tx_ring->tx_buffer_info)
5435 return;
9a799d71 5436
84418e3b 5437 /* Free all the Tx ring sk_buffs */
9a799d71
AK		 5438 for (i = 0; i < tx_ring->count; i++) {
5439 tx_buffer_info = &tx_ring->tx_buffer_info[i];
b6ec895e 5440 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
9a799d71
AK		 5441 }
5442
dad8a3b3
JF		 5443 netdev_tx_reset_queue(txring_txq(tx_ring));
5444
9a799d71
AK		 5445 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
5446 memset(tx_ring->tx_buffer_info, 0, size);
5447
5448 /* Zero out the descriptor ring */
5449 memset(tx_ring->desc, 0, tx_ring->size);
5450
5451 tx_ring->next_to_use = 0;
5452 tx_ring->next_to_clean = 0;
9a799d71
AK		 5453}
5454
5455/**
021230d4 5456 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
9a799d71
AK		 5457 * @adapter: board private structure
5458 **/
021230d4 5459static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
9a799d71
AK		 5460{
5461 int i;
5462
021230d4 5463 for (i = 0; i < adapter->num_rx_queues; i++)
b6ec895e 5464 ixgbe_clean_rx_ring(adapter->rx_ring[i]);
9a799d71
AK		 5465}
5466
5467/**
021230d4 5468 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
9a799d71
AK		 5469 * @adapter: board private structure
5470 **/
021230d4 5471static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
9a799d71
AK		 5472{
5473 int i;
5474
021230d4 5475 for (i = 0; i < adapter->num_tx_queues; i++)
b6ec895e 5476 ixgbe_clean_tx_ring(adapter->tx_ring[i]);
9a799d71
AK		 5477}
5478
e4911d57
AD		 5479static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
5480{
b67bfe0d 5481 struct hlist_node *node2;
e4911d57
AD		 5482 struct ixgbe_fdir_filter *filter;
5483
5484 spin_lock(&adapter->fdir_perfect_lock);
5485
b67bfe0d 5486 hlist_for_each_entry_safe(filter, node2,
e4911d57
AD		 5487 &adapter->fdir_filter_list, fdir_node) {
5488 hlist_del(&filter->fdir_node);
5489 kfree(filter);
5490 }
5491 adapter->fdir_filter_count = 0;
5492
5493 spin_unlock(&adapter->fdir_perfect_lock);
5494}
5495
1cd127fc
DA		 5496static int ixgbe_disable_macvlan(struct net_device *upper, void *data)
5497{
5498 if (netif_is_macvlan(upper)) {
5499 struct macvlan_dev *vlan = netdev_priv(upper);
5500
5501 if (vlan->fwd_priv) {
5502 netif_tx_stop_all_queues(upper);
5503 netif_carrier_off(upper);
5504 netif_tx_disable(upper);
5505 }
5506 }
5507
5508 return 0;
5509}
5510
9a799d71
AK		 5511void ixgbe_down(struct ixgbe_adapter *adapter)
5512{
5513 struct net_device *netdev = adapter->netdev;
7f821875 5514 struct ixgbe_hw *hw = &adapter->hw;
bf29ee6c 5515 int i;
9a799d71
AK		 5516
5517 /* signal that we are down to the interrupt handler */
c3049c8f
MR		 5518 if (test_and_set_bit(__IXGBE_DOWN, &adapter->state))
5519 return; /* do nothing if already down */
9a799d71
AK		 5520
5521 /* disable receives */
1f9ac57c 5522 hw->mac.ops.disable_rx(hw);
9a799d71 5523
2d39d576
YZ		 5524 /* disable all enabled rx queues */
5525 for (i = 0; i < adapter->num_rx_queues; i++)
5526 /* this call also flushes the previous write */
5527 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
5528
032b4325 5529 usleep_range(10000, 20000);
9a799d71 5530
7f821875
JB		 5531 netif_tx_stop_all_queues(netdev);
5532
7086400d 5533 /* call carrier off first to avoid false dev_watchdog timeouts */
c0dfb90e
JF		 5534 netif_carrier_off(netdev);
5535 netif_tx_disable(netdev);
5536
2a47fa45 5537 /* disable any upper devices */
1cd127fc
DA		 5538 netdev_walk_all_upper_dev_rcu(adapter->netdev,
5539 ixgbe_disable_macvlan, NULL);
2a47fa45 5540
c0dfb90e
JF		 5541 ixgbe_irq_disable(adapter);
5542
5543 ixgbe_napi_disable_all(adapter);
5544
57ca2a4f
ET		 5545 clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
5546 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
7086400d
AD		 5547 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
5548
5549 del_timer_sync(&adapter->service_timer);
5550
34cecbbf 5551 if (adapter->num_vfs) {
8e34d1aa
AD		 5552 /* Clear EITR Select mapping */
5553 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
34cecbbf
AD		 5554
5555 /* Mark all the VFs as inactive */
5556 for (i = 0 ; i < adapter->num_vfs; i++)
3db1cd5c 5557 adapter->vfinfo[i].clear_to_send = false;
34cecbbf 5558
34cecbbf
AD		 5559 /* ping all the active vfs to let them know we are going down */
5560 ixgbe_ping_all_vfs(adapter);
5561
5562 /* Disable all VFTE/VFRE TX/RX */
5563 ixgbe_disable_tx_rx(adapter);
b25ebfd2
PW		 5564 }
5565
7f821875
JB		 5566 /* disable transmits in the hardware now that interrupts are off */
5567 for (i = 0; i < adapter->num_tx_queues; i++) {
bf29ee6c 5568 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
34cecbbf 5569 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
7f821875 5570 }
34cecbbf 5571
9a75a1ac 5572 /* Disable the Tx DMA engine on 82599 and later MAC */
bd508178
AD		 5573 switch (hw->mac.type) {
5574 case ixgbe_mac_82599EB:
b93a2226 5575 case ixgbe_mac_X540:
9a75a1ac
DS		 5576 case ixgbe_mac_X550:
5577 case ixgbe_mac_X550EM_x:
49425dfc 5578 case ixgbe_mac_x550em_a:
88512539 5579 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
e8e9f696
JP		 5580 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
5581 ~IXGBE_DMATXCTL_TE));
bd508178
AD		 5582 break;
5583 default:
5584 break;
5585 }
7f821875 5586
6f4a0e45
PL		 5587 if (!pci_channel_offline(adapter->pdev))
5588 ixgbe_reset(adapter);
c6ecf39a 5589
ec74a471
ET		 5590 /* power down the optics for 82599 SFP+ fiber */
5591 if (hw->mac.ops.disable_tx_laser)
c6ecf39a
DS		 5592 hw->mac.ops.disable_tx_laser(hw);
5593
9a799d71
AK		 5594 ixgbe_clean_all_tx_rings(adapter);
5595 ixgbe_clean_all_rx_rings(adapter);
9a799d71
AK		 5596}
5597
b3eb4e18
MR		 5598/**
5599 * ixgbe_eee_capable - helper function to determine EEE support on X550
5600 * @adapter: board private structure
5601 */
5602static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter)
5603{
5604 struct ixgbe_hw *hw = &adapter->hw;
5605
5606 switch (hw->device_id) {
5607 case IXGBE_DEV_ID_X550EM_A_1G_T:
5608 case IXGBE_DEV_ID_X550EM_A_1G_T_L:
5609 if (!hw->phy.eee_speeds_supported)
5610 break;
5611 adapter->flags2 |= IXGBE_FLAG2_EEE_CAPABLE;
5612 if (!hw->phy.eee_speeds_advertised)
5613 break;
5614 adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
5615 break;
5616 default:
5617 adapter->flags2 &= ~IXGBE_FLAG2_EEE_CAPABLE;
5618 adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
5619 break;
5620 }
5621}
5622
9a799d71
AK		 5623/**
5624 * ixgbe_tx_timeout - Respond to a Tx Hang
5625 * @netdev: network interface device structure
5626 **/
5627static void ixgbe_tx_timeout(struct net_device *netdev)
5628{
5629 struct ixgbe_adapter *adapter = netdev_priv(netdev);
5630
5631 /* Do the reset outside of interrupt context */
c83c6cbd 5632 ixgbe_tx_timeout_reset(adapter);
9a799d71
AK		 5633}
5634
8829009d
UK		 5635#ifdef CONFIG_IXGBE_DCB
5636static void ixgbe_init_dcb(struct ixgbe_adapter *adapter)
5637{
5638 struct ixgbe_hw *hw = &adapter->hw;
5639 struct tc_configuration *tc;
5640 int j;
5641
5642 switch (hw->mac.type) {
5643 case ixgbe_mac_82598EB:
5644 case ixgbe_mac_82599EB:
5645 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
5646 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
5647 break;
5648 case ixgbe_mac_X540:
5649 case ixgbe_mac_X550:
5650 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
5651 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
5652 break;
5653 case ixgbe_mac_X550EM_x:
5654 case ixgbe_mac_x550em_a:
5655 default:
5656 adapter->dcb_cfg.num_tcs.pg_tcs = DEF_TRAFFIC_CLASS;
5657 adapter->dcb_cfg.num_tcs.pfc_tcs = DEF_TRAFFIC_CLASS;
5658 break;
5659 }
5660
5661 /* Configure DCB traffic classes */
5662 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
5663 tc = &adapter->dcb_cfg.tc_config[j];
5664 tc->path[DCB_TX_CONFIG].bwg_id = 0;
5665 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
5666 tc->path[DCB_RX_CONFIG].bwg_id = 0;
5667 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
5668 tc->dcb_pfc = pfc_disabled;
5669 }
5670
5671 /* Initialize default user to priority mapping, UPx->TC0 */
5672 tc = &adapter->dcb_cfg.tc_config[0];
5673 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
5674 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
5675
5676 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
5677 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
5678 adapter->dcb_cfg.pfc_mode_enable = false;
5679 adapter->dcb_set_bitmap = 0x00;
5680 if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)
5681 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
5682 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
5683 sizeof(adapter->temp_dcb_cfg));
5684}
5685#endif
5686
9a799d71
AK		 5687/**
5688 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
5689 * @adapter: board private structure to initialize
5690 *
5691 * ixgbe_sw_init initializes the Adapter private data structure.
5692 * Fields are initialized based on PCI device information and
5693 * OS network device settings (MTU size).
5694 **/
55570b6f
ET		 5695static int ixgbe_sw_init(struct ixgbe_adapter *adapter,
5696 const struct ixgbe_info *ii)
9a799d71
AK		 5697{
5698 struct ixgbe_hw *hw = &adapter->hw;
5699 struct pci_dev *pdev = adapter->pdev;
d3cb9869 5700 unsigned int rss, fdir;
cb6d0f5e 5701 u32 fwsm;
1cdaaf54 5702 int i;
021230d4 5703
c44ade9e
JB		 5704 /* PCI config space info */
5705
5706 hw->vendor_id = pdev->vendor;
5707 hw->device_id = pdev->device;
5708 hw->revision_id = pdev->revision;
5709 hw->subsystem_vendor_id = pdev->subsystem_vendor;
5710 hw->subsystem_device_id = pdev->subsystem_device;
5711
55570b6f
ET		 5712 /* get_invariants needs the device IDs */
5713 ii->get_invariants(hw);
5714
8fc3bb6d 5715 /* Set common capability flags and settings */
0f9b232b 5716 rss = min_t(int, ixgbe_max_rss_indices(adapter), num_online_cpus());
c087663e 5717 adapter->ring_feature[RING_F_RSS].limit = rss;
8fc3bb6d 5718 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
8fc3bb6d
ET		 5719 adapter->max_q_vectors = MAX_Q_VECTORS_82599;
5720 adapter->atr_sample_rate = 20;
d3cb9869
AD		 5721 fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus());
5722 adapter->ring_feature[RING_F_FDIR].limit = fdir;
8fc3bb6d
ET		 5723 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
5724#ifdef CONFIG_IXGBE_DCA
5725 adapter->flags |= IXGBE_FLAG_DCA_CAPABLE;
5726#endif
8829009d
UK		 5727#ifdef CONFIG_IXGBE_DCB
5728 adapter->flags |= IXGBE_FLAG_DCB_CAPABLE;
5729 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
5730#endif
8fc3bb6d
ET		 5731#ifdef IXGBE_FCOE
5732 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
5733 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5734#ifdef CONFIG_IXGBE_DCB
5735 /* Default traffic class to use for FCoE */
5736 adapter->fcoe.up = IXGBE_FCOE_DEFTC;
5737#endif /* CONFIG_IXGBE_DCB */
5738#endif /* IXGBE_FCOE */
5739
b82b17d9 5740 /* initialize static ixgbe jump table entries */
1cdaaf54
AN		 5741 adapter->jump_tables[0] = kzalloc(sizeof(*adapter->jump_tables[0]),
5742 GFP_KERNEL);
5743 if (!adapter->jump_tables[0])
5744 return -ENOMEM;
5745 adapter->jump_tables[0]->mat = ixgbe_ipv4_fields;
5746
5747 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++)
5748 adapter->jump_tables[i] = NULL;
b82b17d9 5749
5d7daa35
JK		 5750 adapter->mac_table = kzalloc(sizeof(struct ixgbe_mac_addr) *
5751 hw->mac.num_rar_entries,
5752 GFP_ATOMIC);
530fd82a
AD		 5753 if (!adapter->mac_table)
5754 return -ENOMEM;
5d7daa35 5755
8fc3bb6d 5756 /* Set MAC specific capability flags and exceptions */
bd508178
AD		 5757 switch (hw->mac.type) {
5758 case ixgbe_mac_82598EB:
8fc3bb6d 5759 adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE;
8fc3bb6d 5760
bf069c97
DS		 5761 if (hw->device_id == IXGBE_DEV_ID_82598AT)
5762 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
8fc3bb6d 5763
49c7ffbe 5764 adapter->max_q_vectors = MAX_Q_VECTORS_82598;
8fc3bb6d
ET		 5765 adapter->ring_feature[RING_F_FDIR].limit = 0;
5766 adapter->atr_sample_rate = 0;
5767 adapter->fdir_pballoc = 0;
5768#ifdef IXGBE_FCOE
5769 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
5770 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5771#ifdef CONFIG_IXGBE_DCB
5772 adapter->fcoe.up = 0;
5773#endif /* IXGBE_DCB */
5774#endif /* IXGBE_FCOE */
5775 break;
5776 case ixgbe_mac_82599EB:
5777 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
5778 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
bd508178 5779 break;
b93a2226 5780 case ixgbe_mac_X540:
9a900eca 5781 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));
cb6d0f5e
JK		 5782 if (fwsm & IXGBE_FWSM_TS_ENABLED)
5783 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
bd508178 5784 break;
49425dfc 5785 case ixgbe_mac_x550em_a:
a21d0822 5786 adapter->flags |= IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE;
b3eb4e18
MR		 5787 switch (hw->device_id) {
5788 case IXGBE_DEV_ID_X550EM_A_1G_T:
5789 case IXGBE_DEV_ID_X550EM_A_1G_T_L:
5790 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5791 break;
5792 default:
5793 break;
5794 }
a21d0822
ET		 5795 /* fall through */
5796 case ixgbe_mac_X550EM_x:
8829009d
UK		 5797#ifdef CONFIG_IXGBE_DCB
5798 adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE;
5799#endif
5800#ifdef IXGBE_FCOE
5801 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
5802#ifdef CONFIG_IXGBE_DCB
5803 adapter->fcoe.up = 0;
5804#endif /* IXGBE_DCB */
5805#endif /* IXGBE_FCOE */
5806 /* Fall Through */
9a75a1ac 5807 case ixgbe_mac_X550:
b3eb4e18
MR		 5808 if (hw->mac.type == ixgbe_mac_X550)
5809 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
9a75a1ac
DS		 5810#ifdef CONFIG_IXGBE_DCA
5811 adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE;
67359c3c 5812#endif
67359c3c 5813 adapter->flags |= IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE;
9a75a1ac 5814 break;
bd508178
AD		 5815 default:
5816 break;
f8212f97 5817 }
2f90b865 5818
7c8ae65a
AD		 5819#ifdef IXGBE_FCOE
5820 /* FCoE support exists, always init the FCoE lock */
5821 spin_lock_init(&adapter->fcoe.lock);
5822
5823#endif
1fc5f038
AD		 5824 /* n-tuple support exists, always init our spinlock */
5825 spin_lock_init(&adapter->fdir_perfect_lock);
5826
7a6b6f51 5827#ifdef CONFIG_IXGBE_DCB
8829009d 5828 ixgbe_init_dcb(adapter);
2f90b865 5829#endif
9a799d71
AK		 5830
5831 /* default flow control settings */
cd7664f6 5832 hw->fc.requested_mode = ixgbe_fc_full;
71fd570b 5833 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
9da712d2 5834 ixgbe_pbthresh_setup(adapter);
2b9ade93
JB		 5835 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
5836 hw->fc.send_xon = true;
73d80953 5837 hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw);
9a799d71 5838
99d74487 5839#ifdef CONFIG_PCI_IOV
170e8543
JK		 5840 if (max_vfs > 0)
5841 e_dev_warn("Enabling SR-IOV VFs using the max_vfs module parameter is deprecated - please use the pci sysfs interface instead.\n");
5842
99d74487 5843 /* assign number of SR-IOV VFs */
170e8543 5844 if (hw->mac.type != ixgbe_mac_82598EB) {
dcc23e3a 5845 if (max_vfs > IXGBE_MAX_VFS_DRV_LIMIT) {
170e8543
JK		 5846 adapter->num_vfs = 0;
5847 e_dev_warn("max_vfs parameter out of range. Not assigning any SR-IOV VFs\n");
5848 } else {
5849 adapter->num_vfs = max_vfs;
5850 }
5851 }
5852#endif /* CONFIG_PCI_IOV */
99d74487 5853
30efa5a3 5854 /* enable itr by default in dynamic mode */
f7554a2b 5855 adapter->rx_itr_setting = 1;
f7554a2b 5856 adapter->tx_itr_setting = 1;
30efa5a3 5857
30efa5a3
JB		 5858 /* set default ring sizes */
5859 adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
5860 adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
5861
bd198058 5862 /* set default work limits */
59224555 5863 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
bd198058 5864
9a799d71 5865 /* initialize eeprom parameters */
c44ade9e 5866 if (ixgbe_init_eeprom_params_generic(hw)) {
849c4542 5867 e_dev_err("EEPROM initialization failed\n");
9a799d71
AK		 5868 return -EIO;
5869 }
5870
2a47fa45
JF		 5871 /* PF holds first pool slot */
5872 set_bit(0, &adapter->fwd_bitmask);
9a799d71
AK		 5873 set_bit(__IXGBE_DOWN, &adapter->state);
5874
5875 return 0;
5876}
5877
5878/**
5879 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
3a581073 5880 * @tx_ring: tx descriptor ring (for a specific queue) to setup
9a799d71
AK		 5881 *
5882 * Return 0 on success, negative on failure
5883 **/
b6ec895e 5884int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
9a799d71 5885{
b6ec895e 5886 struct device *dev = tx_ring->dev;
de88eeeb 5887 int orig_node = dev_to_node(dev);
ca8dfe25 5888 int ring_node = -1;
9a799d71
AK		 5889 int size;
5890
3a581073 5891 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
de88eeeb
AD		 5892
5893 if (tx_ring->q_vector)
ca8dfe25 5894 ring_node = tx_ring->q_vector->numa_node;
de88eeeb 5895
ca8dfe25 5896 tx_ring->tx_buffer_info = vzalloc_node(size, ring_node);
1a6c14a2 5897 if (!tx_ring->tx_buffer_info)
89bf67f1 5898 tx_ring->tx_buffer_info = vzalloc(size);
e01c31a5
JB		 5899 if (!tx_ring->tx_buffer_info)
5900 goto err;
9a799d71 5901
827da44c
JS		 5902 u64_stats_init(&tx_ring->syncp);
5903
9a799d71 5904 /* round up to nearest 4K */
12207e49 5905 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
3a581073 5906 tx_ring->size = ALIGN(tx_ring->size, 4096);
9a799d71 5907
ca8dfe25 5908 set_dev_node(dev, ring_node);
de88eeeb
AD		 5909 tx_ring->desc = dma_alloc_coherent(dev,
5910 tx_ring->size,
5911 &tx_ring->dma,
5912 GFP_KERNEL);
5913 set_dev_node(dev, orig_node);
5914 if (!tx_ring->desc)
5915 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
5916 &tx_ring->dma, GFP_KERNEL);
e01c31a5
JB		 5917 if (!tx_ring->desc)
5918 goto err;
9a799d71 5919
3a581073
JB		 5920 tx_ring->next_to_use = 0;
5921 tx_ring->next_to_clean = 0;
9a799d71 5922 return 0;
e01c31a5
JB		 5923
5924err:
5925 vfree(tx_ring->tx_buffer_info);
5926 tx_ring->tx_buffer_info = NULL;
b6ec895e 5927 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
e01c31a5 5928 return -ENOMEM;
9a799d71
AK		 5929}
5930
69888674
AD		 5931/**
5932 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
5933 * @adapter: board private structure
5934 *
5935 * If this function returns with an error, then it's possible one or
5936 * more of the rings is populated (while the rest are not). It is the
5937 * callers duty to clean those orphaned rings.
5938 *
5939 * Return 0 on success, negative on failure
5940 **/
5941static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
5942{
5943 int i, err = 0;
5944
5945 for (i = 0; i < adapter->num_tx_queues; i++) {
b6ec895e 5946 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
69888674
AD		 5947 if (!err)
5948 continue;
de3d5b94 5949
396e799c 5950 e_err(probe, "Allocation for Tx Queue %u failed\n", i);
de3d5b94 5951 goto err_setup_tx;
69888674
AD		 5952 }
5953
de3d5b94
AD		 5954 return 0;
5955err_setup_tx:
5956 /* rewind the index freeing the rings as we go */
5957 while (i--)
5958 ixgbe_free_tx_resources(adapter->tx_ring[i]);
69888674
AD		 5959 return err;
5960}
5961
9a799d71
AK		 5962/**
5963 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
3a581073 5964 * @rx_ring: rx descriptor ring (for a specific queue) to setup
9a799d71
AK		 5965 *
5966 * Returns 0 on success, negative on failure
5967 **/
b6ec895e 5968int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
9a799d71 5969{
b6ec895e 5970 struct device *dev = rx_ring->dev;
de88eeeb 5971 int orig_node = dev_to_node(dev);
ca8dfe25 5972 int ring_node = -1;
021230d4 5973 int size;
9a799d71 5974
3a581073 5975 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
de88eeeb
AD		 5976
5977 if (rx_ring->q_vector)
ca8dfe25 5978 ring_node = rx_ring->q_vector->numa_node;
de88eeeb 5979
ca8dfe25 5980 rx_ring->rx_buffer_info = vzalloc_node(size, ring_node);
1a6c14a2 5981 if (!rx_ring->rx_buffer_info)
89bf67f1 5982 rx_ring->rx_buffer_info = vzalloc(size);
b6ec895e
AD		 5983 if (!rx_ring->rx_buffer_info)
5984 goto err;
9a799d71 5985
827da44c
JS		 5986 u64_stats_init(&rx_ring->syncp);
5987
9a799d71 5988 /* Round up to nearest 4K */
3a581073
JB		 5989 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
5990 rx_ring->size = ALIGN(rx_ring->size, 4096);
9a799d71 5991
ca8dfe25 5992 set_dev_node(dev, ring_node);
de88eeeb
AD		 5993 rx_ring->desc = dma_alloc_coherent(dev,
5994 rx_ring->size,
5995 &rx_ring->dma,
5996 GFP_KERNEL);
5997 set_dev_node(dev, orig_node);
5998 if (!rx_ring->desc)
5999 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
6000 &rx_ring->dma, GFP_KERNEL);
b6ec895e
AD		 6001 if (!rx_ring->desc)
6002 goto err;
9a799d71 6003
3a581073
JB		 6004 rx_ring->next_to_clean = 0;
6005 rx_ring->next_to_use = 0;
9a799d71
AK		 6006
6007 return 0;
b6ec895e
AD		 6008err:
6009 vfree(rx_ring->rx_buffer_info);
6010 rx_ring->rx_buffer_info = NULL;
6011 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
177db6ff 6012 return -ENOMEM;
9a799d71
AK		 6013}
6014
69888674
AD		 6015/**
6016 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
6017 * @adapter: board private structure
6018 *
6019 * If this function returns with an error, then it's possible one or
6020 * more of the rings is populated (while the rest are not). It is the
6021 * callers duty to clean those orphaned rings.
6022 *
6023 * Return 0 on success, negative on failure
6024 **/
69888674
AD		 6025static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
6026{
6027 int i, err = 0;
6028
6029 for (i = 0; i < adapter->num_rx_queues; i++) {
b6ec895e 6030 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
69888674
AD		 6031 if (!err)
6032 continue;
de3d5b94 6033
396e799c 6034 e_err(probe, "Allocation for Rx Queue %u failed\n", i);
de3d5b94 6035 goto err_setup_rx;
69888674
AD		 6036 }
6037
7c8ae65a
AD		 6038#ifdef IXGBE_FCOE
6039 err = ixgbe_setup_fcoe_ddp_resources(adapter);
6040 if (!err)
6041#endif
6042 return 0;
de3d5b94
AD		 6043err_setup_rx:
6044 /* rewind the index freeing the rings as we go */
6045 while (i--)
6046 ixgbe_free_rx_resources(adapter->rx_ring[i]);
69888674
AD		 6047 return err;
6048}
6049
9a799d71
AK		 6050/**
6051 * ixgbe_free_tx_resources - Free Tx Resources per Queue
9a799d71
AK		 6052 * @tx_ring: Tx descriptor ring for a specific queue
6053 *
6054 * Free all transmit software resources
6055 **/
b6ec895e 6056void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
9a799d71 6057{
b6ec895e 6058 ixgbe_clean_tx_ring(tx_ring);
9a799d71
AK		 6059
6060 vfree(tx_ring->tx_buffer_info);
6061 tx_ring->tx_buffer_info = NULL;
6062
b6ec895e
AD		 6063 /* if not set, then don't free */
6064 if (!tx_ring->desc)
6065 return;
6066
6067 dma_free_coherent(tx_ring->dev, tx_ring->size,
6068 tx_ring->desc, tx_ring->dma);
9a799d71
AK		 6069
6070 tx_ring->desc = NULL;
6071}
6072
6073/**
6074 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
6075 * @adapter: board private structure
6076 *
6077 * Free all transmit software resources
6078 **/
6079static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
6080{
6081 int i;
6082
6083 for (i = 0; i < adapter->num_tx_queues; i++)
4a0b9ca0 6084 if (adapter->tx_ring[i]->desc)
b6ec895e 6085 ixgbe_free_tx_resources(adapter->tx_ring[i]);
9a799d71
AK		 6086}
6087
6088/**
b4617240 6089 * ixgbe_free_rx_resources - Free Rx Resources
9a799d71
AK		 6090 * @rx_ring: ring to clean the resources from
6091 *
6092 * Free all receive software resources
6093 **/
b6ec895e 6094void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
9a799d71 6095{
b6ec895e 6096 ixgbe_clean_rx_ring(rx_ring);
9a799d71
AK		 6097
6098 vfree(rx_ring->rx_buffer_info);
6099 rx_ring->rx_buffer_info = NULL;
6100
b6ec895e
AD		 6101 /* if not set, then don't free */
6102 if (!rx_ring->desc)
6103 return;
6104
6105 dma_free_coherent(rx_ring->dev, rx_ring->size,
6106 rx_ring->desc, rx_ring->dma);
9a799d71
AK		 6107
6108 rx_ring->desc = NULL;
6109}
6110
6111/**
6112 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
6113 * @adapter: board private structure
6114 *
6115 * Free all receive software resources
6116 **/
6117static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
6118{
6119 int i;
6120
7c8ae65a
AD		 6121#ifdef IXGBE_FCOE
6122 ixgbe_free_fcoe_ddp_resources(adapter);
6123
6124#endif
9a799d71 6125 for (i = 0; i < adapter->num_rx_queues; i++)
4a0b9ca0 6126 if (adapter->rx_ring[i]->desc)
b6ec895e 6127 ixgbe_free_rx_resources(adapter->rx_ring[i]);
9a799d71
AK		 6128}
6129
9a799d71
AK		 6130/**
6131 * ixgbe_change_mtu - Change the Maximum Transfer Unit
6132 * @netdev: network interface device structure
6133 * @new_mtu: new value for maximum frame size
6134 *
6135 * Returns 0 on success, negative on failure
6136 **/
6137static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
6138{
6139 struct ixgbe_adapter *adapter = netdev_priv(netdev);
655309e9
AD		 6140
6141 /*
872844dd
AD		 6142 * For 82599EB we cannot allow legacy VFs to enable their receive
6143 * paths when MTU greater than 1500 is configured. So display a
6144 * warning that legacy VFs will be disabled.
655309e9
AD		 6145 */
6146 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
6147 (adapter->hw.mac.type == ixgbe_mac_82599EB) &&
91c527a5 6148 (new_mtu > ETH_DATA_LEN))
872844dd 6149 e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n");
9a799d71 6150
396e799c 6151 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
655309e9 6152
021230d4 6153 /* must set new MTU before calling down or up */
9a799d71
AK		 6154 netdev->mtu = new_mtu;
6155
d4f80882
AV		 6156 if (netif_running(netdev))
6157 ixgbe_reinit_locked(adapter);
9a799d71
AK		 6158
6159 return 0;
6160}
6161
6162/**
6163 * ixgbe_open - Called when a network interface is made active
6164 * @netdev: network interface device structure
6165 *
6166 * Returns 0 on success, negative value on failure
6167 *
6168 * The open entry point is called when a network interface is made
6169 * active by the system (IFF_UP). At this point all resources needed
6170 * for transmit and receive operations are allocated, the interrupt
6171 * handler is registered with the OS, the watchdog timer is started,
6172 * and the stack is notified that the interface is ready.
6173 **/
6c211fe1 6174int ixgbe_open(struct net_device *netdev)
9a799d71
AK		 6175{
6176 struct ixgbe_adapter *adapter = netdev_priv(netdev);
961fac88 6177 struct ixgbe_hw *hw = &adapter->hw;
2a47fa45 6178 int err, queues;
4bebfaa5
AK		 6179
6180 /* disallow open during test */
6181 if (test_bit(__IXGBE_TESTING, &adapter->state))
6182 return -EBUSY;
9a799d71 6183
54386467
JB		 6184 netif_carrier_off(netdev);
6185
9a799d71
AK		 6186 /* allocate transmit descriptors */
6187 err = ixgbe_setup_all_tx_resources(adapter);
6188 if (err)
6189 goto err_setup_tx;
6190
9a799d71
AK		 6191 /* allocate receive descriptors */
6192 err = ixgbe_setup_all_rx_resources(adapter);
6193 if (err)
6194 goto err_setup_rx;
6195
6196 ixgbe_configure(adapter);
6197
021230d4 6198 err = ixgbe_request_irq(adapter);
9a799d71
AK		 6199 if (err)
6200 goto err_req_irq;
6201
ac802f5d 6202 /* Notify the stack of the actual queue counts. */
2a47fa45
JF		 6203 if (adapter->num_rx_pools > 1)
6204 queues = adapter->num_rx_queues_per_pool;
6205 else
6206 queues = adapter->num_tx_queues;
6207
6208 err = netif_set_real_num_tx_queues(netdev, queues);
ac802f5d
AD		 6209 if (err)
6210 goto err_set_queues;
6211
2a47fa45
JF		 6212 if (adapter->num_rx_pools > 1 &&
6213 adapter->num_rx_queues > IXGBE_MAX_L2A_QUEUES)
6214 queues = IXGBE_MAX_L2A_QUEUES;
6215 else
6216 queues = adapter->num_rx_queues;
6217 err = netif_set_real_num_rx_queues(netdev, queues);
ac802f5d
AD		 6218 if (err)
6219 goto err_set_queues;
6220
1a71ab24 6221 ixgbe_ptp_init(adapter);
1a71ab24 6222
c7ccde0f 6223 ixgbe_up_complete(adapter);
9a799d71 6224
a21d0822 6225 ixgbe_clear_udp_tunnel_port(adapter, IXGBE_VXLANCTRL_ALL_UDPPORT_MASK);
b3a49557 6226 udp_tunnel_get_rx_info(netdev);
67359c3c 6227
9a799d71
AK		 6228 return 0;
6229
ac802f5d
AD		 6230err_set_queues:
6231 ixgbe_free_irq(adapter);
9a799d71 6232err_req_irq:
a20a1199 6233 ixgbe_free_all_rx_resources(adapter);
961fac88
DS		 6234 if (hw->phy.ops.set_phy_power && !adapter->wol)
6235 hw->phy.ops.set_phy_power(&adapter->hw, false);
de3d5b94 6236err_setup_rx:
a20a1199 6237 ixgbe_free_all_tx_resources(adapter);
de3d5b94 6238err_setup_tx:
9a799d71
AK		 6239 ixgbe_reset(adapter);
6240
6241 return err;
6242}
6243
a0cccce2
JK		 6244static void ixgbe_close_suspend(struct ixgbe_adapter *adapter)
6245{
6246 ixgbe_ptp_suspend(adapter);
6247
6ac74394
DS		 6248 if (adapter->hw.phy.ops.enter_lplu) {
6249 adapter->hw.phy.reset_disable = true;
6250 ixgbe_down(adapter);
6251 adapter->hw.phy.ops.enter_lplu(&adapter->hw);
6252 adapter->hw.phy.reset_disable = false;
6253 } else {
6254 ixgbe_down(adapter);
6255 }
6256
a0cccce2
JK		 6257 ixgbe_free_irq(adapter);
6258
6259 ixgbe_free_all_tx_resources(adapter);
6260 ixgbe_free_all_rx_resources(adapter);
6261}
6262
9a799d71
AK		 6263/**
6264 * ixgbe_close - Disables a network interface
6265 * @netdev: network interface device structure
6266 *
6267 * Returns 0, this is not allowed to fail
6268 *
6269 * The close entry point is called when an interface is de-activated
6270 * by the OS. The hardware is still under the drivers control, but
6271 * needs to be disabled. A global MAC reset is issued to stop the
6272 * hardware, and all transmit and receive resources are freed.
6273 **/
6c211fe1 6274int ixgbe_close(struct net_device *netdev)
9a799d71
AK		 6275{
6276 struct ixgbe_adapter *adapter = netdev_priv(netdev);
9a799d71 6277
1a71ab24 6278 ixgbe_ptp_stop(adapter);
1a71ab24 6279
f7f37e7f
ET		 6280 if (netif_device_present(netdev))
6281 ixgbe_close_suspend(adapter);
9a799d71 6282
e4911d57
AD		 6283 ixgbe_fdir_filter_exit(adapter);
6284
5eba3699 6285 ixgbe_release_hw_control(adapter);
9a799d71
AK		 6286
6287 return 0;
6288}
6289
b3c8b4ba
AD		 6290#ifdef CONFIG_PM
6291static int ixgbe_resume(struct pci_dev *pdev)
6292{
c60fbb00
AD		 6293 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
6294 struct net_device *netdev = adapter->netdev;
b3c8b4ba
AD		 6295 u32 err;
6296
0391bbe3 6297 adapter->hw.hw_addr = adapter->io_addr;
b3c8b4ba
AD		 6298 pci_set_power_state(pdev, PCI_D0);
6299 pci_restore_state(pdev);
656ab817
DS		 6300 /*
6301 * pci_restore_state clears dev->state_saved so call
6302 * pci_save_state to restore it.
6303 */
6304 pci_save_state(pdev);
9ce77666 6305
6306 err = pci_enable_device_mem(pdev);
b3c8b4ba 6307 if (err) {
849c4542 6308 e_dev_err("Cannot enable PCI device from suspend\n");
b3c8b4ba
AD		 6309 return err;
6310 }
4e857c58 6311 smp_mb__before_atomic();
41c62843 6312 clear_bit(__IXGBE_DISABLED, &adapter->state);
b3c8b4ba
AD		 6313 pci_set_master(pdev);
6314
dd4d8ca6 6315 pci_wake_from_d3(pdev, false);
b3c8b4ba 6316
b3c8b4ba
AD		 6317 ixgbe_reset(adapter);
6318
495dce12
WJP		 6319 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
6320
ac802f5d
AD		 6321 rtnl_lock();
6322 err = ixgbe_init_interrupt_scheme(adapter);
6323 if (!err && netif_running(netdev))
c60fbb00 6324 err = ixgbe_open(netdev);
ac802f5d 6325
ac802f5d 6326
f7f37e7f
ET		 6327 if (!err)
6328 netif_device_attach(netdev);
6329 rtnl_unlock();
b3c8b4ba 6330
f7f37e7f 6331 return err;
b3c8b4ba 6332}
b3c8b4ba 6333#endif /* CONFIG_PM */
9d8d05ae
RW		 6334
6335static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
b3c8b4ba 6336{
c60fbb00
AD		 6337 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
6338 struct net_device *netdev = adapter->netdev;
e8e26350
PW		 6339 struct ixgbe_hw *hw = &adapter->hw;
6340 u32 ctrl, fctrl;
6341 u32 wufc = adapter->wol;
b3c8b4ba
AD		 6342#ifdef CONFIG_PM
6343 int retval = 0;
6344#endif
6345
f7f37e7f 6346 rtnl_lock();
b3c8b4ba
AD		 6347 netif_device_detach(netdev);
6348
a0cccce2
JK		 6349 if (netif_running(netdev))
6350 ixgbe_close_suspend(adapter);
b3c8b4ba 6351
5f5ae6fc 6352 ixgbe_clear_interrupt_scheme(adapter);
f7f37e7f 6353 rtnl_unlock();
5f5ae6fc 6354
b3c8b4ba
AD		 6355#ifdef CONFIG_PM
6356 retval = pci_save_state(pdev);
6357 if (retval)
6358 return retval;
4df10466 6359
b3c8b4ba 6360#endif
f4f1040a
JK		 6361 if (hw->mac.ops.stop_link_on_d3)
6362 hw->mac.ops.stop_link_on_d3(hw);
6363
e8e26350
PW		 6364 if (wufc) {
6365 ixgbe_set_rx_mode(netdev);
b3c8b4ba 6366
ec74a471
ET		 6367 /* enable the optics for 82599 SFP+ fiber as we can WoL */
6368 if (hw->mac.ops.enable_tx_laser)
c509e754
DS		 6369 hw->mac.ops.enable_tx_laser(hw);
6370
e8e26350
PW		 6371 /* turn on all-multi mode if wake on multicast is enabled */
6372 if (wufc & IXGBE_WUFC_MC) {
6373 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
6374 fctrl |= IXGBE_FCTRL_MPE;
6375 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
6376 }
6377
6378 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
6379 ctrl |= IXGBE_CTRL_GIO_DIS;
6380 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
6381
6382 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
6383 } else {
6384 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
6385 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
6386 }
6387
bd508178
AD		 6388 switch (hw->mac.type) {
6389 case ixgbe_mac_82598EB:
dd4d8ca6 6390 pci_wake_from_d3(pdev, false);
bd508178
AD		 6391 break;
6392 case ixgbe_mac_82599EB:
b93a2226 6393 case ixgbe_mac_X540:
9a75a1ac
DS		 6394 case ixgbe_mac_X550:
6395 case ixgbe_mac_X550EM_x:
49425dfc 6396 case ixgbe_mac_x550em_a:
bd508178
AD		 6397 pci_wake_from_d3(pdev, !!wufc);
6398 break;
6399 default:
6400 break;
6401 }
b3c8b4ba 6402
9d8d05ae 6403 *enable_wake = !!wufc;
961fac88
DS		 6404 if (hw->phy.ops.set_phy_power && !*enable_wake)
6405 hw->phy.ops.set_phy_power(hw, false);
9d8d05ae 6406
b3c8b4ba
AD		 6407 ixgbe_release_hw_control(adapter);
6408
41c62843
MR		 6409 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))
6410 pci_disable_device(pdev);
b3c8b4ba 6411
9d8d05ae
RW		 6412 return 0;
6413}
6414
6415#ifdef CONFIG_PM
6416static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
6417{
6418 int retval;
6419 bool wake;
6420
6421 retval = __ixgbe_shutdown(pdev, &wake);
6422 if (retval)
6423 return retval;
6424
6425 if (wake) {
6426 pci_prepare_to_sleep(pdev);
6427 } else {
6428 pci_wake_from_d3(pdev, false);
6429 pci_set_power_state(pdev, PCI_D3hot);
6430 }
b3c8b4ba
AD		 6431
6432 return 0;
6433}
9d8d05ae 6434#endif /* CONFIG_PM */
b3c8b4ba
AD		 6435
6436static void ixgbe_shutdown(struct pci_dev *pdev)
6437{
9d8d05ae
RW		 6438 bool wake;
6439
6440 __ixgbe_shutdown(pdev, &wake);
6441
6442 if (system_state == SYSTEM_POWER_OFF) {
6443 pci_wake_from_d3(pdev, wake);
6444 pci_set_power_state(pdev, PCI_D3hot);
6445 }
b3c8b4ba
AD		 6446}
6447
9a799d71
AK		 6448/**
6449 * ixgbe_update_stats - Update the board statistics counters.
6450 * @adapter: board private structure
6451 **/
6452void ixgbe_update_stats(struct ixgbe_adapter *adapter)
6453{
2d86f139 6454 struct net_device *netdev = adapter->netdev;
9a799d71 6455 struct ixgbe_hw *hw = &adapter->hw;
5b7da515 6456 struct ixgbe_hw_stats *hwstats = &adapter->stats;
6f11eef7
AV		 6457 u64 total_mpc = 0;
6458 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
5b7da515
AD		 6459 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
6460 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
8a0da21b 6461 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;
9a799d71 6462
d08935c2
DS		 6463 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6464 test_bit(__IXGBE_RESETTING, &adapter->state))
6465 return;
6466
94b982b2 6467 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
f8212f97 6468 u64 rsc_count = 0;
94b982b2 6469 u64 rsc_flush = 0;
94b982b2 6470 for (i = 0; i < adapter->num_rx_queues; i++) {
5b7da515
AD		 6471 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
6472 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
94b982b2
MC		 6473 }
6474 adapter->rsc_total_count = rsc_count;
6475 adapter->rsc_total_flush = rsc_flush;
d51019a4
PW		 6476 }
6477
5b7da515
AD		 6478 for (i = 0; i < adapter->num_rx_queues; i++) {
6479 struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
6480 non_eop_descs += rx_ring->rx_stats.non_eop_descs;
6481 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
6482 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
8a0da21b 6483 hw_csum_rx_error += rx_ring->rx_stats.csum_err;
5b7da515
AD		 6484 bytes += rx_ring->stats.bytes;
6485 packets += rx_ring->stats.packets;
6486 }
6487 adapter->non_eop_descs = non_eop_descs;
6488 adapter->alloc_rx_page_failed = alloc_rx_page_failed;
6489 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
8a0da21b 6490 adapter->hw_csum_rx_error = hw_csum_rx_error;
5b7da515
AD		 6491 netdev->stats.rx_bytes = bytes;
6492 netdev->stats.rx_packets = packets;
6493
6494 bytes = 0;
6495 packets = 0;
7ca3bc58 6496 /* gather some stats to the adapter struct that are per queue */
5b7da515
AD		 6497 for (i = 0; i < adapter->num_tx_queues; i++) {
6498 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
6499 restart_queue += tx_ring->tx_stats.restart_queue;
6500 tx_busy += tx_ring->tx_stats.tx_busy;
6501 bytes += tx_ring->stats.bytes;
6502 packets += tx_ring->stats.packets;
6503 }
eb985f09 6504 adapter->restart_queue = restart_queue;
5b7da515
AD		 6505 adapter->tx_busy = tx_busy;
6506 netdev->stats.tx_bytes = bytes;
6507 netdev->stats.tx_packets = packets;
7ca3bc58 6508
7ca647bd 6509 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1a70db4b
ET		 6510
6511 /* 8 register reads */
6f11eef7
AV		 6512 for (i = 0; i < 8; i++) {
6513 /* for packet buffers not used, the register should read 0 */
6514 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
6515 missed_rx += mpc;
7ca647bd
JP		 6516 hwstats->mpc[i] += mpc;
6517 total_mpc += hwstats->mpc[i];
1a70db4b
ET		 6518 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
6519 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
bd508178
AD		 6520 switch (hw->mac.type) {
6521 case ixgbe_mac_82598EB:
1a70db4b
ET		 6522 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
6523 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
6524 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
7ca647bd
JP		 6525 hwstats->pxonrxc[i] +=
6526 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
bd508178
AD		 6527 break;
6528 case ixgbe_mac_82599EB:
b93a2226 6529 case ixgbe_mac_X540:
9a75a1ac
DS		 6530 case ixgbe_mac_X550:
6531 case ixgbe_mac_X550EM_x:
49425dfc 6532 case ixgbe_mac_x550em_a:
bd508178
AD		 6533 hwstats->pxonrxc[i] +=
6534 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
bd508178
AD		 6535 break;
6536 default:
6537 break;
e8e26350 6538 }
6f11eef7 6539 }
1a70db4b
ET		 6540
6541 /*16 register reads */
6542 for (i = 0; i < 16; i++) {
6543 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
6544 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
6545 if ((hw->mac.type == ixgbe_mac_82599EB) ||
9a75a1ac
DS		 6546 (hw->mac.type == ixgbe_mac_X540) ||
6547 (hw->mac.type == ixgbe_mac_X550) ||
49425dfc
MR		 6548 (hw->mac.type == ixgbe_mac_X550EM_x) ||
6549 (hw->mac.type == ixgbe_mac_x550em_a)) {
1a70db4b
ET		 6550 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
6551 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
6552 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
6553 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
6554 }
6555 }
6556
7ca647bd 6557 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
6f11eef7 6558 /* work around hardware counting issue */
7ca647bd 6559 hwstats->gprc -= missed_rx;
6f11eef7 6560
c84d324c
JF		 6561 ixgbe_update_xoff_received(adapter);
6562
6f11eef7 6563 /* 82598 hardware only has a 32 bit counter in the high register */
bd508178
AD		 6564 switch (hw->mac.type) {
6565 case ixgbe_mac_82598EB:
6566 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
bd508178
AD		 6567 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
6568 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
6569 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
6570 break;
b93a2226 6571 case ixgbe_mac_X540:
9a75a1ac
DS		 6572 case ixgbe_mac_X550:
6573 case ixgbe_mac_X550EM_x:
49425dfc 6574 case ixgbe_mac_x550em_a:
9a75a1ac 6575 /* OS2BMC stats are X540 and later */
58f6bcf9
ET		 6576 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
6577 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
6578 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
6579 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
6580 case ixgbe_mac_82599EB:
a4d4f629
AD		 6581 for (i = 0; i < 16; i++)
6582 adapter->hw_rx_no_dma_resources +=
6583 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
7ca647bd 6584 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
bd508178 6585 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
7ca647bd 6586 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
bd508178 6587 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
7ca647bd 6588 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
bd508178 6589 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
7ca647bd 6590 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
7ca647bd
JP		 6591 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
6592 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
6d45522c 6593#ifdef IXGBE_FCOE
7ca647bd
JP		 6594 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
6595 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
6596 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
6597 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
6598 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
6599 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
7b859ebc 6600 /* Add up per cpu counters for total ddp aloc fail */
5a1ee270
AD		 6601 if (adapter->fcoe.ddp_pool) {
6602 struct ixgbe_fcoe *fcoe = &adapter->fcoe;
6603 struct ixgbe_fcoe_ddp_pool *ddp_pool;
6604 unsigned int cpu;
6605 u64 noddp = 0, noddp_ext_buff = 0;
7b859ebc 6606 for_each_possible_cpu(cpu) {
5a1ee270
AD		 6607 ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
6608 noddp += ddp_pool->noddp;
6609 noddp_ext_buff += ddp_pool->noddp_ext_buff;
7b859ebc 6610 }
5a1ee270
AD		 6611 hwstats->fcoe_noddp = noddp;
6612 hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;
7b859ebc 6613 }
6d45522c 6614#endif /* IXGBE_FCOE */
bd508178
AD		 6615 break;
6616 default:
6617 break;
e8e26350 6618 }
9a799d71 6619 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
7ca647bd
JP		 6620 hwstats->bprc += bprc;
6621 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
e8e26350 6622 if (hw->mac.type == ixgbe_mac_82598EB)
7ca647bd
JP		 6623 hwstats->mprc -= bprc;
6624 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
6625 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
6626 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
6627 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
6628 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
6629 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
6630 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
6631 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
6f11eef7 6632 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
7ca647bd 6633 hwstats->lxontxc += lxon;
6f11eef7 6634 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
7ca647bd 6635 hwstats->lxofftxc += lxoff;
7ca647bd
JP		 6636 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
6637 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
6f11eef7
AV		 6638 /*
6639 * 82598 errata - tx of flow control packets is included in tx counters
6640 */
6641 xon_off_tot = lxon + lxoff;
7ca647bd
JP		 6642 hwstats->gptc -= xon_off_tot;
6643 hwstats->mptc -= xon_off_tot;
6644 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
6645 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
6646 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
6647 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
6648 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
6649 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
6650 hwstats->ptc64 -= xon_off_tot;
6651 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
6652 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
6653 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
6654 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
6655 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
6656 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
9a799d71
AK		 6657
6658 /* Fill out the OS statistics structure */
7ca647bd 6659 netdev->stats.multicast = hwstats->mprc;
9a799d71
AK		 6660
6661 /* Rx Errors */
7ca647bd 6662 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
2d86f139 6663 netdev->stats.rx_dropped = 0;
7ca647bd
JP		 6664 netdev->stats.rx_length_errors = hwstats->rlec;
6665 netdev->stats.rx_crc_errors = hwstats->crcerrs;
2d86f139 6666 netdev->stats.rx_missed_errors = total_mpc;
9a799d71
AK		 6667}
6668
6669/**
d034acf1 6670 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
49ce9c2c 6671 * @adapter: pointer to the device adapter structure
9a799d71 6672 **/
d034acf1 6673static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
9a799d71 6674{
cf8280ee 6675 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 6676 int i;
cf8280ee 6677
d034acf1
AD		 6678 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
6679 return;
6680
6681 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
22d5a71b 6682
d034acf1 6683 /* if interface is down do nothing */
fe49f04a 6684 if (test_bit(__IXGBE_DOWN, &adapter->state))
d034acf1
AD		 6685 return;
6686
6687 /* do nothing if we are not using signature filters */
6688 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
6689 return;
6690
6691 adapter->fdir_overflow++;
6692
93c52dd0
AD		 6693 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
6694 for (i = 0; i < adapter->num_tx_queues; i++)
6695 set_bit(__IXGBE_TX_FDIR_INIT_DONE,
e7cf745b 6696 &(adapter->tx_ring[i]->state));
d034acf1
AD		 6697 /* re-enable flow director interrupts */
6698 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
93c52dd0
AD		 6699 } else {
6700 e_err(probe, "failed to finish FDIR re-initialization, "
6701 "ignored adding FDIR ATR filters\n");
6702 }
93c52dd0
AD		 6703}
6704
6705/**
6706 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
49ce9c2c 6707 * @adapter: pointer to the device adapter structure
93c52dd0
AD		 6708 *
6709 * This function serves two purposes. First it strobes the interrupt lines
52f33af8 6710 * in order to make certain interrupts are occurring. Secondly it sets the
93c52dd0 6711 * bits needed to check for TX hangs. As a result we should immediately
52f33af8 6712 * determine if a hang has occurred.
93c52dd0
AD		 6713 */
6714static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
9a799d71 6715{
cf8280ee 6716 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a
AD		 6717 u64 eics = 0;
6718 int i;
cf8280ee 6719
09f40aed 6720 /* If we're down, removing or resetting, just bail */
93c52dd0 6721 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
09f40aed 6722 test_bit(__IXGBE_REMOVING, &adapter->state) ||
93c52dd0
AD		 6723 test_bit(__IXGBE_RESETTING, &adapter->state))
6724 return;
22d5a71b 6725
93c52dd0
AD		 6726 /* Force detection of hung controller */
6727 if (netif_carrier_ok(adapter->netdev)) {
6728 for (i = 0; i < adapter->num_tx_queues; i++)
6729 set_check_for_tx_hang(adapter->tx_ring[i]);
6730 }
22d5a71b 6731
fe49f04a
AD		 6732 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
6733 /*
6734 * for legacy and MSI interrupts don't set any bits
6735 * that are enabled for EIAM, because this operation
6736 * would set *both* EIMS and EICS for any bit in EIAM
6737 */
6738 IXGBE_WRITE_REG(hw, IXGBE_EICS,
6739 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
93c52dd0
AD		 6740 } else {
6741 /* get one bit for every active tx/rx interrupt vector */
49c7ffbe 6742 for (i = 0; i < adapter->num_q_vectors; i++) {
93c52dd0 6743 struct ixgbe_q_vector *qv = adapter->q_vector[i];
efe3d3c8 6744 if (qv->rx.ring || qv->tx.ring)
b4f47a48 6745 eics |= BIT_ULL(i);
93c52dd0 6746 }
cf8280ee 6747 }
9a799d71 6748
93c52dd0 6749 /* Cause software interrupt to ensure rings are cleaned */
fe49f04a 6750 ixgbe_irq_rearm_queues(adapter, eics);
cf8280ee
JB		 6751}
6752
e8e26350 6753/**
93c52dd0 6754 * ixgbe_watchdog_update_link - update the link status
49ce9c2c
BH		 6755 * @adapter: pointer to the device adapter structure
6756 * @link_speed: pointer to a u32 to store the link_speed
e8e26350 6757 **/
93c52dd0 6758static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
e8e26350 6759{
e8e26350 6760 struct ixgbe_hw *hw = &adapter->hw;
93c52dd0
AD		 6761 u32 link_speed = adapter->link_speed;
6762 bool link_up = adapter->link_up;
041441d0 6763 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
e8e26350 6764
93c52dd0
AD		 6765 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
6766 return;
6767
6768 if (hw->mac.ops.check_link) {
6769 hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
c4cf55e5 6770 } else {
93c52dd0
AD		 6771 /* always assume link is up, if no check link function */
6772 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
6773 link_up = true;
c4cf55e5 6774 }
041441d0
AD		 6775
6776 if (adapter->ixgbe_ieee_pfc)
6777 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
6778
3ebe8fde 6779 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
041441d0 6780 hw->mac.ops.fc_enable(hw);
3ebe8fde
AD		 6781 ixgbe_set_rx_drop_en(adapter);
6782 }
93c52dd0
AD		 6783
6784 if (link_up ||
6785 time_after(jiffies, (adapter->link_check_timeout +
6786 IXGBE_TRY_LINK_TIMEOUT))) {
6787 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
6788 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
6789 IXGBE_WRITE_FLUSH(hw);
6790 }
6791
6792 adapter->link_up = link_up;
6793 adapter->link_speed = link_speed;
e8e26350
PW		 6794}
6795
107d3018
AD		 6796static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)
6797{
6798#ifdef CONFIG_IXGBE_DCB
6799 struct net_device *netdev = adapter->netdev;
6800 struct dcb_app app = {
6801 .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE,
6802 .protocol = 0,
6803 };
6804 u8 up = 0;
6805
6806 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)
6807 up = dcb_ieee_getapp_mask(netdev, &app);
6808
6809 adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0;
6810#endif
6811}
6812
1cd127fc
DA		 6813static int ixgbe_enable_macvlan(struct net_device *upper, void *data)
6814{
6815 if (netif_is_macvlan(upper)) {
6816 struct macvlan_dev *vlan = netdev_priv(upper);
6817
6818 if (vlan->fwd_priv)
6819 netif_tx_wake_all_queues(upper);
6820 }
6821
6822 return 0;
6823}
6824
e8e26350 6825/**
93c52dd0
AD		 6826 * ixgbe_watchdog_link_is_up - update netif_carrier status and
6827 * print link up message
49ce9c2c 6828 * @adapter: pointer to the device adapter structure
e8e26350 6829 **/
93c52dd0 6830static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
e8e26350 6831{
93c52dd0 6832 struct net_device *netdev = adapter->netdev;
e8e26350 6833 struct ixgbe_hw *hw = &adapter->hw;
93c52dd0 6834 u32 link_speed = adapter->link_speed;
454adb00 6835 const char *speed_str;
93c52dd0 6836 bool flow_rx, flow_tx;
e8e26350 6837
93c52dd0
AD		 6838 /* only continue if link was previously down */
6839 if (netif_carrier_ok(netdev))
a985b6c3 6840 return;
63d6e1d8 6841
93c52dd0 6842 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
63d6e1d8 6843
93c52dd0
AD		 6844 switch (hw->mac.type) {
6845 case ixgbe_mac_82598EB: {
6846 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
6847 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
6848 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
6849 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
6850 }
6851 break;
6852 case ixgbe_mac_X540:
9a75a1ac
DS		 6853 case ixgbe_mac_X550:
6854 case ixgbe_mac_X550EM_x:
49425dfc 6855 case ixgbe_mac_x550em_a:
93c52dd0
AD		 6856 case ixgbe_mac_82599EB: {
6857 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
6858 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
6859 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
6860 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
6861 }
6862 break;
6863 default:
6864 flow_tx = false;
6865 flow_rx = false;
6866 break;
e8e26350 6867 }
3a6a4eda 6868
6cb562d6
JK		 6869 adapter->last_rx_ptp_check = jiffies;
6870
8fecf67c 6871 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
1a71ab24 6872 ixgbe_ptp_start_cyclecounter(adapter);
3a6a4eda 6873
454adb00
MR		 6874 switch (link_speed) {
6875 case IXGBE_LINK_SPEED_10GB_FULL:
6876 speed_str = "10 Gbps";
6877 break;
6878 case IXGBE_LINK_SPEED_2_5GB_FULL:
6879 speed_str = "2.5 Gbps";
6880 break;
6881 case IXGBE_LINK_SPEED_1GB_FULL:
6882 speed_str = "1 Gbps";
6883 break;
6884 case IXGBE_LINK_SPEED_100_FULL:
6885 speed_str = "100 Mbps";
6886 break;
b3eb4e18
MR		 6887 case IXGBE_LINK_SPEED_10_FULL:
6888 speed_str = "10 Mbps";
6889 break;
454adb00
MR		 6890 default:
6891 speed_str = "unknown speed";
6892 break;
6893 }
6894 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str,
93c52dd0
AD		 6895 ((flow_rx && flow_tx) ? "RX/TX" :
6896 (flow_rx ? "RX" :
6897 (flow_tx ? "TX" : "None"))));
e8e26350 6898
93c52dd0 6899 netif_carrier_on(netdev);
93c52dd0 6900 ixgbe_check_vf_rate_limit(adapter);
befa2af7 6901
cdc04dcc
ET		 6902 /* enable transmits */
6903 netif_tx_wake_all_queues(adapter->netdev);
6904
6905 /* enable any upper devices */
6906 rtnl_lock();
1cd127fc
DA		 6907 netdev_walk_all_upper_dev_rcu(adapter->netdev,
6908 ixgbe_enable_macvlan, NULL);
cdc04dcc
ET		 6909 rtnl_unlock();
6910
107d3018
AD		 6911 /* update the default user priority for VFs */
6912 ixgbe_update_default_up(adapter);
6913
befa2af7
AD		 6914 /* ping all the active vfs to let them know link has changed */
6915 ixgbe_ping_all_vfs(adapter);
e8e26350
PW		 6916}
6917
c4cf55e5 6918/**
93c52dd0
AD		 6919 * ixgbe_watchdog_link_is_down - update netif_carrier status and
6920 * print link down message
49ce9c2c 6921 * @adapter: pointer to the adapter structure
c4cf55e5 6922 **/
581330ba 6923static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
c4cf55e5 6924{
cf8280ee 6925 struct net_device *netdev = adapter->netdev;
c4cf55e5 6926 struct ixgbe_hw *hw = &adapter->hw;
10eec955 6927
93c52dd0
AD		 6928 adapter->link_up = false;
6929 adapter->link_speed = 0;
cf8280ee 6930
93c52dd0
AD		 6931 /* only continue if link was up previously */
6932 if (!netif_carrier_ok(netdev))
6933 return;
264857b8 6934
93c52dd0
AD		 6935 /* poll for SFP+ cable when link is down */
6936 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
6937 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
9a799d71 6938
8fecf67c 6939 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
1a71ab24 6940 ixgbe_ptp_start_cyclecounter(adapter);
3a6a4eda 6941
93c52dd0
AD		 6942 e_info(drv, "NIC Link is Down\n");
6943 netif_carrier_off(netdev);
befa2af7
AD		 6944
6945 /* ping all the active vfs to let them know link has changed */
6946 ixgbe_ping_all_vfs(adapter);
93c52dd0 6947}
e8e26350 6948
07923c17
ET		 6949static bool ixgbe_ring_tx_pending(struct ixgbe_adapter *adapter)
6950{
6951 int i;
6952
6953 for (i = 0; i < adapter->num_tx_queues; i++) {
6954 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
6955
6956 if (tx_ring->next_to_use != tx_ring->next_to_clean)
6957 return true;
6958 }
6959
6960 return false;
6961}
6962
6963static bool ixgbe_vf_tx_pending(struct ixgbe_adapter *adapter)
6964{
6965 struct ixgbe_hw *hw = &adapter->hw;
6966 struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
6967 u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask);
6968
6969 int i, j;
6970
6971 if (!adapter->num_vfs)
6972 return false;
6973
9a75a1ac
DS		 6974 /* resetting the PF is only needed for MAC before X550 */
6975 if (hw->mac.type >= ixgbe_mac_X550)
6976 return false;
6977
07923c17
ET		 6978 for (i = 0; i < adapter->num_vfs; i++) {
6979 for (j = 0; j < q_per_pool; j++) {
6980 u32 h, t;
6981
6982 h = IXGBE_READ_REG(hw, IXGBE_PVFTDHN(q_per_pool, i, j));
6983 t = IXGBE_READ_REG(hw, IXGBE_PVFTDTN(q_per_pool, i, j));
6984
6985 if (h != t)
6986 return true;
6987 }
6988 }
6989
6990 return false;
6991}
6992
93c52dd0
AD		 6993/**
6994 * ixgbe_watchdog_flush_tx - flush queues on link down
49ce9c2c 6995 * @adapter: pointer to the device adapter structure
93c52dd0
AD		 6996 **/
6997static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
6998{
93c52dd0 6999 if (!netif_carrier_ok(adapter->netdev)) {
07923c17
ET		 7000 if (ixgbe_ring_tx_pending(adapter) ||
7001 ixgbe_vf_tx_pending(adapter)) {
bc59fcda
NS		 7002 /* We've lost link, so the controller stops DMA,
7003 * but we've got queued Tx work that's never going
7004 * to get done, so reset controller to flush Tx.
7005 * (Do the reset outside of interrupt context).
7006 */
12ff3f3b 7007 e_warn(drv, "initiating reset to clear Tx work after link loss\n");
57ca2a4f 7008 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);
bc59fcda 7009 }
c4cf55e5 7010 }
c4cf55e5
PWJ		 7011}
7012
9079e416
ET		 7013#ifdef CONFIG_PCI_IOV
7014static inline void ixgbe_issue_vf_flr(struct ixgbe_adapter *adapter,
7015 struct pci_dev *vfdev)
7016{
7017 if (!pci_wait_for_pending_transaction(vfdev))
7018 e_dev_warn("Issuing VFLR with pending transactions\n");
7019
7020 e_dev_err("Issuing VFLR for VF %s\n", pci_name(vfdev));
7021 pcie_capability_set_word(vfdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
7022
7023 msleep(100);
7024}
7025
7026static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
7027{
7028 struct ixgbe_hw *hw = &adapter->hw;
7029 struct pci_dev *pdev = adapter->pdev;
988d1307 7030 unsigned int vf;
9079e416 7031 u32 gpc;
9079e416
ET		 7032
7033 if (!(netif_carrier_ok(adapter->netdev)))
7034 return;
7035
7036 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
7037 if (gpc) /* If incrementing then no need for the check below */
7038 return;
7039 /* Check to see if a bad DMA write target from an errant or
7040 * malicious VF has caused a PCIe error. If so then we can
7041 * issue a VFLR to the offending VF(s) and then resume without
7042 * requesting a full slot reset.
7043 */
7044
7045 if (!pdev)
7046 return;
7047
9079e416 7048 /* check status reg for all VFs owned by this PF */
988d1307
MR		 7049 for (vf = 0; vf < adapter->num_vfs; ++vf) {
7050 struct pci_dev *vfdev = adapter->vfinfo[vf].vfdev;
7051 u16 status_reg;
9079e416 7052
988d1307
MR		 7053 if (!vfdev)
7054 continue;
7055 pci_read_config_word(vfdev, PCI_STATUS, &status_reg);
7056 if (status_reg != IXGBE_FAILED_READ_CFG_WORD &&
7057 status_reg & PCI_STATUS_REC_MASTER_ABORT)
7058 ixgbe_issue_vf_flr(adapter, vfdev);
9079e416
ET		 7059 }
7060}
7061
a985b6c3
GR		 7062static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
7063{
7064 u32 ssvpc;
7065
0584d999
GR		 7066 /* Do not perform spoof check for 82598 or if not in IOV mode */
7067 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
7068 adapter->num_vfs == 0)
a985b6c3
GR		 7069 return;
7070
7071 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
7072
7073 /*
7074 * ssvpc register is cleared on read, if zero then no
7075 * spoofed packets in the last interval.
7076 */
7077 if (!ssvpc)
7078 return;
7079
d6ea0754 7080 e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
a985b6c3 7081}
9079e416
ET		 7082#else
7083static void ixgbe_spoof_check(struct ixgbe_adapter __always_unused *adapter)
7084{
7085}
7086
7087static void
7088ixgbe_check_for_bad_vf(struct ixgbe_adapter __always_unused *adapter)
7089{
7090}
7091#endif /* CONFIG_PCI_IOV */
7092
a985b6c3 7093
93c52dd0
AD		 7094/**
7095 * ixgbe_watchdog_subtask - check and bring link up
49ce9c2c 7096 * @adapter: pointer to the device adapter structure
93c52dd0
AD		 7097 **/
7098static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
7099{
09f40aed 7100 /* if interface is down, removing or resetting, do nothing */
7edebf9a 7101 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
09f40aed 7102 test_bit(__IXGBE_REMOVING, &adapter->state) ||
7edebf9a 7103 test_bit(__IXGBE_RESETTING, &adapter->state))
93c52dd0
AD		 7104 return;
7105
7106 ixgbe_watchdog_update_link(adapter);
7107
7108 if (adapter->link_up)
7109 ixgbe_watchdog_link_is_up(adapter);
7110 else
7111 ixgbe_watchdog_link_is_down(adapter);
bc59fcda 7112
9079e416 7113 ixgbe_check_for_bad_vf(adapter);
a985b6c3 7114 ixgbe_spoof_check(adapter);
9a799d71 7115 ixgbe_update_stats(adapter);
93c52dd0
AD		 7116
7117 ixgbe_watchdog_flush_tx(adapter);
9a799d71 7118}
10eec955 7119
cf8280ee 7120/**
7086400d 7121 * ixgbe_sfp_detection_subtask - poll for SFP+ cable
49ce9c2c 7122 * @adapter: the ixgbe adapter structure
cf8280ee 7123 **/
7086400d 7124static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
cf8280ee 7125{
cf8280ee 7126 struct ixgbe_hw *hw = &adapter->hw;
7086400d 7127 s32 err;
cf8280ee 7128
7086400d
AD		 7129 /* not searching for SFP so there is nothing to do here */
7130 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
7131 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
7132 return;
10eec955 7133
58e7cd24
MR		 7134 if (adapter->sfp_poll_time &&
7135 time_after(adapter->sfp_poll_time, jiffies))
7136 return; /* If not yet time to poll for SFP */
7137
7086400d
AD		 7138 /* someone else is in init, wait until next service event */
7139 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
7140 return;
cf8280ee 7141
58e7cd24
MR		 7142 adapter->sfp_poll_time = jiffies + IXGBE_SFP_POLL_JIFFIES - 1;
7143
7086400d
AD		 7144 err = hw->phy.ops.identify_sfp(hw);
7145 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
7146 goto sfp_out;
264857b8 7147
7086400d
AD		 7148 if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
7149 /* If no cable is present, then we need to reset
7150 * the next time we find a good cable. */
7151 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
cf8280ee 7152 }
9a799d71 7153
7086400d
AD		 7154 /* exit on error */
7155 if (err)
7156 goto sfp_out;
e8e26350 7157
7086400d
AD		 7158 /* exit if reset not needed */
7159 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
7160 goto sfp_out;
9a799d71 7161
7086400d 7162 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
bc59fcda 7163
7086400d
AD		 7164 /*
7165 * A module may be identified correctly, but the EEPROM may not have
7166 * support for that module. setup_sfp() will fail in that case, so
7167 * we should not allow that module to load.
7168 */
7169 if (hw->mac.type == ixgbe_mac_82598EB)
7170 err = hw->phy.ops.reset(hw);
7171 else
7172 err = hw->mac.ops.setup_sfp(hw);
7173
7174 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
7175 goto sfp_out;
7176
7177 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
7178 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
7179
7180sfp_out:
7181 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
7182
7183 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
7184 (adapter->netdev->reg_state == NETREG_REGISTERED)) {
7185 e_dev_err("failed to initialize because an unsupported "
7186 "SFP+ module type was detected.\n");
7187 e_dev_err("Reload the driver after installing a "
7188 "supported module.\n");
7189 unregister_netdev(adapter->netdev);
bc59fcda 7190 }
7086400d 7191}
bc59fcda 7192
7086400d
AD		 7193/**
7194 * ixgbe_sfp_link_config_subtask - set up link SFP after module install
49ce9c2c 7195 * @adapter: the ixgbe adapter structure
7086400d
AD		 7196 **/
7197static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
7198{
7199 struct ixgbe_hw *hw = &adapter->hw;
3d292265
JH		 7200 u32 speed;
7201 bool autoneg = false;
7086400d
AD		 7202
7203 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
7204 return;
7205
7206 /* someone else is in init, wait until next service event */
7207 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
7208 return;
7209
7210 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
7211
3d292265 7212 speed = hw->phy.autoneg_advertised;
ed33ff66 7213 if ((!speed) && (hw->mac.ops.get_link_capabilities)) {
3d292265 7214 hw->mac.ops.get_link_capabilities(hw, &speed, &autoneg);
ed33ff66
ET		 7215
7216 /* setup the highest link when no autoneg */
7217 if (!autoneg) {
7218 if (speed & IXGBE_LINK_SPEED_10GB_FULL)
7219 speed = IXGBE_LINK_SPEED_10GB_FULL;
7220 }
7221 }
7222
7086400d 7223 if (hw->mac.ops.setup_link)
fd0326f2 7224 hw->mac.ops.setup_link(hw, speed, true);
7086400d
AD		 7225
7226 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
7227 adapter->link_check_timeout = jiffies;
7228 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
7229}
7230
7231/**
7232 * ixgbe_service_timer - Timer Call-back
7233 * @data: pointer to adapter cast into an unsigned long
7234 **/
7235static void ixgbe_service_timer(unsigned long data)
7236{
7237 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
7238 unsigned long next_event_offset;
7239
6bb78cfb
AD		 7240 /* poll faster when waiting for link */
7241 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
7242 next_event_offset = HZ / 10;
7243 else
7244 next_event_offset = HZ * 2;
83c61fa9 7245
7086400d
AD		 7246 /* Reset the timer */
7247 mod_timer(&adapter->service_timer, next_event_offset + jiffies);
7248
9079e416 7249 ixgbe_service_event_schedule(adapter);
7086400d
AD		 7250}
7251
597f22d6
DS		 7252static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter)
7253{
7254 struct ixgbe_hw *hw = &adapter->hw;
7255 u32 status;
7256
7257 if (!(adapter->flags2 & IXGBE_FLAG2_PHY_INTERRUPT))
7258 return;
7259
7260 adapter->flags2 &= ~IXGBE_FLAG2_PHY_INTERRUPT;
7261
7262 if (!hw->phy.ops.handle_lasi)
7263 return;
7264
7265 status = hw->phy.ops.handle_lasi(&adapter->hw);
7266 if (status != IXGBE_ERR_OVERTEMP)
7267 return;
7268
7269 e_crit(drv, "%s\n", ixgbe_overheat_msg);
7270}
7271
c83c6cbd
AD		 7272static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
7273{
57ca2a4f 7274 if (!test_and_clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state))
c83c6cbd
AD		 7275 return;
7276
09f40aed 7277 /* If we're already down, removing or resetting, just bail */
c83c6cbd 7278 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
09f40aed 7279 test_bit(__IXGBE_REMOVING, &adapter->state) ||
c83c6cbd
AD		 7280 test_bit(__IXGBE_RESETTING, &adapter->state))
7281 return;
7282
7283 ixgbe_dump(adapter);
7284 netdev_err(adapter->netdev, "Reset adapter\n");
7285 adapter->tx_timeout_count++;
7286
8f4c5c9f 7287 rtnl_lock();
c83c6cbd 7288 ixgbe_reinit_locked(adapter);
8f4c5c9f 7289 rtnl_unlock();
c83c6cbd
AD		 7290}
7291
7086400d
AD		 7292/**
7293 * ixgbe_service_task - manages and runs subtasks
7294 * @work: pointer to work_struct containing our data
7295 **/
7296static void ixgbe_service_task(struct work_struct *work)
7297{
7298 struct ixgbe_adapter *adapter = container_of(work,
7299 struct ixgbe_adapter,
7300 service_task);
b0483c8f
MR		 7301 if (ixgbe_removed(adapter->hw.hw_addr)) {
7302 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
7303 rtnl_lock();
7304 ixgbe_down(adapter);
7305 rtnl_unlock();
7306 }
7307 ixgbe_service_event_complete(adapter);
7308 return;
7309 }
a21d0822 7310 if (adapter->flags2 & IXGBE_FLAG2_UDP_TUN_REREG_NEEDED) {
b3a49557 7311 rtnl_lock();
a21d0822 7312 adapter->flags2 &= ~IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
b3a49557
AD		 7313 udp_tunnel_get_rx_info(adapter->netdev);
7314 rtnl_unlock();
67359c3c 7315 }
c83c6cbd 7316 ixgbe_reset_subtask(adapter);
597f22d6 7317 ixgbe_phy_interrupt_subtask(adapter);
7086400d
AD		 7318 ixgbe_sfp_detection_subtask(adapter);
7319 ixgbe_sfp_link_config_subtask(adapter);
f0f9778d 7320 ixgbe_check_overtemp_subtask(adapter);
93c52dd0 7321 ixgbe_watchdog_subtask(adapter);
d034acf1 7322 ixgbe_fdir_reinit_subtask(adapter);
93c52dd0 7323 ixgbe_check_hang_subtask(adapter);
891dc082 7324
8fecf67c 7325 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) {
891dc082
JK		 7326 ixgbe_ptp_overflow_check(adapter);
7327 ixgbe_ptp_rx_hang(adapter);
7328 }
7086400d
AD		 7329
7330 ixgbe_service_event_complete(adapter);
9a799d71
AK		 7331}
7332
fd0db0ed
AD		 7333static int ixgbe_tso(struct ixgbe_ring *tx_ring,
7334 struct ixgbe_tx_buffer *first,
244e27ad 7335 u8 *hdr_len)
897ab156 7336{
b83e3010 7337 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
fd0db0ed 7338 struct sk_buff *skb = first->skb;
b83e3010
AD		 7339 union {
7340 struct iphdr *v4;
7341 struct ipv6hdr *v6;
7342 unsigned char *hdr;
7343 } ip;
7344 union {
7345 struct tcphdr *tcp;
7346 unsigned char *hdr;
7347 } l4;
7348 u32 paylen, l4_offset;
2049e1f6 7349 int err;
9a799d71 7350
8f4fbb9b
AD		 7351 if (skb->ip_summed != CHECKSUM_PARTIAL)
7352 return 0;
7353
897ab156
AD		 7354 if (!skb_is_gso(skb))
7355 return 0;
9a799d71 7356
2049e1f6
FR		 7357 err = skb_cow_head(skb, 0);
7358 if (err < 0)
7359 return err;
9a799d71 7360
b83e3010
AD		 7361 ip.hdr = skb_network_header(skb);
7362 l4.hdr = skb_checksum_start(skb);
7363
897ab156
AD		 7364 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
7365 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
7366
b83e3010
AD		 7367 /* initialize outer IP header fields */
7368 if (ip.v4->version == 4) {
c54cdc31
AD		 7369 unsigned char *csum_start = skb_checksum_start(skb);
7370 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
7371
b83e3010
AD		 7372 /* IP header will have to cancel out any data that
7373 * is not a part of the outer IP header
7374 */
c54cdc31
AD		 7375 ip.v4->check = csum_fold(csum_partial(trans_start,
7376 csum_start - trans_start,
7377 0));
897ab156 7378 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
b83e3010
AD		 7379
7380 ip.v4->tot_len = 0;
244e27ad
AD		 7381 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
7382 IXGBE_TX_FLAGS_CSUM |
7383 IXGBE_TX_FLAGS_IPV4;
b83e3010
AD		 7384 } else {
7385 ip.v6->payload_len = 0;
244e27ad
AD		 7386 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
7387 IXGBE_TX_FLAGS_CSUM;
897ab156
AD		 7388 }
7389
b83e3010
AD		 7390 /* determine offset of inner transport header */
7391 l4_offset = l4.hdr - skb->data;
7392
7393 /* compute length of segmentation header */
7394 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
7395
7396 /* remove payload length from inner checksum */
7397 paylen = skb->len - l4_offset;
7398 csum_replace_by_diff(&l4.tcp->check, htonl(paylen));
897ab156 7399
091a6246
AD		 7400 /* update gso size and bytecount with header size */
7401 first->gso_segs = skb_shinfo(skb)->gso_segs;
7402 first->bytecount += (first->gso_segs - 1) * *hdr_len;
7403
c44f5f51 7404 /* mss_l4len_id: use 0 as index for TSO */
b83e3010 7405 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;
897ab156 7406 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
897ab156
AD		 7407
7408 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
b83e3010
AD		 7409 vlan_macip_lens = l4.hdr - ip.hdr;
7410 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;
244e27ad 7411 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
897ab156
AD		 7412
7413 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
244e27ad 7414 mss_l4len_idx);
897ab156
AD		 7415
7416 return 1;
7417}
7418
49763de0
AD		 7419static inline bool ixgbe_ipv6_csum_is_sctp(struct sk_buff *skb)
7420{
7421 unsigned int offset = 0;
7422
7423 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL);
7424
7425 return offset == skb_checksum_start_offset(skb);
7426}
7427
244e27ad
AD		 7428static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
7429 struct ixgbe_tx_buffer *first)
7ca647bd 7430{
fd0db0ed 7431 struct sk_buff *skb = first->skb;
897ab156 7432 u32 vlan_macip_lens = 0;
897ab156 7433 u32 type_tucmd = 0;
7ca647bd 7434
897ab156 7435 if (skb->ip_summed != CHECKSUM_PARTIAL) {
49763de0
AD		 7436csum_failed:
7437 if (!(first->tx_flags & (IXGBE_TX_FLAGS_HW_VLAN |
7438 IXGBE_TX_FLAGS_CC)))
472148c3 7439 return;
49763de0
AD		 7440 goto no_csum;
7441 }
897ab156 7442
49763de0
AD		 7443 switch (skb->csum_offset) {
7444 case offsetof(struct tcphdr, check):
7445 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
7446 /* fall through */
7447 case offsetof(struct udphdr, check):
7448 break;
7449 case offsetof(struct sctphdr, checksum):
7450 /* validate that this is actually an SCTP request */
7451 if (((first->protocol == htons(ETH_P_IP)) &&
7452 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) ||
7453 ((first->protocol == htons(ETH_P_IPV6)) &&
7454 ixgbe_ipv6_csum_is_sctp(skb))) {
7455 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;
897ab156 7456 break;
7ca647bd 7457 }
49763de0
AD		 7458 /* fall through */
7459 default:
7460 skb_checksum_help(skb);
7461 goto csum_failed;
7ca647bd
JP		 7462 }
7463
49763de0
AD		 7464 /* update TX checksum flag */
7465 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
7466 vlan_macip_lens = skb_checksum_start_offset(skb) -
7467 skb_network_offset(skb);
36a92d71 7468no_csum:
244e27ad 7469 /* vlan_macip_lens: MACLEN, VLAN tag */
49763de0 7470 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
244e27ad 7471 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
9a799d71 7472
49763de0 7473 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 0);
9a799d71
AK		 7474}
7475
472148c3
AD		 7476#define IXGBE_SET_FLAG(_input, _flag, _result) \
7477 ((_flag <= _result) ? \
7478 ((u32)(_input & _flag) * (_result / _flag)) : \
7479 ((u32)(_input & _flag) / (_flag / _result)))
7480
7481static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
9a799d71 7482{
d3d00239 7483 /* set type for advanced descriptor with frame checksum insertion */
472148c3
AD		 7484 u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |
7485 IXGBE_ADVTXD_DCMD_DEXT |
7486 IXGBE_ADVTXD_DCMD_IFCS;
9a799d71 7487
d3d00239 7488 /* set HW vlan bit if vlan is present */
472148c3
AD		 7489 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN,
7490 IXGBE_ADVTXD_DCMD_VLE);
3a6a4eda 7491
d3d00239 7492 /* set segmentation enable bits for TSO/FSO */
472148c3
AD		 7493 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO,
7494 IXGBE_ADVTXD_DCMD_TSE);
7495
7496 /* set timestamp bit if present */
7497 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP,
7498 IXGBE_ADVTXD_MAC_TSTAMP);
eacd73f7 7499
62748b7b 7500 /* insert frame checksum */
472148c3 7501 cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS);
62748b7b 7502
d3d00239
AD		 7503 return cmd_type;
7504}
9a799d71 7505
729739b7
AD		 7506static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
7507 u32 tx_flags, unsigned int paylen)
d3d00239 7508{
472148c3 7509 u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
9a799d71 7510
d3d00239 7511 /* enable L4 checksum for TSO and TX checksum offload */
472148c3
AD		 7512 olinfo_status |= IXGBE_SET_FLAG(tx_flags,
7513 IXGBE_TX_FLAGS_CSUM,
7514 IXGBE_ADVTXD_POPTS_TXSM);
9a799d71 7515
93f5b3c1 7516 /* enble IPv4 checksum for TSO */
472148c3
AD		 7517 olinfo_status |= IXGBE_SET_FLAG(tx_flags,
7518 IXGBE_TX_FLAGS_IPV4,
7519 IXGBE_ADVTXD_POPTS_IXSM);
9a799d71 7520
7f9643fd
AD		 7521 /*
7522 * Check Context must be set if Tx switch is enabled, which it
7523 * always is for case where virtual functions are running
7524 */
472148c3
AD		 7525 olinfo_status |= IXGBE_SET_FLAG(tx_flags,
7526 IXGBE_TX_FLAGS_CC,
7527 IXGBE_ADVTXD_CC);
7f9643fd 7528
472148c3 7529 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
d3d00239 7530}
44df32c5 7531
2367a173
DB		 7532static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
7533{
7534 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
7535
7536 /* Herbert's original patch had:
7537 * smp_mb__after_netif_stop_queue();
7538 * but since that doesn't exist yet, just open code it.
7539 */
7540 smp_mb();
7541
7542 /* We need to check again in a case another CPU has just
7543 * made room available.
7544 */
7545 if (likely(ixgbe_desc_unused(tx_ring) < size))
7546 return -EBUSY;
7547
7548 /* A reprieve! - use start_queue because it doesn't call schedule */
7549 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
7550 ++tx_ring->tx_stats.restart_queue;
7551 return 0;
7552}
7553
7554static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
7555{
7556 if (likely(ixgbe_desc_unused(tx_ring) >= size))
7557 return 0;
7558
7559 return __ixgbe_maybe_stop_tx(tx_ring, size);
7560}
7561
d3d00239
AD		 7562#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \
7563 IXGBE_TXD_CMD_RS)
7564
7565static void ixgbe_tx_map(struct ixgbe_ring *tx_ring,
d3d00239 7566 struct ixgbe_tx_buffer *first,
d3d00239
AD		 7567 const u8 hdr_len)
7568{
fd0db0ed 7569 struct sk_buff *skb = first->skb;
729739b7 7570 struct ixgbe_tx_buffer *tx_buffer;
d3d00239 7571 union ixgbe_adv_tx_desc *tx_desc;
ec718254
AD		 7572 struct skb_frag_struct *frag;
7573 dma_addr_t dma;
7574 unsigned int data_len, size;
244e27ad 7575 u32 tx_flags = first->tx_flags;
472148c3 7576 u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags);
d3d00239 7577 u16 i = tx_ring->next_to_use;
d3d00239 7578
729739b7
AD		 7579 tx_desc = IXGBE_TX_DESC(tx_ring, i);
7580
ec718254
AD		 7581 ixgbe_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
7582
7583 size = skb_headlen(skb);
7584 data_len = skb->data_len;
729739b7 7585
d3d00239
AD		 7586#ifdef IXGBE_FCOE
7587 if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
729739b7 7588 if (data_len < sizeof(struct fcoe_crc_eof)) {
d3d00239
AD		 7589 size -= sizeof(struct fcoe_crc_eof) - data_len;
7590 data_len = 0;
729739b7
AD		 7591 } else {
7592 data_len -= sizeof(struct fcoe_crc_eof);
9a799d71
AK		 7593 }
7594 }
44df32c5 7595
d3d00239 7596#endif
729739b7 7597 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
8ad494b0 7598
ec718254 7599 tx_buffer = first;
9a799d71 7600
ec718254
AD		 7601 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
7602 if (dma_mapping_error(tx_ring->dev, dma))
7603 goto dma_error;
7604
7605 /* record length, and DMA address */
7606 dma_unmap_len_set(tx_buffer, len, size);
7607 dma_unmap_addr_set(tx_buffer, dma, dma);
7608
7609 tx_desc->read.buffer_addr = cpu_to_le64(dma);
e5a43549 7610
729739b7 7611 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
d3d00239 7612 tx_desc->read.cmd_type_len =
472148c3 7613 cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD);
e5a43549 7614
d3d00239 7615 i++;
729739b7 7616 tx_desc++;
d3d00239 7617 if (i == tx_ring->count) {
e4f74028 7618 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
d3d00239
AD		 7619 i = 0;
7620 }
ec718254 7621 tx_desc->read.olinfo_status = 0;
729739b7
AD		 7622
7623 dma += IXGBE_MAX_DATA_PER_TXD;
7624 size -= IXGBE_MAX_DATA_PER_TXD;
7625
7626 tx_desc->read.buffer_addr = cpu_to_le64(dma);
d3d00239 7627 }
e5a43549 7628
729739b7
AD		 7629 if (likely(!data_len))
7630 break;
9a799d71 7631
472148c3 7632 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
9a799d71 7633
729739b7
AD		 7634 i++;
7635 tx_desc++;
7636 if (i == tx_ring->count) {
7637 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
7638 i = 0;
7639 }
ec718254 7640 tx_desc->read.olinfo_status = 0;
9a799d71 7641
d3d00239 7642#ifdef IXGBE_FCOE
9e903e08 7643 size = min_t(unsigned int, data_len, skb_frag_size(frag));
d3d00239 7644#else
9e903e08 7645 size = skb_frag_size(frag);
d3d00239
AD		 7646#endif
7647 data_len -= size;
9a799d71 7648
729739b7
AD		 7649 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
7650 DMA_TO_DEVICE);
9a799d71 7651
729739b7 7652 tx_buffer = &tx_ring->tx_buffer_info[i];
729739b7 7653 }
9a799d71 7654
729739b7 7655 /* write last descriptor with RS and EOP bits */
472148c3
AD		 7656 cmd_type |= size | IXGBE_TXD_CMD;
7657 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
eacd73f7 7658
091a6246 7659 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
b2d96e0a 7660
d3d00239
AD		 7661 /* set the timestamp */
7662 first->time_stamp = jiffies;
9a799d71
AK		 7663
7664 /*
729739b7
AD		 7665 * Force memory writes to complete before letting h/w know there
7666 * are new descriptors to fetch. (Only applicable for weak-ordered
7667 * memory model archs, such as IA-64).
7668 *
7669 * We also need this memory barrier to make certain all of the
7670 * status bits have been updated before next_to_watch is written.
9a799d71
AK		 7671 */
7672 wmb();
7673
d3d00239
AD		 7674 /* set next_to_watch value indicating a packet is present */
7675 first->next_to_watch = tx_desc;
7676
729739b7
AD		 7677 i++;
7678 if (i == tx_ring->count)
7679 i = 0;
7680
7681 tx_ring->next_to_use = i;
7682
2367a173
DB		 7683 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
7684
7685 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
ad435ec6
AD		 7686 writel(i, tx_ring->tail);
7687
7688 /* we need this if more than one processor can write to our tail
7689 * at a time, it synchronizes IO on IA64/Altix systems
7690 */
7691 mmiowb();
9c938cdd 7692 }
2367a173 7693
d3d00239
AD		 7694 return;
7695dma_error:
729739b7 7696 dev_err(tx_ring->dev, "TX DMA map failed\n");
d3d00239
AD		 7697
7698 /* clear dma mappings for failed tx_buffer_info map */
7699 for (;;) {
729739b7
AD		 7700 tx_buffer = &tx_ring->tx_buffer_info[i];
7701 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer);
7702 if (tx_buffer == first)
d3d00239
AD		 7703 break;
7704 if (i == 0)
7705 i = tx_ring->count;
7706 i--;
7707 }
7708
d3d00239 7709 tx_ring->next_to_use = i;
9a799d71
AK		 7710}
7711
fd0db0ed 7712static void ixgbe_atr(struct ixgbe_ring *ring,
244e27ad 7713 struct ixgbe_tx_buffer *first)
69830529
AD		 7714{
7715 struct ixgbe_q_vector *q_vector = ring->q_vector;
7716 union ixgbe_atr_hash_dword input = { .dword = 0 };
7717 union ixgbe_atr_hash_dword common = { .dword = 0 };
7718 union {
7719 unsigned char *network;
7720 struct iphdr *ipv4;
7721 struct ipv6hdr *ipv6;
7722 } hdr;
ee9e0f0b 7723 struct tcphdr *th;
e2873d43 7724 unsigned int hlen;
67359c3c 7725 struct sk_buff *skb;
905e4a41 7726 __be16 vlan_id;
e2873d43 7727 int l4_proto;
c4cf55e5 7728
69830529
AD		 7729 /* if ring doesn't have a interrupt vector, cannot perform ATR */
7730 if (!q_vector)
7731 return;
7732
7733 /* do nothing if sampling is disabled */
7734 if (!ring->atr_sample_rate)
d3ead241 7735 return;
c4cf55e5 7736
69830529 7737 ring->atr_count++;
c4cf55e5 7738
e2873d43
AD		 7739 /* currently only IPv4/IPv6 with TCP is supported */
7740 if ((first->protocol != htons(ETH_P_IP)) &&
7741 (first->protocol != htons(ETH_P_IPV6)))
7742 return;
7743
69830529 7744 /* snag network header to get L4 type and address */
67359c3c
MR		 7745 skb = first->skb;
7746 hdr.network = skb_network_header(skb);
9f3c7504
SV		 7747 if (unlikely(hdr.network <= skb->data))
7748 return;
9f12df90
AD		 7749 if (skb->encapsulation &&
7750 first->protocol == htons(ETH_P_IP) &&
52028821 7751 hdr.ipv4->protocol == IPPROTO_UDP) {
67359c3c 7752 struct ixgbe_adapter *adapter = q_vector->adapter;
69830529 7753
9f3c7504
SV		 7754 if (unlikely(skb_tail_pointer(skb) < hdr.network +
7755 VXLAN_HEADROOM))
7756 return;
7757
9f12df90
AD		 7758 /* verify the port is recognized as VXLAN */
7759 if (adapter->vxlan_port &&
e2873d43 7760 udp_hdr(skb)->dest == adapter->vxlan_port)
9f12df90 7761 hdr.network = skb_inner_network_header(skb);
a21d0822
ET		 7762
7763 if (adapter->geneve_port &&
7764 udp_hdr(skb)->dest == adapter->geneve_port)
7765 hdr.network = skb_inner_network_header(skb);
e19dcdeb
MR		 7766 }
7767
9f3c7504
SV		 7768 /* Make sure we have at least [minimum IPv4 header + TCP]
7769 * or [IPv6 header] bytes
7770 */
7771 if (unlikely(skb_tail_pointer(skb) < hdr.network + 40))
7772 return;
7773
e19dcdeb
MR		 7774 /* Currently only IPv4/IPv6 with TCP is supported */
7775 switch (hdr.ipv4->version) {
7776 case IPVERSION:
e2873d43
AD		 7777 /* access ihl as u8 to avoid unaligned access on ia64 */
7778 hlen = (hdr.network[0] & 0x0F) << 2;
7779 l4_proto = hdr.ipv4->protocol;
e19dcdeb
MR		 7780 break;
7781 case 6:
e2873d43
AD		 7782 hlen = hdr.network - skb->data;
7783 l4_proto = ipv6_find_hdr(skb, &hlen, IPPROTO_TCP, NULL, NULL);
7784 hlen -= hdr.network - skb->data;
e19dcdeb
MR		 7785 break;
7786 default:
7787 return;
67359c3c 7788 }
c4cf55e5 7789
e2873d43
AD		 7790 if (l4_proto != IPPROTO_TCP)
7791 return;
7792
9f3c7504
SV		 7793 if (unlikely(skb_tail_pointer(skb) < hdr.network +
7794 hlen + sizeof(struct tcphdr)))
7795 return;
7796
e2873d43
AD		 7797 th = (struct tcphdr *)(hdr.network + hlen);
7798
7799 /* skip this packet since the socket is closing */
7800 if (th->fin)
69830529
AD		 7801 return;
7802
7803 /* sample on all syn packets or once every atr sample count */
7804 if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
7805 return;
7806
7807 /* reset sample count */
7808 ring->atr_count = 0;
7809
244e27ad 7810 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
69830529
AD		 7811
7812 /*
7813 * src and dst are inverted, think how the receiver sees them
7814 *
7815 * The input is broken into two sections, a non-compressed section
7816 * containing vm_pool, vlan_id, and flow_type. The rest of the data
7817 * is XORed together and stored in the compressed dword.
7818 */
7819 input.formatted.vlan_id = vlan_id;
7820
7821 /*
7822 * since src port and flex bytes occupy the same word XOR them together
7823 * and write the value to source port portion of compressed dword
7824 */
244e27ad 7825 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
a1108ffd 7826 common.port.src ^= th->dest ^ htons(ETH_P_8021Q);
69830529 7827 else
244e27ad 7828 common.port.src ^= th->dest ^ first->protocol;
69830529
AD		 7829 common.port.dst ^= th->source;
7830
e19dcdeb
MR		 7831 switch (hdr.ipv4->version) {
7832 case IPVERSION:
69830529
AD		 7833 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
7834 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
e19dcdeb
MR		 7835 break;
7836 case 6:
69830529
AD		 7837 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
7838 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
7839 hdr.ipv6->saddr.s6_addr32[1] ^
7840 hdr.ipv6->saddr.s6_addr32[2] ^
7841 hdr.ipv6->saddr.s6_addr32[3] ^
7842 hdr.ipv6->daddr.s6_addr32[0] ^
7843 hdr.ipv6->daddr.s6_addr32[1] ^
7844 hdr.ipv6->daddr.s6_addr32[2] ^
7845 hdr.ipv6->daddr.s6_addr32[3];
e19dcdeb
MR		 7846 break;
7847 default:
7848 break;
69830529 7849 }
c4cf55e5 7850
9f12df90 7851 if (hdr.network != skb_network_header(skb))
67359c3c 7852 input.formatted.flow_type |= IXGBE_ATR_L4TYPE_TUNNEL_MASK;
67359c3c 7853
c4cf55e5 7854 /* This assumes the Rx queue and Tx queue are bound to the same CPU */
69830529
AD		 7855 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
7856 input, common, ring->queue_index);
c4cf55e5
PWJ		 7857}
7858
f663dd9a 7859static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
99932d4f 7860 void *accel_priv, select_queue_fallback_t fallback)
09a3b1f8 7861{
f663dd9a
JW		 7862 struct ixgbe_fwd_adapter *fwd_adapter = accel_priv;
7863#ifdef IXGBE_FCOE
97488bd1
AD		 7864 struct ixgbe_adapter *adapter;
7865 struct ixgbe_ring_feature *f;
7866 int txq;
f663dd9a
JW		 7867#endif
7868
7869 if (fwd_adapter)
7870 return skb->queue_mapping + fwd_adapter->tx_base_queue;
7871
7872#ifdef IXGBE_FCOE
5e09a105 7873
97488bd1
AD		 7874 /*
7875 * only execute the code below if protocol is FCoE
7876 * or FIP and we have FCoE enabled on the adapter
7877 */
7878 switch (vlan_get_protocol(skb)) {
a1108ffd
JP		 7879 case htons(ETH_P_FCOE):
7880 case htons(ETH_P_FIP):
97488bd1 7881 adapter = netdev_priv(dev);
c087663e 7882
97488bd1
AD		 7883 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
7884 break;
7885 default:
99932d4f 7886 return fallback(dev, skb);
97488bd1 7887 }
c087663e 7888
97488bd1 7889 f = &adapter->ring_feature[RING_F_FCOE];
c087663e 7890
97488bd1
AD		 7891 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
7892 smp_processor_id();
56075a98 7893
97488bd1
AD		 7894 while (txq >= f->indices)
7895 txq -= f->indices;
c4cf55e5 7896
97488bd1 7897 return txq + f->offset;
f663dd9a 7898#else
99932d4f 7899 return fallback(dev, skb);
f663dd9a 7900#endif
09a3b1f8
SH		 7901}
7902
fc77dc3c 7903netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
84418e3b
AD		 7904 struct ixgbe_adapter *adapter,
7905 struct ixgbe_ring *tx_ring)
9a799d71 7906{
d3d00239 7907 struct ixgbe_tx_buffer *first;
5f715823 7908 int tso;
d3d00239 7909 u32 tx_flags = 0;
a535c30e 7910 unsigned short f;
a535c30e 7911 u16 count = TXD_USE_COUNT(skb_headlen(skb));
66f32a8b 7912 __be16 protocol = skb->protocol;
63544e9c 7913 u8 hdr_len = 0;
5e09a105 7914
a535c30e
AD		 7915 /*
7916 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
24ddd967 7917 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
a535c30e
AD		 7918 * + 2 desc gap to keep tail from touching head,
7919 * + 1 desc for context descriptor,
7920 * otherwise try next time
7921 */
a535c30e
AD		 7922 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
7923 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
7f66162b 7924
a535c30e
AD		 7925 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
7926 tx_ring->tx_stats.tx_busy++;
7927 return NETDEV_TX_BUSY;
7928 }
7929
fd0db0ed
AD		 7930 /* record the location of the first descriptor for this packet */
7931 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
7932 first->skb = skb;
091a6246
AD		 7933 first->bytecount = skb->len;
7934 first->gso_segs = 1;
fd0db0ed 7935
66f32a8b 7936 /* if we have a HW VLAN tag being added default to the HW one */
df8a39de
JP		 7937 if (skb_vlan_tag_present(skb)) {
7938 tx_flags |= skb_vlan_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
66f32a8b
AD		 7939 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
7940 /* else if it is a SW VLAN check the next protocol and store the tag */
a1108ffd 7941 } else if (protocol == htons(ETH_P_8021Q)) {
66f32a8b
AD		 7942 struct vlan_hdr *vhdr, _vhdr;
7943 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
7944 if (!vhdr)
7945 goto out_drop;
7946
9e0c5648
AD		 7947 tx_flags |= ntohs(vhdr->h_vlan_TCI) <<
7948 IXGBE_TX_FLAGS_VLAN_SHIFT;
66f32a8b
AD		 7949 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
7950 }
0213668f 7951 protocol = vlan_get_protocol(skb);
66f32a8b 7952
d5234933
MR		 7953 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
7954 adapter->ptp_clock &&
7955 !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS,
7956 &adapter->state)) {
3a6a4eda
JK		 7957 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
7958 tx_flags |= IXGBE_TX_FLAGS_TSTAMP;
891dc082
JK		 7959
7960 /* schedule check for Tx timestamp */
7961 adapter->ptp_tx_skb = skb_get(skb);
7962 adapter->ptp_tx_start = jiffies;
7963 schedule_work(&adapter->ptp_tx_work);
3a6a4eda 7964 }
3a6a4eda 7965
ff29a86e
JK		 7966 skb_tx_timestamp(skb);
7967
9e0c5648
AD		 7968#ifdef CONFIG_PCI_IOV
7969 /*
7970 * Use the l2switch_enable flag - would be false if the DMA
7971 * Tx switch had been disabled.
7972 */
7973 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
472148c3 7974 tx_flags |= IXGBE_TX_FLAGS_CC;
9e0c5648
AD		 7975
7976#endif
32701dc2 7977 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
66f32a8b 7978 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
09dca476
AD		 7979 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
7980 (skb->priority != TC_PRIO_CONTROL))) {
66f32a8b 7981 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
32701dc2
JF		 7982 tx_flags |= (skb->priority & 0x7) <<
7983 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
66f32a8b
AD		 7984 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
7985 struct vlan_ethhdr *vhdr;
2049e1f6
FR		 7986
7987 if (skb_cow_head(skb, 0))
66f32a8b
AD		 7988 goto out_drop;
7989 vhdr = (struct vlan_ethhdr *)skb->data;
7990 vhdr->h_vlan_TCI = htons(tx_flags >>
7991 IXGBE_TX_FLAGS_VLAN_SHIFT);
7992 } else {
7993 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
2f90b865 7994 }
9a799d71 7995 }
eacd73f7 7996
244e27ad
AD		 7997 /* record initial flags and protocol */
7998 first->tx_flags = tx_flags;
7999 first->protocol = protocol;
8000
eacd73f7 8001#ifdef IXGBE_FCOE
66f32a8b 8002 /* setup tx offload for FCoE */
a1108ffd 8003 if ((protocol == htons(ETH_P_FCOE)) &&
a58915c7 8004 (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) {
244e27ad 8005 tso = ixgbe_fso(tx_ring, first, &hdr_len);
897ab156
AD		 8006 if (tso < 0)
8007 goto out_drop;
9a799d71 8008
66f32a8b 8009 goto xmit_fcoe;
eacd73f7 8010 }
9a799d71 8011
66f32a8b 8012#endif /* IXGBE_FCOE */
244e27ad 8013 tso = ixgbe_tso(tx_ring, first, &hdr_len);
66f32a8b 8014 if (tso < 0)
897ab156 8015 goto out_drop;
244e27ad
AD		 8016 else if (!tso)
8017 ixgbe_tx_csum(tx_ring, first);
66f32a8b
AD		 8018
8019 /* add the ATR filter if ATR is on */
8020 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
244e27ad 8021 ixgbe_atr(tx_ring, first);
66f32a8b
AD		 8022
8023#ifdef IXGBE_FCOE
8024xmit_fcoe:
8025#endif /* IXGBE_FCOE */
244e27ad 8026 ixgbe_tx_map(tx_ring, first, hdr_len);
d3d00239 8027
9a799d71 8028 return NETDEV_TX_OK;
897ab156
AD		 8029
8030out_drop:
fd0db0ed
AD		 8031 dev_kfree_skb_any(first->skb);
8032 first->skb = NULL;
8033
897ab156 8034 return NETDEV_TX_OK;
9a799d71
AK		 8035}
8036
2a47fa45
JF		 8037static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,
8038 struct net_device *netdev,
8039 struct ixgbe_ring *ring)
84418e3b
AD		 8040{
8041 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8042 struct ixgbe_ring *tx_ring;
8043
a50c29dd
AD		 8044 /*
8045 * The minimum packet size for olinfo paylen is 17 so pad the skb
8046 * in order to meet this minimum size requirement.
8047 */
a94d9e22
AD		 8048 if (skb_put_padto(skb, 17))
8049 return NETDEV_TX_OK;
a50c29dd 8050
2a47fa45
JF		 8051 tx_ring = ring ? ring : adapter->tx_ring[skb->queue_mapping];
8052
fc77dc3c 8053 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
84418e3b
AD		 8054}
8055
2a47fa45
JF		 8056static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
8057 struct net_device *netdev)
8058{
8059 return __ixgbe_xmit_frame(skb, netdev, NULL);
8060}
8061
9a799d71
AK		 8062/**
8063 * ixgbe_set_mac - Change the Ethernet Address of the NIC
8064 * @netdev: network interface device structure
8065 * @p: pointer to an address structure
8066 *
8067 * Returns 0 on success, negative on failure
8068 **/
8069static int ixgbe_set_mac(struct net_device *netdev, void *p)
8070{
8071 struct ixgbe_adapter *adapter = netdev_priv(netdev);
b4617240 8072 struct ixgbe_hw *hw = &adapter->hw;
9a799d71
AK		 8073 struct sockaddr *addr = p;
8074
8075 if (!is_valid_ether_addr(addr->sa_data))
8076 return -EADDRNOTAVAIL;
8077
8078 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
b4617240 8079 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9a799d71 8080
c9f53e63
AD		 8081 ixgbe_mac_set_default_filter(adapter);
8082
8083 return 0;
9a799d71
AK		 8084}
8085
6b73e10d
BH		 8086static int
8087ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
8088{
8089 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8090 struct ixgbe_hw *hw = &adapter->hw;
8091 u16 value;
8092 int rc;
8093
8094 if (prtad != hw->phy.mdio.prtad)
8095 return -EINVAL;
8096 rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
8097 if (!rc)
8098 rc = value;
8099 return rc;
8100}
8101
8102static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
8103 u16 addr, u16 value)
8104{
8105 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8106 struct ixgbe_hw *hw = &adapter->hw;
8107
8108 if (prtad != hw->phy.mdio.prtad)
8109 return -EINVAL;
8110 return hw->phy.ops.write_reg(hw, addr, devad, value);
8111}
8112
8113static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
8114{
8115 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8116
3a6a4eda 8117 switch (cmd) {
3a6a4eda 8118 case SIOCSHWTSTAMP:
93501d48
JK		 8119 return ixgbe_ptp_set_ts_config(adapter, req);
8120 case SIOCGHWTSTAMP:
8121 return ixgbe_ptp_get_ts_config(adapter, req);
3a6a4eda
JK		 8122 default:
8123 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
8124 }
6b73e10d
BH		 8125}
8126
0365e6e4
PW		 8127/**
8128 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
31278e71 8129 * netdev->dev_addrs
0365e6e4
PW		 8130 * @netdev: network interface device structure
8131 *
8132 * Returns non-zero on failure
8133 **/
8134static int ixgbe_add_sanmac_netdev(struct net_device *dev)
8135{
8136 int err = 0;
8137 struct ixgbe_adapter *adapter = netdev_priv(dev);
7fa7c9dc 8138 struct ixgbe_hw *hw = &adapter->hw;
0365e6e4 8139
7fa7c9dc 8140 if (is_valid_ether_addr(hw->mac.san_addr)) {
0365e6e4 8141 rtnl_lock();
7fa7c9dc 8142 err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN);
0365e6e4 8143 rtnl_unlock();
7fa7c9dc
AD		 8144
8145 /* update SAN MAC vmdq pool selection */
8146 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
0365e6e4
PW		 8147 }
8148 return err;
8149}
8150
8151/**
8152 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
31278e71 8153 * netdev->dev_addrs
0365e6e4
PW		 8154 * @netdev: network interface device structure
8155 *
8156 * Returns non-zero on failure
8157 **/
8158static int ixgbe_del_sanmac_netdev(struct net_device *dev)
8159{
8160 int err = 0;
8161 struct ixgbe_adapter *adapter = netdev_priv(dev);
8162 struct ixgbe_mac_info *mac = &adapter->hw.mac;
8163
8164 if (is_valid_ether_addr(mac->san_addr)) {
8165 rtnl_lock();
8166 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
8167 rtnl_unlock();
8168 }
8169 return err;
8170}
8171
9a799d71
AK		 8172#ifdef CONFIG_NET_POLL_CONTROLLER
8173/*
8174 * Polling 'interrupt' - used by things like netconsole to send skbs
8175 * without having to re-enable interrupts. It's not called while
8176 * the interrupt routine is executing.
8177 */
8178static void ixgbe_netpoll(struct net_device *netdev)
8179{
8180 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8f9a7167 8181 int i;
9a799d71 8182
1a647bd2
AD		 8183 /* if interface is down do nothing */
8184 if (test_bit(__IXGBE_DOWN, &adapter->state))
8185 return;
8186
856f606e
AD		 8187 /* loop through and schedule all active queues */
8188 for (i = 0; i < adapter->num_q_vectors; i++)
8189 ixgbe_msix_clean_rings(0, adapter->q_vector[i]);
9a799d71 8190}
9a799d71 8191
581330ba 8192#endif
bc1f4470 8193
8194static void ixgbe_get_stats64(struct net_device *netdev,
8195 struct rtnl_link_stats64 *stats)
de1036b1
ED		 8196{
8197 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8198 int i;
8199
1a51502b 8200 rcu_read_lock();
de1036b1 8201 for (i = 0; i < adapter->num_rx_queues; i++) {
1a51502b 8202 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
de1036b1
ED		 8203 u64 bytes, packets;
8204 unsigned int start;
8205
1a51502b
ED		 8206 if (ring) {
8207 do {
57a7744e 8208 start = u64_stats_fetch_begin_irq(&ring->syncp);
1a51502b
ED		 8209 packets = ring->stats.packets;
8210 bytes = ring->stats.bytes;
57a7744e 8211 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1a51502b
ED		 8212 stats->rx_packets += packets;
8213 stats->rx_bytes += bytes;
8214 }
de1036b1 8215 }
1ac9ad13
ED		 8216
8217 for (i = 0; i < adapter->num_tx_queues; i++) {
8218 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
8219 u64 bytes, packets;
8220 unsigned int start;
8221
8222 if (ring) {
8223 do {
57a7744e 8224 start = u64_stats_fetch_begin_irq(&ring->syncp);
1ac9ad13
ED		 8225 packets = ring->stats.packets;
8226 bytes = ring->stats.bytes;
57a7744e 8227 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1ac9ad13
ED		 8228 stats->tx_packets += packets;
8229 stats->tx_bytes += bytes;
8230 }
8231 }
1a51502b 8232 rcu_read_unlock();
bc1f4470 8233
de1036b1
ED		 8234 /* following stats updated by ixgbe_watchdog_task() */
8235 stats->multicast = netdev->stats.multicast;
8236 stats->rx_errors = netdev->stats.rx_errors;
8237 stats->rx_length_errors = netdev->stats.rx_length_errors;
8238 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
8239 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
de1036b1
ED		 8240}
8241
8af3c33f 8242#ifdef CONFIG_IXGBE_DCB
49ce9c2c
BH		 8243/**
8244 * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
8245 * @adapter: pointer to ixgbe_adapter
8b1c0b24
JF		 8246 * @tc: number of traffic classes currently enabled
8247 *
8248 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
8249 * 802.1Q priority maps to a packet buffer that exists.
8250 */
8251static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
8252{
8253 struct ixgbe_hw *hw = &adapter->hw;
8254 u32 reg, rsave;
8255 int i;
8256
8257 /* 82598 have a static priority to TC mapping that can not
8258 * be changed so no validation is needed.
8259 */
8260 if (hw->mac.type == ixgbe_mac_82598EB)
8261 return;
8262
8263 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
8264 rsave = reg;
8265
8266 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
8267 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
8268
8269 /* If up2tc is out of bounds default to zero */
8270 if (up2tc > tc)
8271 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
8272 }
8273
8274 if (reg != rsave)
8275 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
8276
8277 return;
8278}
8279
02debdc9
AD		 8280/**
8281 * ixgbe_set_prio_tc_map - Configure netdev prio tc map
8282 * @adapter: Pointer to adapter struct
8283 *
8284 * Populate the netdev user priority to tc map
8285 */
8286static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
8287{
8288 struct net_device *dev = adapter->netdev;
8289 struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg;
8290 struct ieee_ets *ets = adapter->ixgbe_ieee_ets;
8291 u8 prio;
8292
8293 for (prio = 0; prio < MAX_USER_PRIORITY; prio++) {
8294 u8 tc = 0;
8295
8296 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE)
8297 tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio);
8298 else if (ets)
8299 tc = ets->prio_tc[prio];
8300
8301 netdev_set_prio_tc_map(dev, prio, tc);
8302 }
8303}
8304
cca73c59 8305#endif /* CONFIG_IXGBE_DCB */
49ce9c2c
BH		 8306/**
8307 * ixgbe_setup_tc - configure net_device for multiple traffic classes
8b1c0b24
JF		 8308 *
8309 * @netdev: net device to configure
8310 * @tc: number of traffic classes to enable
8311 */
8312int ixgbe_setup_tc(struct net_device *dev, u8 tc)
8313{
8b1c0b24
JF		 8314 struct ixgbe_adapter *adapter = netdev_priv(dev);
8315 struct ixgbe_hw *hw = &adapter->hw;
2a47fa45 8316 bool pools;
8b1c0b24 8317
8b1c0b24 8318 /* Hardware supports up to 8 traffic classes */
7e3f5c88
ET		 8319 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs)
8320 return -EINVAL;
8321
8322 if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS)
8b1c0b24
JF		 8323 return -EINVAL;
8324
2a47fa45
JF		 8325 pools = (find_first_zero_bit(&adapter->fwd_bitmask, 32) > 1);
8326 if (tc && pools && adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS)
8327 return -EBUSY;
8328
8b1c0b24 8329 /* Hardware has to reinitialize queues and interrupts to
52f33af8 8330 * match packet buffer alignment. Unfortunately, the
8b1c0b24
JF		 8331 * hardware is not flexible enough to do this dynamically.
8332 */
8333 if (netif_running(dev))
8334 ixgbe_close(dev);
bf4d67d9
AD		 8335 else
8336 ixgbe_reset(adapter);
8337
8b1c0b24
JF		 8338 ixgbe_clear_interrupt_scheme(adapter);
8339
cca73c59 8340#ifdef CONFIG_IXGBE_DCB
e7589eab 8341 if (tc) {
8b1c0b24 8342 netdev_set_num_tc(dev, tc);
02debdc9
AD		 8343 ixgbe_set_prio_tc_map(adapter);
8344
e7589eab 8345 adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
e7589eab 8346
943561d3
AD		 8347 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
8348 adapter->last_lfc_mode = adapter->hw.fc.requested_mode;
e7589eab 8349 adapter->hw.fc.requested_mode = ixgbe_fc_none;
943561d3 8350 }
e7589eab 8351 } else {
8b1c0b24 8352 netdev_reset_tc(dev);
02debdc9 8353
943561d3
AD		 8354 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
8355 adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
e7589eab
JF		 8356
8357 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
e7589eab
JF		 8358
8359 adapter->temp_dcb_cfg.pfc_mode_enable = false;
8360 adapter->dcb_cfg.pfc_mode_enable = false;
8361 }
8362
8b1c0b24 8363 ixgbe_validate_rtr(adapter, tc);
cca73c59
AD		 8364
8365#endif /* CONFIG_IXGBE_DCB */
8366 ixgbe_init_interrupt_scheme(adapter);
8367
8b1c0b24 8368 if (netif_running(dev))
cca73c59 8369 return ixgbe_open(dev);
8b1c0b24
JF		 8370
8371 return 0;
8372}
de1036b1 8373
b82b17d9
JF		 8374static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter,
8375 struct tc_cls_u32_offload *cls)
8376{
1ecedc92 8377 u32 hdl = cls->knode.handle;
176621c9 8378 u32 uhtid = TC_U32_USERHTID(cls->knode.handle);
1ecedc92
AN		 8379 u32 loc = cls->knode.handle & 0xfffff;
8380 int err = 0, i, j;
8381 struct ixgbe_jump_table *jump = NULL;
8382
8383 if (loc > IXGBE_MAX_HW_ENTRIES)
8384 return -EINVAL;
b82b17d9 8385
176621c9
SS		 8386 if ((uhtid != 0x800) && (uhtid >= IXGBE_MAX_LINK_HANDLE))
8387 return -EINVAL;
8388
1ecedc92
AN		 8389 /* Clear this filter in the link data it is associated with */
8390 if (uhtid != 0x800) {
8391 jump = adapter->jump_tables[uhtid];
12746fd2
AN		 8392 if (!jump)
8393 return -EINVAL;
8394 if (!test_bit(loc - 1, jump->child_loc_map))
8395 return -EINVAL;
8396 clear_bit(loc - 1, jump->child_loc_map);
1ecedc92
AN		 8397 }
8398
8399 /* Check if the filter being deleted is a link */
8400 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {
8401 jump = adapter->jump_tables[i];
8402 if (jump && jump->link_hdl == hdl) {
8403 /* Delete filters in the hardware in the child hash
8404 * table associated with this link
8405 */
8406 for (j = 0; j < IXGBE_MAX_HW_ENTRIES; j++) {
8407 if (!test_bit(j, jump->child_loc_map))
8408 continue;
8409 spin_lock(&adapter->fdir_perfect_lock);
8410 err = ixgbe_update_ethtool_fdir_entry(adapter,
8411 NULL,
8412 j + 1);
8413 spin_unlock(&adapter->fdir_perfect_lock);
8414 clear_bit(j, jump->child_loc_map);
8415 }
8416 /* Remove resources for this link */
8417 kfree(jump->input);
8418 kfree(jump->mask);
8419 kfree(jump);
8420 adapter->jump_tables[i] = NULL;
8421 return err;
8422 }
8423 }
176621c9 8424
b82b17d9 8425 spin_lock(&adapter->fdir_perfect_lock);
176621c9 8426 err = ixgbe_update_ethtool_fdir_entry(adapter, NULL, loc);
b82b17d9
JF		 8427 spin_unlock(&adapter->fdir_perfect_lock);
8428 return err;
8429}
8430
db956ae8
JF		 8431static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter,
8432 __be16 protocol,
8433 struct tc_cls_u32_offload *cls)
8434{
176621c9
SS		 8435 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
8436
8437 if (uhtid >= IXGBE_MAX_LINK_HANDLE)
8438 return -EINVAL;
8439
db956ae8
JF		 8440 /* This ixgbe devices do not support hash tables at the moment
8441 * so abort when given hash tables.
8442 */
8443 if (cls->hnode.divisor > 0)
8444 return -EINVAL;
8445
176621c9 8446 set_bit(uhtid - 1, &adapter->tables);
db956ae8
JF		 8447 return 0;
8448}
8449
8450static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter,
8451 struct tc_cls_u32_offload *cls)
8452{
176621c9
SS		 8453 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
8454
8455 if (uhtid >= IXGBE_MAX_LINK_HANDLE)
8456 return -EINVAL;
8457
8458 clear_bit(uhtid - 1, &adapter->tables);
db956ae8
JF		 8459 return 0;
8460}
8461
947f8a45 8462#ifdef CONFIG_NET_CLS_ACT
1cd127fc
DA		 8463struct upper_walk_data {
8464 struct ixgbe_adapter *adapter;
8465 u64 action;
8466 int ifindex;
8467 u8 queue;
8468};
8469
8470static int get_macvlan_queue(struct net_device *upper, void *_data)
8471{
8472 if (netif_is_macvlan(upper)) {
8473 struct macvlan_dev *dfwd = netdev_priv(upper);
8474 struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv;
8475 struct upper_walk_data *data = _data;
8476 struct ixgbe_adapter *adapter = data->adapter;
8477 int ifindex = data->ifindex;
8478
8479 if (vadapter && vadapter->netdev->ifindex == ifindex) {
8480 data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx;
8481 data->action = data->queue;
8482 return 1;
8483 }
8484 }
8485
8486 return 0;
8487}
8488
947f8a45
SS		 8489static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex,
8490 u8 *queue, u64 *action)
8491{
8492 unsigned int num_vfs = adapter->num_vfs, vf;
1cd127fc 8493 struct upper_walk_data data;
947f8a45 8494 struct net_device *upper;
947f8a45
SS		 8495
8496 /* redirect to a SRIOV VF */
8497 for (vf = 0; vf < num_vfs; ++vf) {
8498 upper = pci_get_drvdata(adapter->vfinfo[vf].vfdev);
8499 if (upper->ifindex == ifindex) {
8500 if (adapter->num_rx_pools > 1)
8501 *queue = vf * 2;
8502 else
8503 *queue = vf * adapter->num_rx_queues_per_pool;
8504
8505 *action = vf + 1;
8506 *action <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;
8507 return 0;
8508 }
8509 }
8510
8511 /* redirect to a offloaded macvlan netdev */
1cd127fc
DA		 8512 data.adapter = adapter;
8513 data.ifindex = ifindex;
8514 data.action = 0;
8515 data.queue = 0;
8516 if (netdev_walk_all_upper_dev_rcu(adapter->netdev,
8517 get_macvlan_queue, &data)) {
8518 *action = data.action;
8519 *queue = data.queue;
8520
8521 return 0;
947f8a45
SS		 8522 }
8523
8524 return -EINVAL;
8525}
8526
8527static int parse_tc_actions(struct ixgbe_adapter *adapter,
8528 struct tcf_exts *exts, u64 *action, u8 *queue)
8529{
8530 const struct tc_action *a;
22dc13c8 8531 LIST_HEAD(actions);
947f8a45
SS		 8532 int err;
8533
8534 if (tc_no_actions(exts))
8535 return -EINVAL;
8536
22dc13c8
WC		 8537 tcf_exts_to_list(exts, &actions);
8538 list_for_each_entry(a, &actions, list) {
947f8a45
SS		 8539
8540 /* Drop action */
8541 if (is_tcf_gact_shot(a)) {
8542 *action = IXGBE_FDIR_DROP_QUEUE;
8543 *queue = IXGBE_FDIR_DROP_QUEUE;
8544 return 0;
8545 }
8546
8547 /* Redirect to a VF or a offloaded macvlan */
5724b8b5 8548 if (is_tcf_mirred_egress_redirect(a)) {
947f8a45
SS		 8549 int ifindex = tcf_mirred_ifindex(a);
8550
8551 err = handle_redirect_action(adapter, ifindex, queue,
8552 action);
8553 if (err == 0)
8554 return err;
8555 }
8556 }
8557
8558 return -EINVAL;
8559}
8560#else
8561static int parse_tc_actions(struct ixgbe_adapter *adapter,
8562 struct tcf_exts *exts, u64 *action, u8 *queue)
8563{
8564 return -EINVAL;
8565}
8566#endif /* CONFIG_NET_CLS_ACT */
8567
1cdaaf54
AN		 8568static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,
8569 union ixgbe_atr_input *mask,
8570 struct tc_cls_u32_offload *cls,
8571 struct ixgbe_mat_field *field_ptr,
8572 struct ixgbe_nexthdr *nexthdr)
8573{
8574 int i, j, off;
8575 __be32 val, m;
8576 bool found_entry = false, found_jump_field = false;
8577
8578 for (i = 0; i < cls->knode.sel->nkeys; i++) {
8579 off = cls->knode.sel->keys[i].off;
8580 val = cls->knode.sel->keys[i].val;
8581 m = cls->knode.sel->keys[i].mask;
8582
8583 for (j = 0; field_ptr[j].val; j++) {
8584 if (field_ptr[j].off == off) {
8585 field_ptr[j].val(input, mask, val, m);
8586 input->filter.formatted.flow_type |=
8587 field_ptr[j].type;
8588 found_entry = true;
8589 break;
8590 }
8591 }
8592 if (nexthdr) {
8593 if (nexthdr->off == cls->knode.sel->keys[i].off &&
8594 nexthdr->val == cls->knode.sel->keys[i].val &&
8595 nexthdr->mask == cls->knode.sel->keys[i].mask)
8596 found_jump_field = true;
8597 else
8598 continue;
8599 }
8600 }
8601
8602 if (nexthdr && !found_jump_field)
8603 return -EINVAL;
8604
8605 if (!found_entry)
8606 return 0;
8607
8608 mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK |
8609 IXGBE_ATR_L4TYPE_MASK;
8610
8611 if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4)
8612 mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK;
8613
8614 return 0;
8615}
8616
b82b17d9
JF		 8617static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
8618 __be16 protocol,
8619 struct tc_cls_u32_offload *cls)
8620{
8621 u32 loc = cls->knode.handle & 0xfffff;
8622 struct ixgbe_hw *hw = &adapter->hw;
8623 struct ixgbe_mat_field *field_ptr;
1cdaaf54
AN		 8624 struct ixgbe_fdir_filter *input = NULL;
8625 union ixgbe_atr_input *mask = NULL;
8626 struct ixgbe_jump_table *jump = NULL;
8627 int i, err = -EINVAL;
b82b17d9 8628 u8 queue;
176621c9 8629 u32 uhtid, link_uhtid;
b82b17d9 8630
176621c9
SS		 8631 uhtid = TC_U32_USERHTID(cls->knode.handle);
8632 link_uhtid = TC_U32_USERHTID(cls->knode.link_handle);
b82b17d9 8633
176621c9 8634 /* At the moment cls_u32 jumps to network layer and skips past
b82b17d9
JF		 8635 * L2 headers. The canonical method to match L2 frames is to use
8636 * negative values. However this is error prone at best but really
8637 * just broken because there is no way to "know" what sort of hdr
176621c9 8638 * is in front of the network layer. Fix cls_u32 to support L2
b82b17d9
JF		 8639 * headers when needed.
8640 */
8641 if (protocol != htons(ETH_P_IP))
1cdaaf54 8642 return err;
b82b17d9
JF		 8643
8644 if (loc >= ((1024 << adapter->fdir_pballoc) - 2)) {
8645 e_err(drv, "Location out of range\n");
1cdaaf54 8646 return err;
b82b17d9
JF		 8647 }
8648
8649 /* cls u32 is a graph starting at root node 0x800. The driver tracks
8650 * links and also the fields used to advance the parser across each
8651 * link (e.g. nexthdr/eat parameters from 'tc'). This way we can map
8652 * the u32 graph onto the hardware parse graph denoted in ixgbe_model.h
8653 * To add support for new nodes update ixgbe_model.h parse structures
8654 * this function _should_ be generic try not to hardcode values here.
8655 */
176621c9 8656 if (uhtid == 0x800) {
1cdaaf54 8657 field_ptr = (adapter->jump_tables[0])->mat;
b82b17d9 8658 } else {
176621c9 8659 if (uhtid >= IXGBE_MAX_LINK_HANDLE)
1cdaaf54
AN		 8660 return err;
8661 if (!adapter->jump_tables[uhtid])
8662 return err;
8663 field_ptr = (adapter->jump_tables[uhtid])->mat;
b82b17d9
JF		 8664 }
8665
8666 if (!field_ptr)
1cdaaf54 8667 return err;
b82b17d9 8668
1cdaaf54
AN		 8669 /* At this point we know the field_ptr is valid and need to either
8670 * build cls_u32 link or attach filter. Because adding a link to
8671 * a handle that does not exist is invalid and the same for adding
8672 * rules to handles that don't exist.
8673 */
b82b17d9 8674
1cdaaf54
AN		 8675 if (link_uhtid) {
8676 struct ixgbe_nexthdr *nexthdr = ixgbe_ipv4_jumps;
b82b17d9 8677
1cdaaf54
AN		 8678 if (link_uhtid >= IXGBE_MAX_LINK_HANDLE)
8679 return err;
8680
8681 if (!test_bit(link_uhtid - 1, &adapter->tables))
8682 return err;
8683
1ecedc92
AN		 8684 /* Multiple filters as links to the same hash table are not
8685 * supported. To add a new filter with the same next header
8686 * but different match/jump conditions, create a new hash table
8687 * and link to it.
8688 */
8689 if (adapter->jump_tables[link_uhtid] &&
8690 (adapter->jump_tables[link_uhtid])->link_hdl) {
8691 e_err(drv, "Link filter exists for link: %x\n",
8692 link_uhtid);
8693 return err;
8694 }
8695
1cdaaf54
AN		 8696 for (i = 0; nexthdr[i].jump; i++) {
8697 if (nexthdr[i].o != cls->knode.sel->offoff ||
8698 nexthdr[i].s != cls->knode.sel->offshift ||
8699 nexthdr[i].m != cls->knode.sel->offmask)
8700 return err;
8701
8702 jump = kzalloc(sizeof(*jump), GFP_KERNEL);
8703 if (!jump)
8704 return -ENOMEM;
8705 input = kzalloc(sizeof(*input), GFP_KERNEL);
8706 if (!input) {
8707 err = -ENOMEM;
8708 goto free_jump;
8709 }
8710 mask = kzalloc(sizeof(*mask), GFP_KERNEL);
8711 if (!mask) {
8712 err = -ENOMEM;
12746fd2 8713 goto free_input;
1cdaaf54
AN		 8714 }
8715 jump->input = input;
8716 jump->mask = mask;
1ecedc92
AN		 8717 jump->link_hdl = cls->knode.handle;
8718
1cdaaf54
AN		 8719 err = ixgbe_clsu32_build_input(input, mask, cls,
8720 field_ptr, &nexthdr[i]);
8721 if (!err) {
8722 jump->mat = nexthdr[i].jump;
8723 adapter->jump_tables[link_uhtid] = jump;
b82b17d9
JF		 8724 break;
8725 }
8726 }
1cdaaf54 8727 return 0;
b82b17d9
JF		 8728 }
8729
1cdaaf54
AN		 8730 input = kzalloc(sizeof(*input), GFP_KERNEL);
8731 if (!input)
8732 return -ENOMEM;
8733 mask = kzalloc(sizeof(*mask), GFP_KERNEL);
8734 if (!mask) {
8735 err = -ENOMEM;
12746fd2 8736 goto free_input;
1cdaaf54 8737 }
b82b17d9 8738
1cdaaf54
AN		 8739 if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) {
8740 if ((adapter->jump_tables[uhtid])->input)
8741 memcpy(input, (adapter->jump_tables[uhtid])->input,
8742 sizeof(*input));
8743 if ((adapter->jump_tables[uhtid])->mask)
8744 memcpy(mask, (adapter->jump_tables[uhtid])->mask,
8745 sizeof(*mask));
12746fd2
AN		 8746
8747 /* Lookup in all child hash tables if this location is already
8748 * filled with a filter
8749 */
8750 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {
8751 struct ixgbe_jump_table *link = adapter->jump_tables[i];
8752
8753 if (link && (test_bit(loc - 1, link->child_loc_map))) {
8754 e_err(drv, "Filter exists in location: %x\n",
8755 loc);
8756 err = -EINVAL;
8757 goto err_out;
8758 }
8759 }
1cdaaf54
AN		 8760 }
8761 err = ixgbe_clsu32_build_input(input, mask, cls, field_ptr, NULL);
8762 if (err)
b82b17d9
JF		 8763 goto err_out;
8764
947f8a45
SS		 8765 err = parse_tc_actions(adapter, cls->knode.exts, &input->action,
8766 &queue);
8767 if (err < 0)
b82b17d9 8768 goto err_out;
b82b17d9 8769
b82b17d9
JF		 8770 input->sw_idx = loc;
8771
8772 spin_lock(&adapter->fdir_perfect_lock);
8773
8774 if (hlist_empty(&adapter->fdir_filter_list)) {
1cdaaf54
AN		 8775 memcpy(&adapter->fdir_mask, mask, sizeof(*mask));
8776 err = ixgbe_fdir_set_input_mask_82599(hw, mask);
b82b17d9
JF		 8777 if (err)
8778 goto err_out_w_lock;
1cdaaf54 8779 } else if (memcmp(&adapter->fdir_mask, mask, sizeof(*mask))) {
b82b17d9
JF		 8780 err = -EINVAL;
8781 goto err_out_w_lock;
8782 }
8783
1cdaaf54 8784 ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask);
b82b17d9
JF		 8785 err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter,
8786 input->sw_idx, queue);
8787 if (!err)
8788 ixgbe_update_ethtool_fdir_entry(adapter, input, input->sw_idx);
8789 spin_unlock(&adapter->fdir_perfect_lock);
8790
12746fd2
AN		 8791 if ((uhtid != 0x800) && (adapter->jump_tables[uhtid]))
8792 set_bit(loc - 1, (adapter->jump_tables[uhtid])->child_loc_map);
1ecedc92 8793
1cdaaf54 8794 kfree(mask);
b82b17d9
JF		 8795 return err;
8796err_out_w_lock:
8797 spin_unlock(&adapter->fdir_perfect_lock);
8798err_out:
1ecedc92 8799 kfree(mask);
12746fd2
AN		 8800free_input:
8801 kfree(input);
1cdaaf54
AN		 8802free_jump:
8803 kfree(jump);
8804 return err;
b82b17d9
JF		 8805}
8806
6e2a60b5
ET		 8807static int __ixgbe_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
8808 struct tc_to_netdev *tc)
e4c6734e 8809{
b82b17d9
JF		 8810 struct ixgbe_adapter *adapter = netdev_priv(dev);
8811
8812 if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) &&
8813 tc->type == TC_SETUP_CLSU32) {
b82b17d9
JF		 8814 switch (tc->cls_u32->command) {
8815 case TC_CLSU32_NEW_KNODE:
8816 case TC_CLSU32_REPLACE_KNODE:
8817 return ixgbe_configure_clsu32(adapter,
8818 proto, tc->cls_u32);
8819 case TC_CLSU32_DELETE_KNODE:
8820 return ixgbe_delete_clsu32(adapter, tc->cls_u32);
db956ae8
JF		 8821 case TC_CLSU32_NEW_HNODE:
8822 case TC_CLSU32_REPLACE_HNODE:
8823 return ixgbe_configure_clsu32_add_hnode(adapter, proto,
8824 tc->cls_u32);
8825 case TC_CLSU32_DELETE_HNODE:
8826 return ixgbe_configure_clsu32_del_hnode(adapter,
8827 tc->cls_u32);
b82b17d9
JF		 8828 default:
8829 return -EINVAL;
8830 }
8831 }
8832
5eb4dce3 8833 if (tc->type != TC_SETUP_MQPRIO)
e4c6734e
JF		 8834 return -EINVAL;
8835
16e5cc64 8836 return ixgbe_setup_tc(dev, tc->tc);
e4c6734e
JF		 8837}
8838
da36b647
GR		 8839#ifdef CONFIG_PCI_IOV
8840void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
8841{
8842 struct net_device *netdev = adapter->netdev;
8843
8844 rtnl_lock();
da36b647 8845 ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev));
da36b647
GR		 8846 rtnl_unlock();
8847}
8848
8849#endif
082757af
DS		 8850void ixgbe_do_reset(struct net_device *netdev)
8851{
8852 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8853
8854 if (netif_running(netdev))
8855 ixgbe_reinit_locked(adapter);
8856 else
8857 ixgbe_reset(adapter);
8858}
8859
c8f44aff 8860static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
567d2de2 8861 netdev_features_t features)
082757af
DS		 8862{
8863 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8864
082757af 8865 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
567d2de2
AD		 8866 if (!(features & NETIF_F_RXCSUM))
8867 features &= ~NETIF_F_LRO;
082757af 8868
567d2de2
AD		 8869 /* Turn off LRO if not RSC capable */
8870 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE))
8871 features &= ~NETIF_F_LRO;
8e2813f5 8872
567d2de2 8873 return features;
082757af
DS		 8874}
8875
c8f44aff 8876static int ixgbe_set_features(struct net_device *netdev,
567d2de2 8877 netdev_features_t features)
082757af
DS		 8878{
8879 struct ixgbe_adapter *adapter = netdev_priv(netdev);
567d2de2 8880 netdev_features_t changed = netdev->features ^ features;
082757af
DS		 8881 bool need_reset = false;
8882
082757af 8883 /* Make sure RSC matches LRO, reset if change */
567d2de2
AD		 8884 if (!(features & NETIF_F_LRO)) {
8885 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
082757af 8886 need_reset = true;
567d2de2
AD		 8887 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED;
8888 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) &&
8889 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
8890 if (adapter->rx_itr_setting == 1 ||
8891 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) {
8892 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
8893 need_reset = true;
8894 } else if ((changed ^ features) & NETIF_F_LRO) {
8895 e_info(probe, "rx-usecs set too low, "
8896 "disabling RSC\n");
082757af
DS		 8897 }
8898 }
8899
8900 /*
b82b17d9
JF		 8901 * Check if Flow Director n-tuple support or hw_tc support was
8902 * enabled or disabled. If the state changed, we need to reset.
082757af 8903 */
b82b17d9 8904 if ((features & NETIF_F_NTUPLE) || (features & NETIF_F_HW_TC)) {
567d2de2 8905 /* turn off ATR, enable perfect filters and reset */
39cb681b
AD		 8906 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
8907 need_reset = true;
8908
567d2de2
AD		 8909 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
8910 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
b82b17d9 8911 } else {
39cb681b
AD		 8912 /* turn off perfect filters, enable ATR and reset */
8913 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
8914 need_reset = true;
8915
8916 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
8917
8918 /* We cannot enable ATR if SR-IOV is enabled */
b82b17d9
JF		 8919 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED ||
8920 /* We cannot enable ATR if we have 2 or more tcs */
8921 (netdev_get_num_tc(netdev) > 1) ||
8922 /* We cannot enable ATR if RSS is disabled */
8923 (adapter->ring_feature[RING_F_RSS].limit <= 1) ||
8924 /* A sample rate of 0 indicates ATR disabled */
8925 (!adapter->atr_sample_rate))
8926 ; /* do nothing not supported */
8927 else /* otherwise supported and set the flag */
8928 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
082757af
DS		 8929 }
8930
3f2d1c0f
BG		 8931 if (changed & NETIF_F_RXALL)
8932 need_reset = true;
8933
567d2de2 8934 netdev->features = features;
67359c3c 8935
67359c3c 8936 if ((adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) {
a21d0822
ET		 8937 if (features & NETIF_F_RXCSUM) {
8938 adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
8939 } else {
8940 u32 port_mask = IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK;
8941
8942 ixgbe_clear_udp_tunnel_port(adapter, port_mask);
8943 }
8944 }
8945
8946 if ((adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)) {
8947 if (features & NETIF_F_RXCSUM) {
8948 adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
8949 } else {
8950 u32 port_mask = IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK;
8951
8952 ixgbe_clear_udp_tunnel_port(adapter, port_mask);
8953 }
67359c3c 8954 }
67359c3c 8955
082757af
DS		 8956 if (need_reset)
8957 ixgbe_do_reset(netdev);
0c5a6166
AD		 8958 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX |
8959 NETIF_F_HW_VLAN_CTAG_FILTER))
8960 ixgbe_set_rx_mode(netdev);
082757af
DS		 8961
8962 return 0;
082757af
DS		 8963}
8964
3f207800 8965/**
a21d0822 8966 * ixgbe_add_udp_tunnel_port - Get notifications about adding UDP tunnel ports
3f207800 8967 * @dev: The port's netdev
e5de25dc 8968 * @ti: Tunnel endpoint information
3f207800 8969 **/
a21d0822
ET		 8970static void ixgbe_add_udp_tunnel_port(struct net_device *dev,
8971 struct udp_tunnel_info *ti)
3f207800
DS		 8972{
8973 struct ixgbe_adapter *adapter = netdev_priv(dev);
8974 struct ixgbe_hw *hw = &adapter->hw;
b3a49557 8975 __be16 port = ti->port;
a21d0822
ET		 8976 u32 port_shift = 0;
8977 u32 reg;
67359c3c 8978
b3a49557
AD		 8979 if (ti->sa_family != AF_INET)
8980 return;
8981
a21d0822
ET		 8982 switch (ti->type) {
8983 case UDP_TUNNEL_TYPE_VXLAN:
8984 if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
8985 return;
3f207800 8986
a21d0822
ET		 8987 if (adapter->vxlan_port == port)
8988 return;
8989
8990 if (adapter->vxlan_port) {
8991 netdev_info(dev,
8992 "VXLAN port %d set, not adding port %d\n",
8993 ntohs(adapter->vxlan_port),
8994 ntohs(port));
8995 return;
8996 }
8997
8998 adapter->vxlan_port = port;
8999 break;
9000 case UDP_TUNNEL_TYPE_GENEVE:
9001 if (!(adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE))
9002 return;
9003
9004 if (adapter->geneve_port == port)
9005 return;
9006
9007 if (adapter->geneve_port) {
9008 netdev_info(dev,
9009 "GENEVE port %d set, not adding port %d\n",
9010 ntohs(adapter->geneve_port),
9011 ntohs(port));
9012 return;
9013 }
3f207800 9014
a21d0822
ET		 9015 port_shift = IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT;
9016 adapter->geneve_port = port;
9017 break;
9018 default:
3f207800
DS		 9019 return;
9020 }
9021
a21d0822
ET		 9022 reg = IXGBE_READ_REG(hw, IXGBE_VXLANCTRL) | ntohs(port) << port_shift;
9023 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, reg);
3f207800
DS		 9024}
9025
9026/**
a21d0822 9027 * ixgbe_del_udp_tunnel_port - Get notifications about removing UDP tunnel ports
3f207800 9028 * @dev: The port's netdev
e5de25dc 9029 * @ti: Tunnel endpoint information
3f207800 9030 **/
a21d0822
ET		 9031static void ixgbe_del_udp_tunnel_port(struct net_device *dev,
9032 struct udp_tunnel_info *ti)
3f207800
DS		 9033{
9034 struct ixgbe_adapter *adapter = netdev_priv(dev);
a21d0822 9035 u32 port_mask;
3f207800 9036
a21d0822
ET		 9037 if (ti->type != UDP_TUNNEL_TYPE_VXLAN &&
9038 ti->type != UDP_TUNNEL_TYPE_GENEVE)
67359c3c
MR		 9039 return;
9040
b3a49557 9041 if (ti->sa_family != AF_INET)
3f207800
DS		 9042 return;
9043
a21d0822
ET		 9044 switch (ti->type) {
9045 case UDP_TUNNEL_TYPE_VXLAN:
9046 if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
9047 return;
b3a49557 9048
a21d0822
ET		 9049 if (adapter->vxlan_port != ti->port) {
9050 netdev_info(dev, "VXLAN port %d not found\n",
9051 ntohs(ti->port));
9052 return;
9053 }
9054
9055 port_mask = IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK;
9056 break;
9057 case UDP_TUNNEL_TYPE_GENEVE:
9058 if (!(adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE))
9059 return;
9060
9061 if (adapter->geneve_port != ti->port) {
9062 netdev_info(dev, "GENEVE port %d not found\n",
9063 ntohs(ti->port));
9064 return;
9065 }
9066
9067 port_mask = IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK;
9068 break;
9069 default:
3f207800
DS		 9070 return;
9071 }
9072
a21d0822
ET		 9073 ixgbe_clear_udp_tunnel_port(adapter, port_mask);
9074 adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED;
3f207800
DS		 9075}
9076
edc7d573 9077static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
0f4b0add 9078 struct net_device *dev,
f6f6424b 9079 const unsigned char *addr, u16 vid,
0f4b0add
JF		 9080 u16 flags)
9081{
bcfd3432 9082 /* guarantee we can provide a unique filter for the unicast address */
46acc460 9083 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2f9be166
AD		 9084 struct ixgbe_adapter *adapter = netdev_priv(dev);
9085 u16 pool = VMDQ_P(0);
9086
9087 if (netdev_uc_count(dev) >= ixgbe_available_rars(adapter, pool))
bcfd3432 9088 return -ENOMEM;
0f4b0add
JF		 9089 }
9090
f6f6424b 9091 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
0f4b0add
JF		 9092}
9093
219efe97
DS		 9094/**
9095 * ixgbe_configure_bridge_mode - set various bridge modes
9096 * @adapter - the private structure
9097 * @mode - requested bridge mode
9098 *
9099 * Configure some settings require for various bridge modes.
9100 **/
9101static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter,
9102 __u16 mode)
9103{
6d4c96ad
DS		 9104 struct ixgbe_hw *hw = &adapter->hw;
9105 unsigned int p, num_pools;
9106 u32 vmdctl;
9107
219efe97
DS		 9108 switch (mode) {
9109 case BRIDGE_MODE_VEPA:
6d4c96ad 9110 /* disable Tx loopback, rely on switch hairpin mode */
219efe97 9111 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 0);
6d4c96ad
DS		 9112
9113 /* must enable Rx switching replication to allow multicast
9114 * packet reception on all VFs, and to enable source address
9115 * pruning.
9116 */
9117 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
9118 vmdctl |= IXGBE_VT_CTL_REPLEN;
9119 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
9120
9121 /* enable Rx source address pruning. Note, this requires
9122 * replication to be enabled or else it does nothing.
9123 */
9124 num_pools = adapter->num_vfs + adapter->num_rx_pools;
9125 for (p = 0; p < num_pools; p++) {
9126 if (hw->mac.ops.set_source_address_pruning)
9127 hw->mac.ops.set_source_address_pruning(hw,
9128 true,
9129 p);
9130 }
219efe97
DS		 9131 break;
9132 case BRIDGE_MODE_VEB:
6d4c96ad 9133 /* enable Tx loopback for internal VF/PF communication */
219efe97
DS		 9134 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC,
9135 IXGBE_PFDTXGSWC_VT_LBEN);
6d4c96ad
DS		 9136
9137 /* disable Rx switching replication unless we have SR-IOV
9138 * virtual functions
9139 */
9140 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
9141 if (!adapter->num_vfs)
9142 vmdctl &= ~IXGBE_VT_CTL_REPLEN;
9143 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
9144
9145 /* disable Rx source address pruning, since we don't expect to
9146 * be receiving external loopback of our transmitted frames.
9147 */
9148 num_pools = adapter->num_vfs + adapter->num_rx_pools;
9149 for (p = 0; p < num_pools; p++) {
9150 if (hw->mac.ops.set_source_address_pruning)
9151 hw->mac.ops.set_source_address_pruning(hw,
9152 false,
9153 p);
9154 }
219efe97
DS		 9155 break;
9156 default:
9157 return -EINVAL;
9158 }
9159
9160 adapter->bridge_mode = mode;
9161
9162 e_info(drv, "enabling bridge mode: %s\n",
9163 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
9164
9165 return 0;
9166}
9167
815cccbf 9168static int ixgbe_ndo_bridge_setlink(struct net_device *dev,
add511b3 9169 struct nlmsghdr *nlh, u16 flags)
815cccbf
JF		 9170{
9171 struct ixgbe_adapter *adapter = netdev_priv(dev);
9172 struct nlattr *attr, *br_spec;
9173 int rem;
9174
9175 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
9176 return -EOPNOTSUPP;
9177
9178 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4ea85e83
TG		 9179 if (!br_spec)
9180 return -EINVAL;
815cccbf
JF		 9181
9182 nla_for_each_nested(attr, br_spec, rem) {
a1e869de 9183 int status;
815cccbf 9184 __u16 mode;
815cccbf
JF		 9185
9186 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9187 continue;
9188
b7c1a314
TG		 9189 if (nla_len(attr) < sizeof(mode))
9190 return -EINVAL;
9191
815cccbf 9192 mode = nla_get_u16(attr);
219efe97
DS		 9193 status = ixgbe_configure_bridge_mode(adapter, mode);
9194 if (status)
9195 return status;
aa2bacb6
DS		 9196
9197 break;
815cccbf
JF		 9198 }
9199
9200 return 0;
9201}
9202
9203static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
6cbdceeb 9204 struct net_device *dev,
46c264da 9205 u32 filter_mask, int nlflags)
815cccbf
JF		 9206{
9207 struct ixgbe_adapter *adapter = netdev_priv(dev);
815cccbf
JF		 9208
9209 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
9210 return 0;
9211
aa2bacb6 9212 return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
7d4f8d87
SF		 9213 adapter->bridge_mode, 0, 0, nlflags,
9214 filter_mask, NULL);
815cccbf
JF		 9215}
9216
2a47fa45
JF		 9217static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
9218{
9219 struct ixgbe_fwd_adapter *fwd_adapter = NULL;
9220 struct ixgbe_adapter *adapter = netdev_priv(pdev);
aac2f1bf 9221 int used_pools = adapter->num_vfs + adapter->num_rx_pools;
51f3773b 9222 unsigned int limit;
2a47fa45
JF		 9223 int pool, err;
9224
aac2f1bf
JK		 9225 /* Hardware has a limited number of available pools. Each VF, and the
9226 * PF require a pool. Check to ensure we don't attempt to use more
9227 * then the available number of pools.
9228 */
9229 if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
9230 return ERR_PTR(-EINVAL);
9231
219354d4
JF		 9232#ifdef CONFIG_RPS
9233 if (vdev->num_rx_queues != vdev->num_tx_queues) {
9234 netdev_info(pdev, "%s: Only supports a single queue count for TX and RX\n",
9235 vdev->name);
9236 return ERR_PTR(-EINVAL);
9237 }
9238#endif
2a47fa45 9239 /* Check for hardware restriction on number of rx/tx queues */
219354d4 9240 if (vdev->num_tx_queues > IXGBE_MAX_L2A_QUEUES ||
2a47fa45
JF		 9241 vdev->num_tx_queues == IXGBE_BAD_L2A_QUEUE) {
9242 netdev_info(pdev,
9243 "%s: Supports RX/TX Queue counts 1,2, and 4\n",
9244 pdev->name);
9245 return ERR_PTR(-EINVAL);
9246 }
9247
9248 if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
9249 adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS - 1) ||
9250 (adapter->num_rx_pools > IXGBE_MAX_MACVLANS))
9251 return ERR_PTR(-EBUSY);
9252
bc52f951 9253 fwd_adapter = kzalloc(sizeof(*fwd_adapter), GFP_KERNEL);
2a47fa45
JF		 9254 if (!fwd_adapter)
9255 return ERR_PTR(-ENOMEM);
9256
9257 pool = find_first_zero_bit(&adapter->fwd_bitmask, 32);
9258 adapter->num_rx_pools++;
9259 set_bit(pool, &adapter->fwd_bitmask);
51f3773b 9260 limit = find_last_bit(&adapter->fwd_bitmask, 32);
2a47fa45
JF		 9261
9262 /* Enable VMDq flag so device will be set in VM mode */
9263 adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED;
51f3773b 9264 adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
219354d4 9265 adapter->ring_feature[RING_F_RSS].limit = vdev->num_tx_queues;
2a47fa45
JF		 9266
9267 /* Force reinit of ring allocation with VMDQ enabled */
9268 err = ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev));
9269 if (err)
9270 goto fwd_add_err;
9271 fwd_adapter->pool = pool;
9272 fwd_adapter->real_adapter = adapter;
a3b8cb1f
ET		 9273
9274 if (netif_running(pdev)) {
9275 err = ixgbe_fwd_ring_up(vdev, fwd_adapter);
9276 if (err)
9277 goto fwd_add_err;
9278 netif_tx_start_all_queues(vdev);
9279 }
9280
2a47fa45
JF		 9281 return fwd_adapter;
9282fwd_add_err:
9283 /* unwind counter and free adapter struct */
9284 netdev_info(pdev,
9285 "%s: dfwd hardware acceleration failed\n", vdev->name);
9286 clear_bit(pool, &adapter->fwd_bitmask);
9287 adapter->num_rx_pools--;
9288 kfree(fwd_adapter);
9289 return ERR_PTR(err);
9290}
9291
9292static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
9293{
9294 struct ixgbe_fwd_adapter *fwd_adapter = priv;
9295 struct ixgbe_adapter *adapter = fwd_adapter->real_adapter;
51f3773b 9296 unsigned int limit;
2a47fa45
JF		 9297
9298 clear_bit(fwd_adapter->pool, &adapter->fwd_bitmask);
9299 adapter->num_rx_pools--;
9300
51f3773b
JF		 9301 limit = find_last_bit(&adapter->fwd_bitmask, 32);
9302 adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
2a47fa45
JF		 9303 ixgbe_fwd_ring_down(fwd_adapter->netdev, fwd_adapter);
9304 ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev));
9305 netdev_dbg(pdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n",
9306 fwd_adapter->pool, adapter->num_rx_pools,
9307 fwd_adapter->rx_base_queue,
9308 fwd_adapter->rx_base_queue + adapter->num_rx_queues_per_pool,
9309 adapter->fwd_bitmask);
9310 kfree(fwd_adapter);
9311}
9312
b83e3010
AD		 9313#define IXGBE_MAX_MAC_HDR_LEN 127
9314#define IXGBE_MAX_NETWORK_HDR_LEN 511
9315
f467bc06
MR		 9316static netdev_features_t
9317ixgbe_features_check(struct sk_buff *skb, struct net_device *dev,
9318 netdev_features_t features)
9319{
b83e3010
AD		 9320 unsigned int network_hdr_len, mac_hdr_len;
9321
9322 /* Make certain the headers can be described by a context descriptor */
9323 mac_hdr_len = skb_network_header(skb) - skb->data;
9324 if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN))
9325 return features & ~(NETIF_F_HW_CSUM |
9326 NETIF_F_SCTP_CRC |
9327 NETIF_F_HW_VLAN_CTAG_TX |
9328 NETIF_F_TSO |
9329 NETIF_F_TSO6);
9330
9331 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
9332 if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN))
9333 return features & ~(NETIF_F_HW_CSUM |
9334 NETIF_F_SCTP_CRC |
9335 NETIF_F_TSO |
9336 NETIF_F_TSO6);
9337
9338 /* We can only support IPV4 TSO in tunnels if we can mangle the
9339 * inner IP ID field, so strip TSO if MANGLEID is not supported.
9340 */
9341 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
9342 features &= ~NETIF_F_TSO;
f467bc06
MR		 9343
9344 return features;
9345}
9346
0edc3527 9347static const struct net_device_ops ixgbe_netdev_ops = {
e8e9f696 9348 .ndo_open = ixgbe_open,
0edc3527 9349 .ndo_stop = ixgbe_close,
00829823 9350 .ndo_start_xmit = ixgbe_xmit_frame,
09a3b1f8 9351 .ndo_select_queue = ixgbe_select_queue,
581330ba 9352 .ndo_set_rx_mode = ixgbe_set_rx_mode,
0edc3527
SH		 9353 .ndo_validate_addr = eth_validate_addr,
9354 .ndo_set_mac_address = ixgbe_set_mac,
9355 .ndo_change_mtu = ixgbe_change_mtu,
9356 .ndo_tx_timeout = ixgbe_tx_timeout,
c04f90e5 9357 .ndo_set_tx_maxrate = ixgbe_tx_maxrate,
0edc3527
SH		 9358 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
9359 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
6b73e10d 9360 .ndo_do_ioctl = ixgbe_ioctl,
7f01648a
GR		 9361 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
9362 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
ed616689 9363 .ndo_set_vf_rate = ixgbe_ndo_set_vf_bw,
581330ba 9364 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
e65ce0d3 9365 .ndo_set_vf_rss_query_en = ixgbe_ndo_set_vf_rss_query_en,
54011e4d 9366 .ndo_set_vf_trust = ixgbe_ndo_set_vf_trust,
7f01648a 9367 .ndo_get_vf_config = ixgbe_ndo_get_vf_config,
de1036b1 9368 .ndo_get_stats64 = ixgbe_get_stats64,
e4c6734e 9369 .ndo_setup_tc = __ixgbe_setup_tc,
0edc3527
SH		 9370#ifdef CONFIG_NET_POLL_CONTROLLER
9371 .ndo_poll_controller = ixgbe_netpoll,
9372#endif
332d4a7d
YZ		 9373#ifdef IXGBE_FCOE
9374 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
68a683cf 9375 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
332d4a7d 9376 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
8450ff8c
YZ		 9377 .ndo_fcoe_enable = ixgbe_fcoe_enable,
9378 .ndo_fcoe_disable = ixgbe_fcoe_disable,
61a1fa10 9379 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
ea81875a 9380 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,
332d4a7d 9381#endif /* IXGBE_FCOE */
082757af
DS		 9382 .ndo_set_features = ixgbe_set_features,
9383 .ndo_fix_features = ixgbe_fix_features,
0f4b0add 9384 .ndo_fdb_add = ixgbe_ndo_fdb_add,
815cccbf
JF		 9385 .ndo_bridge_setlink = ixgbe_ndo_bridge_setlink,
9386 .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink,
2a47fa45
JF		 9387 .ndo_dfwd_add_station = ixgbe_fwd_add,
9388 .ndo_dfwd_del_station = ixgbe_fwd_del,
a21d0822
ET		 9389 .ndo_udp_tunnel_add = ixgbe_add_udp_tunnel_port,
9390 .ndo_udp_tunnel_del = ixgbe_del_udp_tunnel_port,
f467bc06 9391 .ndo_features_check = ixgbe_features_check,
0edc3527
SH		 9392};
9393
e027d1ae
JK		 9394/**
9395 * ixgbe_enumerate_functions - Get the number of ports this device has
9396 * @adapter: adapter structure
9397 *
9398 * This function enumerates the phsyical functions co-located on a single slot,
9399 * in order to determine how many ports a device has. This is most useful in
9400 * determining the required GT/s of PCIe bandwidth necessary for optimal
9401 * performance.
9402 **/
9403static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter)
9404{
caafb95d 9405 struct pci_dev *entry, *pdev = adapter->pdev;
e027d1ae
JK		 9406 int physfns = 0;
9407
f1f96579
JK		 9408 /* Some cards can not use the generic count PCIe functions method,
9409 * because they are behind a parent switch, so we hardcode these with
9410 * the correct number of functions.
e027d1ae 9411 */
8818970d 9412 if (ixgbe_pcie_from_parent(&adapter->hw))
e027d1ae 9413 physfns = 4;
8818970d
JK		 9414
9415 list_for_each_entry(entry, &adapter->pdev->bus->devices, bus_list) {
9416 /* don't count virtual functions */
caafb95d
JK		 9417 if (entry->is_virtfn)
9418 continue;
9419
9420 /* When the devices on the bus don't all match our device ID,
9421 * we can't reliably determine the correct number of
9422 * functions. This can occur if a function has been direct
9423 * attached to a virtual machine using VT-d, for example. In
9424 * this case, simply return -1 to indicate this.
9425 */
9426 if ((entry->vendor != pdev->vendor) ||
9427 (entry->device != pdev->device))
9428 return -1;
9429
9430 physfns++;
e027d1ae
JK		 9431 }
9432
9433 return physfns;
9434}
9435
8e2813f5
JK		 9436/**
9437 * ixgbe_wol_supported - Check whether device supports WoL
740234f0 9438 * @adapter: the adapter private structure
8e2813f5
JK		 9439 * @device_id: the device ID
9440 * @subdev_id: the subsystem device ID
9441 *
9442 * This function is used by probe and ethtool to determine
9443 * which devices have WoL support
9444 *
9445 **/
740234f0
ET		 9446bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,
9447 u16 subdevice_id)
8e2813f5
JK		 9448{
9449 struct ixgbe_hw *hw = &adapter->hw;
9450 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
8e2813f5 9451
740234f0
ET		 9452 /* WOL not supported on 82598 */
9453 if (hw->mac.type == ixgbe_mac_82598EB)
9454 return false;
9455
9456 /* check eeprom to see if WOL is enabled for X540 and newer */
9457 if (hw->mac.type >= ixgbe_mac_X540) {
9458 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
9459 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
9460 (hw->bus.func == 0)))
9461 return true;
9462 }
9463
9464 /* WOL is determined based on device IDs for 82599 MACs */
8e2813f5
JK		 9465 switch (device_id) {
9466 case IXGBE_DEV_ID_82599_SFP:
9467 /* Only these subdevices could supports WOL */
9468 switch (subdevice_id) {
9469 case IXGBE_SUBDEV_ID_82599_560FLR:
00103a6c
ET		 9470 case IXGBE_SUBDEV_ID_82599_LOM_SNAP6:
9471 case IXGBE_SUBDEV_ID_82599_SFP_WOL0:
9472 case IXGBE_SUBDEV_ID_82599_SFP_2OCP:
8e2813f5
JK		 9473 /* only support first port */
9474 if (hw->bus.func != 0)
9475 break;
5700ff26 9476 case IXGBE_SUBDEV_ID_82599_SP_560FLR:
8e2813f5 9477 case IXGBE_SUBDEV_ID_82599_SFP:
b6dfd939 9478 case IXGBE_SUBDEV_ID_82599_RNDC:
f8a06c2c 9479 case IXGBE_SUBDEV_ID_82599_ECNA_DP:
00103a6c
ET		 9480 case IXGBE_SUBDEV_ID_82599_SFP_1OCP:
9481 case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM1:
9482 case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM2:
740234f0 9483 return true;
8e2813f5
JK		 9484 }
9485 break;
5daebbb0 9486 case IXGBE_DEV_ID_82599EN_SFP:
740234f0 9487 /* Only these subdevices support WOL */
5daebbb0
DS		 9488 switch (subdevice_id) {
9489 case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1:
740234f0 9490 return true;
5daebbb0
DS		 9491 }
9492 break;
8e2813f5
JK		 9493 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
9494 /* All except this subdevice support WOL */
9495 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
740234f0 9496 return true;
8e2813f5
JK		 9497 break;
9498 case IXGBE_DEV_ID_82599_KX4:
740234f0
ET		 9499 return true;
9500 default:
8e2813f5
JK		 9501 break;
9502 }
9503
740234f0 9504 return false;
8e2813f5
JK		 9505}
9506
9a799d71
AK		 9507/**
9508 * ixgbe_probe - Device Initialization Routine
9509 * @pdev: PCI device information struct
9510 * @ent: entry in ixgbe_pci_tbl
9511 *
9512 * Returns 0 on success, negative on failure
9513 *
9514 * ixgbe_probe initializes an adapter identified by a pci_dev structure.
9515 * The OS initialization, configuring of the adapter private structure,
9516 * and a hardware reset occur.
9517 **/
1dd06ae8 9518static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
9a799d71
AK		 9519{
9520 struct net_device *netdev;
9521 struct ixgbe_adapter *adapter = NULL;
9522 struct ixgbe_hw *hw;
9523 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
e027d1ae 9524 int i, err, pci_using_dac, expected_gts;
d3cb9869 9525 unsigned int indices = MAX_TX_QUEUES;
289700db 9526 u8 part_str[IXGBE_PBANUM_LENGTH];
b5b2ffc0 9527 bool disable_dev = false;
eacd73f7
YZ		 9528#ifdef IXGBE_FCOE
9529 u16 device_caps;
9530#endif
289700db 9531 u32 eec;
9a799d71 9532
bded64a7
AG		 9533 /* Catch broken hardware that put the wrong VF device ID in
9534 * the PCIe SR-IOV capability.
9535 */
9536 if (pdev->is_virtfn) {
9537 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
9538 pci_name(pdev), pdev->vendor, pdev->device);
9539 return -EINVAL;
9540 }
9541
9ce77666 9542 err = pci_enable_device_mem(pdev);
9a799d71
AK		 9543 if (err)
9544 return err;
9545
f5f2eda8 9546 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
9a799d71
AK		 9547 pci_using_dac = 1;
9548 } else {
f5f2eda8 9549 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
9a799d71 9550 if (err) {
f5f2eda8
RK		 9551 dev_err(&pdev->dev,
9552 "No usable DMA configuration, aborting\n");
9553 goto err_dma;
9a799d71
AK		 9554 }
9555 pci_using_dac = 0;
9556 }
9557
56d766d6 9558 err = pci_request_mem_regions(pdev, ixgbe_driver_name);
9a799d71 9559 if (err) {
b8bc0421
DC		 9560 dev_err(&pdev->dev,
9561 "pci_request_selected_regions failed 0x%x\n", err);
9a799d71
AK		 9562 goto err_pci_reg;
9563 }
9564
19d5afd4 9565 pci_enable_pcie_error_reporting(pdev);
6fabd715 9566
9a799d71 9567 pci_set_master(pdev);
fb3b27bc 9568 pci_save_state(pdev);
9a799d71 9569
d3cb9869 9570 if (ii->mac == ixgbe_mac_82598EB) {
e901acd6 9571#ifdef CONFIG_IXGBE_DCB
d3cb9869
AD		 9572 /* 8 TC w/ 4 queues per TC */
9573 indices = 4 * MAX_TRAFFIC_CLASS;
9574#else
9575 indices = IXGBE_MAX_RSS_INDICES;
e901acd6 9576#endif
d3cb9869 9577 }
e901acd6 9578
c85a2618 9579 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
9a799d71
AK		 9580 if (!netdev) {
9581 err = -ENOMEM;
9582 goto err_alloc_etherdev;
9583 }
9584
9a799d71
AK		 9585 SET_NETDEV_DEV(netdev, &pdev->dev);
9586
9a799d71
AK		 9587 adapter = netdev_priv(netdev);
9588
9589 adapter->netdev = netdev;
9590 adapter->pdev = pdev;
9591 hw = &adapter->hw;
9592 hw->back = adapter;
b3f4d599 9593 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
9a799d71 9594
05857980 9595 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
e8e9f696 9596 pci_resource_len(pdev, 0));
2a1a091c 9597 adapter->io_addr = hw->hw_addr;
9a799d71
AK		 9598 if (!hw->hw_addr) {
9599 err = -EIO;
9600 goto err_ioremap;
9601 }
9602
0edc3527 9603 netdev->netdev_ops = &ixgbe_netdev_ops;
9a799d71 9604 ixgbe_set_ethtool_ops(netdev);
9a799d71 9605 netdev->watchdog_timeo = 5 * HZ;
339de30f 9606 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
9a799d71 9607
9a799d71 9608 /* Setup hw api */
37689010 9609 hw->mac.ops = *ii->mac_ops;
021230d4 9610 hw->mac.type = ii->mac;
9a900eca 9611 hw->mvals = ii->mvals;
b71f6c40
ET		 9612 if (ii->link_ops)
9613 hw->link.ops = *ii->link_ops;
9a799d71 9614
c44ade9e 9615 /* EEPROM */
37689010 9616 hw->eeprom.ops = *ii->eeprom_ops;
9a900eca 9617 eec = IXGBE_READ_REG(hw, IXGBE_EEC(hw));
58cf663f
MR		 9618 if (ixgbe_removed(hw->hw_addr)) {
9619 err = -EIO;
9620 goto err_ioremap;
9621 }
c44ade9e 9622 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
b4f47a48 9623 if (!(eec & BIT(8)))
c44ade9e
JB		 9624 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
9625
9626 /* PHY */
37689010 9627 hw->phy.ops = *ii->phy_ops;
c4900be0 9628 hw->phy.sfp_type = ixgbe_sfp_type_unknown;
6b73e10d
BH		 9629 /* ixgbe_identify_phy_generic will set prtad and mmds properly */
9630 hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
9631 hw->phy.mdio.mmds = 0;
9632 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
9633 hw->phy.mdio.dev = netdev;
9634 hw->phy.mdio.mdio_read = ixgbe_mdio_read;
9635 hw->phy.mdio.mdio_write = ixgbe_mdio_write;
c4900be0 9636
9a799d71 9637 /* setup the private structure */
55570b6f 9638 err = ixgbe_sw_init(adapter, ii);
9a799d71
AK		 9639 if (err)
9640 goto err_sw_init;
9641
dbd15b8f
DS		 9642 /* Make sure the SWFW semaphore is in a valid state */
9643 if (hw->mac.ops.init_swfw_sync)
9644 hw->mac.ops.init_swfw_sync(hw);
9645
e86bff0e 9646 /* Make it possible the adapter to be woken up via WOL */
b93a2226
DS		 9647 switch (adapter->hw.mac.type) {
9648 case ixgbe_mac_82599EB:
9649 case ixgbe_mac_X540:
9a75a1ac
DS		 9650 case ixgbe_mac_X550:
9651 case ixgbe_mac_X550EM_x:
49425dfc 9652 case ixgbe_mac_x550em_a:
e86bff0e 9653 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
b93a2226
DS		 9654 break;
9655 default:
9656 break;
9657 }
e86bff0e 9658
bf069c97
DS		 9659 /*
9660 * If there is a fan on this device and it has failed log the
9661 * failure.
9662 */
9663 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
9664 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
9665 if (esdp & IXGBE_ESDP_SDP1)
396e799c 9666 e_crit(probe, "Fan has stopped, replace the adapter\n");
bf069c97
DS		 9667 }
9668
8ef78adc
PWJ		 9669 if (allow_unsupported_sfp)
9670 hw->allow_unsupported_sfp = allow_unsupported_sfp;
9671
c44ade9e 9672 /* reset_hw fills in the perm_addr as well */
119fc60a 9673 hw->phy.reset_if_overtemp = true;
c44ade9e 9674 err = hw->mac.ops.reset_hw(hw);
119fc60a 9675 hw->phy.reset_if_overtemp = false;
b3eb4e18 9676 ixgbe_set_eee_capable(adapter);
29a8dca1 9677 if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
8ca783ab
DS		 9678 err = 0;
9679 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1b1bf31a
DS		 9680 e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n");
9681 e_dev_err("Reload the driver after installing a supported module.\n");
04f165ef
PW		 9682 goto err_sw_init;
9683 } else if (err) {
849c4542 9684 e_dev_err("HW Init failed: %d\n", err);
c44ade9e
JB		 9685 goto err_sw_init;
9686 }
9687
99d74487 9688#ifdef CONFIG_PCI_IOV
60a1a680
GR		 9689 /* SR-IOV not supported on the 82598 */
9690 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
9691 goto skip_sriov;
9692 /* Mailbox */
9693 ixgbe_init_mbx_params_pf(hw);
37689010 9694 hw->mbx.ops = ii->mbx_ops;
dcc23e3a 9695 pci_sriov_set_totalvfs(pdev, IXGBE_MAX_VFS_DRV_LIMIT);
31ac910e 9696 ixgbe_enable_sriov(adapter);
60a1a680 9697skip_sriov:
1cdd1ec8 9698
99d74487 9699#endif
396e799c 9700 netdev->features = NETIF_F_SG |
082757af
DS		 9701 NETIF_F_TSO |
9702 NETIF_F_TSO6 |
082757af 9703 NETIF_F_RXHASH |
49763de0 9704 NETIF_F_RXCSUM |
b83e3010
AD		 9705 NETIF_F_HW_CSUM;
9706
9707#define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
9708 NETIF_F_GSO_GRE_CSUM | \
7e13318d 9709 NETIF_F_GSO_IPXIP4 | \
bf2d1df3 9710 NETIF_F_GSO_IPXIP6 | \
b83e3010
AD		 9711 NETIF_F_GSO_UDP_TUNNEL | \
9712 NETIF_F_GSO_UDP_TUNNEL_CSUM)
9713
9714 netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES;
9715 netdev->features |= NETIF_F_GSO_PARTIAL |
9716 IXGBE_GSO_PARTIAL_FEATURES;
ad31c402 9717
49763de0 9718 if (hw->mac.type >= ixgbe_mac_82599EB)
53692b1d 9719 netdev->features |= NETIF_F_SCTP_CRC;
49763de0
AD		 9720
9721 /* copy netdev features into list of user selectable features */
b83e3010 9722 netdev->hw_features |= netdev->features |
3d951822 9723 NETIF_F_HW_VLAN_CTAG_FILTER |
b83e3010
AD		 9724 NETIF_F_HW_VLAN_CTAG_RX |
9725 NETIF_F_HW_VLAN_CTAG_TX |
9726 NETIF_F_RXALL |
49763de0
AD		 9727 NETIF_F_HW_L2FW_DOFFLOAD;
9728
9729 if (hw->mac.type >= ixgbe_mac_82599EB)
9730 netdev->hw_features |= NETIF_F_NTUPLE |
b82b17d9 9731 NETIF_F_HW_TC;
45a5ead0 9732
b83e3010
AD		 9733 if (pci_using_dac)
9734 netdev->features |= NETIF_F_HIGHDMA;
9735
5eee87cd
AD		 9736 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
9737 netdev->hw_enc_features |= netdev->vlan_features;
9738 netdev->mpls_features |= NETIF_F_HW_CSUM;
9739
b83e3010
AD		 9740 /* set this bit last since it cannot be part of vlan_features */
9741 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
9742 NETIF_F_HW_VLAN_CTAG_RX |
9743 NETIF_F_HW_VLAN_CTAG_TX;
ad31c402 9744
01789349 9745 netdev->priv_flags |= IFF_UNICAST_FLT;
f43f313e 9746 netdev->priv_flags |= IFF_SUPP_NOFCS;
01789349 9747
91c527a5
JW		 9748 /* MTU range: 68 - 9710 */
9749 netdev->min_mtu = ETH_MIN_MTU;
9750 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
9751
7a6b6f51 9752#ifdef CONFIG_IXGBE_DCB
8829009d 9753 if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)
3f40c74c 9754 netdev->dcbnl_ops = &ixgbe_dcbnl_ops;
2f90b865
AD		 9755#endif
9756
eacd73f7 9757#ifdef IXGBE_FCOE
0d551589 9758 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
d3cb9869
AD		 9759 unsigned int fcoe_l;
9760
eacd73f7
YZ		 9761 if (hw->mac.ops.get_device_caps) {
9762 hw->mac.ops.get_device_caps(hw, &device_caps);
0d551589
YZ		 9763 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
9764 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
eacd73f7 9765 }
7c8ae65a 9766
d3cb9869
AD		 9767
9768 fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus());
9769 adapter->ring_feature[RING_F_FCOE].limit = fcoe_l;
7c8ae65a 9770
a58915c7
AD		 9771 netdev->features |= NETIF_F_FSO |
9772 NETIF_F_FCOE_CRC;
9773
7c8ae65a
AD		 9774 netdev->vlan_features |= NETIF_F_FSO |
9775 NETIF_F_FCOE_CRC |
9776 NETIF_F_FCOE_MTU;
5e09d7f6 9777 }
eacd73f7 9778#endif /* IXGBE_FCOE */
9a799d71 9779
082757af
DS		 9780 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
9781 netdev->hw_features |= NETIF_F_LRO;
0c19d6af 9782 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
f8212f97
AD		 9783 netdev->features |= NETIF_F_LRO;
9784
9a799d71 9785 /* make sure the EEPROM is good */
c44ade9e 9786 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
849c4542 9787 e_dev_err("The EEPROM Checksum Is Not Valid\n");
9a799d71 9788 err = -EIO;
35937c05 9789 goto err_sw_init;
9a799d71
AK		 9790 }
9791
c7374b5a
SV		 9792 eth_platform_get_mac_address(&adapter->pdev->dev,
9793 adapter->hw.mac.perm_addr);
c762dff2 9794
9a799d71 9795 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
9a799d71 9796
aaeb6cdf 9797 if (!is_valid_ether_addr(netdev->dev_addr)) {
849c4542 9798 e_dev_err("invalid MAC address\n");
9a799d71 9799 err = -EIO;
35937c05 9800 goto err_sw_init;
9a799d71
AK		 9801 }
9802
56768045
TD		 9803 /* Set hw->mac.addr to permanent MAC address */
9804 ether_addr_copy(hw->mac.addr, hw->mac.perm_addr);
c9f53e63 9805 ixgbe_mac_set_default_filter(adapter);
5d7daa35 9806
7086400d 9807 setup_timer(&adapter->service_timer, &ixgbe_service_timer,
581330ba 9808 (unsigned long) adapter);
9a799d71 9809
58cf663f
MR		 9810 if (ixgbe_removed(hw->hw_addr)) {
9811 err = -EIO;
9812 goto err_sw_init;
9813 }
7086400d 9814 INIT_WORK(&adapter->service_task, ixgbe_service_task);
58cf663f 9815 set_bit(__IXGBE_SERVICE_INITED, &adapter->state);
7086400d 9816 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
9a799d71 9817
021230d4
AV		 9818 err = ixgbe_init_interrupt_scheme(adapter);
9819 if (err)
9820 goto err_sw_init;
9a799d71 9821
8e2813f5 9822 /* WOL not supported for all devices */
c23f5b6b 9823 adapter->wol = 0;
8e2813f5 9824 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
6b92b0ba 9825 hw->wol_enabled = ixgbe_wol_supported(adapter, pdev->device,
b8f83638 9826 pdev->subsystem_device);
6b92b0ba 9827 if (hw->wol_enabled)
9417c464 9828 adapter->wol = IXGBE_WUFC_MAG;
c23f5b6b 9829
e8e26350
PW		 9830 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9831
15e5209f
ET		 9832 /* save off EEPROM version number */
9833 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh);
9834 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl);
9835
04f165ef 9836 /* pick up the PCI bus settings for reporting later */
e027d1ae 9837 if (ixgbe_pcie_from_parent(hw))
b8e82001 9838 ixgbe_get_parent_bus_info(adapter);
f9328bc6
DS		 9839 else
9840 hw->mac.ops.get_bus_info(hw);
04f165ef 9841
e027d1ae
JK		 9842 /* calculate the expected PCIe bandwidth required for optimal
9843 * performance. Note that some older parts will never have enough
9844 * bandwidth due to being older generation PCIe parts. We clamp these
9845 * parts to ensure no warning is displayed if it can't be fixed.
9846 */
9847 switch (hw->mac.type) {
9848 case ixgbe_mac_82598EB:
9849 expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16);
9850 break;
9851 default:
9852 expected_gts = ixgbe_enumerate_functions(adapter) * 10;
9853 break;
0c254d86 9854 }
caafb95d
JK		 9855
9856 /* don't check link if we failed to enumerate functions */
9857 if (expected_gts > 0)
9858 ixgbe_check_minimum_link(adapter, expected_gts);
0c254d86 9859
339de30f 9860 err = ixgbe_read_pba_string_generic(hw, part_str, sizeof(part_str));
6a2aae5a 9861 if (err)
339de30f 9862 strlcpy(part_str, "Unknown", sizeof(part_str));
6a2aae5a
JK		 9863 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
9864 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
9865 hw->mac.type, hw->phy.type, hw->phy.sfp_type,
e7cf745b 9866 part_str);
6a2aae5a
JK		 9867 else
9868 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
9869 hw->mac.type, hw->phy.type, part_str);
9870
9871 e_dev_info("%pM\n", netdev->dev_addr);
9872
9a799d71 9873 /* reset the hardware with the new settings */
794caeb2 9874 err = hw->mac.ops.start_hw(hw);
794caeb2
PWJ		 9875 if (err == IXGBE_ERR_EEPROM_VERSION) {
9876 /* We are running on a pre-production device, log a warning */
849c4542
ET		 9877 e_dev_warn("This device is a pre-production adapter/LOM. "
9878 "Please be aware there may be issues associated "
9879 "with your hardware. If you are experiencing "
9880 "problems please contact your Intel or hardware "
9881 "representative who provided you with this "
9882 "hardware.\n");
794caeb2 9883 }
9a799d71
AK		 9884 strcpy(netdev->name, "eth%d");
9885 err = register_netdev(netdev);
9886 if (err)
9887 goto err_register;
9888
0fb6a55c
ET		 9889 pci_set_drvdata(pdev, adapter);
9890
ec74a471
ET		 9891 /* power down the optics for 82599 SFP+ fiber */
9892 if (hw->mac.ops.disable_tx_laser)
93d3ce8f
ET		 9893 hw->mac.ops.disable_tx_laser(hw);
9894
54386467
JB		 9895 /* carrier off reporting is important to ethtool even BEFORE open */
9896 netif_carrier_off(netdev);
9897
5dd2d332 9898#ifdef CONFIG_IXGBE_DCA
652f093f 9899 if (dca_add_requester(&pdev->dev) == 0) {
bd0362dd 9900 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
bd0362dd
JC		 9901 ixgbe_setup_dca(adapter);
9902 }
9903#endif
1cdd1ec8 9904 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
396e799c 9905 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
1cdd1ec8
GR		 9906 for (i = 0; i < adapter->num_vfs; i++)
9907 ixgbe_vf_configuration(pdev, (i | 0x10000000));
9908 }
9909
2466dd9c
JK		 9910 /* firmware requires driver version to be 0xFFFFFFFF
9911 * since os does not support feature
9912 */
9612de92 9913 if (hw->mac.ops.set_fw_drv_ver)
cb8e0514
TN		 9914 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 0xFF,
9915 sizeof(ixgbe_driver_version) - 1,
9916 ixgbe_driver_version);
9612de92 9917
0365e6e4
PW		 9918 /* add san mac addr to netdev */
9919 ixgbe_add_sanmac_netdev(netdev);
9a799d71 9920
ea81875a 9921 e_dev_info("%s\n", ixgbe_default_device_descr);
3ca8bc6d 9922
1210982b 9923#ifdef CONFIG_IXGBE_HWMON
3ca8bc6d
DS		 9924 if (ixgbe_sysfs_init(adapter))
9925 e_err(probe, "failed to allocate sysfs resources\n");
1210982b 9926#endif /* CONFIG_IXGBE_HWMON */
3ca8bc6d 9927
00949167 9928 ixgbe_dbg_adapter_init(adapter);
00949167 9929
d1a35ee2
ET		 9930 /* setup link for SFP devices with MNG FW, else wait for IXGBE_UP */
9931 if (ixgbe_mng_enabled(hw) && ixgbe_is_sfp(hw) && hw->mac.ops.setup_link)
0b2679d6
DS		 9932 hw->mac.ops.setup_link(hw,
9933 IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL,
9934 true);
9935
9a799d71
AK		 9936 return 0;
9937
9938err_register:
5eba3699 9939 ixgbe_release_hw_control(adapter);
7a921c93 9940 ixgbe_clear_interrupt_scheme(adapter);
9a799d71 9941err_sw_init:
99d74487 9942 ixgbe_disable_sriov(adapter);
7086400d 9943 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
2a1a091c 9944 iounmap(adapter->io_addr);
1cdaaf54 9945 kfree(adapter->jump_tables[0]);
5d7daa35 9946 kfree(adapter->mac_table);
9a799d71 9947err_ioremap:
b5b2ffc0 9948 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state);
9a799d71
AK		 9949 free_netdev(netdev);
9950err_alloc_etherdev:
56d766d6 9951 pci_release_mem_regions(pdev);
9a799d71
AK		 9952err_pci_reg:
9953err_dma:
b5b2ffc0 9954 if (!adapter || disable_dev)
41c62843 9955 pci_disable_device(pdev);
9a799d71
AK		 9956 return err;
9957}
9958
9959/**
9960 * ixgbe_remove - Device Removal Routine
9961 * @pdev: PCI device information struct
9962 *
9963 * ixgbe_remove is called by the PCI subsystem to alert the driver
9964 * that it should release a PCI device. The could be caused by a
9965 * Hot-Plug event, or because the driver is going to be removed from
9966 * memory.
9967 **/
9f9a12f8 9968static void ixgbe_remove(struct pci_dev *pdev)
9a799d71 9969{
c60fbb00 9970 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
0fb6a55c 9971 struct net_device *netdev;
b5b2ffc0 9972 bool disable_dev;
1cdaaf54 9973 int i;
9a799d71 9974
0fb6a55c
ET		 9975 /* if !adapter then we already cleaned up in probe */
9976 if (!adapter)
9977 return;
9978
9979 netdev = adapter->netdev;
00949167 9980 ixgbe_dbg_adapter_exit(adapter);
00949167 9981
09f40aed 9982 set_bit(__IXGBE_REMOVING, &adapter->state);
7086400d 9983 cancel_work_sync(&adapter->service_task);
9a799d71 9984
3a6a4eda 9985
5dd2d332 9986#ifdef CONFIG_IXGBE_DCA
bd0362dd
JC		 9987 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
9988 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
9989 dca_remove_requester(&pdev->dev);
9de7605e
MR		 9990 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
9991 IXGBE_DCA_CTRL_DCA_DISABLE);
bd0362dd
JC		 9992 }
9993
9994#endif
1210982b 9995#ifdef CONFIG_IXGBE_HWMON
3ca8bc6d 9996 ixgbe_sysfs_exit(adapter);
1210982b 9997#endif /* CONFIG_IXGBE_HWMON */
3ca8bc6d 9998
0365e6e4
PW		 9999 /* remove the added san mac */
10000 ixgbe_del_sanmac_netdev(netdev);
10001
da36b647 10002#ifdef CONFIG_PCI_IOV
7837e286 10003 ixgbe_disable_sriov(adapter);
da36b647 10004#endif
6b010e9b
AW		 10005 if (netdev->reg_state == NETREG_REGISTERED)
10006 unregister_netdev(netdev);
10007
7a921c93 10008 ixgbe_clear_interrupt_scheme(adapter);
5eba3699 10009
021230d4 10010 ixgbe_release_hw_control(adapter);
9a799d71 10011
2b1588c3
AD		 10012#ifdef CONFIG_DCB
10013 kfree(adapter->ixgbe_ieee_pfc);
10014 kfree(adapter->ixgbe_ieee_ets);
10015
10016#endif
2a1a091c 10017 iounmap(adapter->io_addr);
56d766d6 10018 pci_release_mem_regions(pdev);
9a799d71 10019
849c4542 10020 e_dev_info("complete\n");
021230d4 10021
1cdaaf54
AN		 10022 for (i = 0; i < IXGBE_MAX_LINK_HANDLE; i++) {
10023 if (adapter->jump_tables[i]) {
10024 kfree(adapter->jump_tables[i]->input);
10025 kfree(adapter->jump_tables[i]->mask);
10026 }
10027 kfree(adapter->jump_tables[i]);
10028 }
10029
5d7daa35 10030 kfree(adapter->mac_table);
b5b2ffc0 10031 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state);
9a799d71
AK		 10032 free_netdev(netdev);
10033
19d5afd4 10034 pci_disable_pcie_error_reporting(pdev);
6fabd715 10035
b5b2ffc0 10036 if (disable_dev)
41c62843 10037 pci_disable_device(pdev);
9a799d71
AK		 10038}
10039
10040/**
10041 * ixgbe_io_error_detected - called when PCI error is detected
10042 * @pdev: Pointer to PCI device
10043 * @state: The current pci connection state
10044 *
10045 * This function is called after a PCI bus error affecting
10046 * this device has been detected.
10047 */
10048static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
e8e9f696 10049 pci_channel_state_t state)
9a799d71 10050{
c60fbb00
AD		 10051 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
10052 struct net_device *netdev = adapter->netdev;
9a799d71 10053
83c61fa9 10054#ifdef CONFIG_PCI_IOV
14438464 10055 struct ixgbe_hw *hw = &adapter->hw;
83c61fa9
GR		 10056 struct pci_dev *bdev, *vfdev;
10057 u32 dw0, dw1, dw2, dw3;
10058 int vf, pos;
10059 u16 req_id, pf_func;
10060
10061 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
10062 adapter->num_vfs == 0)
10063 goto skip_bad_vf_detection;
10064
10065 bdev = pdev->bus->self;
62f87c0e 10066 while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT))
83c61fa9
GR		 10067 bdev = bdev->bus->self;
10068
10069 if (!bdev)
10070 goto skip_bad_vf_detection;
10071
10072 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
10073 if (!pos)
10074 goto skip_bad_vf_detection;
10075
14438464
MR		 10076 dw0 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG);
10077 dw1 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 4);
10078 dw2 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 8);
10079 dw3 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 12);
10080 if (ixgbe_removed(hw->hw_addr))
10081 goto skip_bad_vf_detection;
83c61fa9
GR		 10082
10083 req_id = dw1 >> 16;
10084 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
10085 if (!(req_id & 0x0080))
10086 goto skip_bad_vf_detection;
10087
10088 pf_func = req_id & 0x01;
10089 if ((pf_func & 1) == (pdev->devfn & 1)) {
10090 unsigned int device_id;
10091
10092 vf = (req_id & 0x7F) >> 1;
10093 e_dev_err("VF %d has caused a PCIe error\n", vf);
10094 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
10095 "%8.8x\tdw3: %8.8x\n",
10096 dw0, dw1, dw2, dw3);
10097 switch (adapter->hw.mac.type) {
10098 case ixgbe_mac_82599EB:
10099 device_id = IXGBE_82599_VF_DEVICE_ID;
10100 break;
10101 case ixgbe_mac_X540:
10102 device_id = IXGBE_X540_VF_DEVICE_ID;
10103 break;
9a75a1ac
DS		 10104 case ixgbe_mac_X550:
10105 device_id = IXGBE_DEV_ID_X550_VF;
10106 break;
10107 case ixgbe_mac_X550EM_x:
10108 device_id = IXGBE_DEV_ID_X550EM_X_VF;
10109 break;
49425dfc
MR		 10110 case ixgbe_mac_x550em_a:
10111 device_id = IXGBE_DEV_ID_X550EM_A_VF;
10112 break;
83c61fa9
GR		 10113 default:
10114 device_id = 0;
10115 break;
10116 }
10117
10118 /* Find the pci device of the offending VF */
36e90319 10119 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL);
83c61fa9
GR		 10120 while (vfdev) {
10121 if (vfdev->devfn == (req_id & 0xFF))
10122 break;
36e90319 10123 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
83c61fa9
GR		 10124 device_id, vfdev);
10125 }
10126 /*
10127 * There's a slim chance the VF could have been hot plugged,
10128 * so if it is no longer present we don't need to issue the
10129 * VFLR. Just clean up the AER in that case.
10130 */
10131 if (vfdev) {
9079e416 10132 ixgbe_issue_vf_flr(adapter, vfdev);
b4fafbe9
GR		 10133 /* Free device reference count */
10134 pci_dev_put(vfdev);
83c61fa9
GR		 10135 }
10136
10137 pci_cleanup_aer_uncorrect_error_status(pdev);
10138 }
10139
10140 /*
10141 * Even though the error may have occurred on the other port
10142 * we still need to increment the vf error reference count for
10143 * both ports because the I/O resume function will be called
10144 * for both of them.
10145 */
10146 adapter->vferr_refcount++;
10147
10148 return PCI_ERS_RESULT_RECOVERED;
10149
10150skip_bad_vf_detection:
10151#endif /* CONFIG_PCI_IOV */
58cf663f
MR		 10152 if (!test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
10153 return PCI_ERS_RESULT_DISCONNECT;
10154
41c62843 10155 rtnl_lock();
9a799d71
AK		 10156 netif_device_detach(netdev);
10157
41c62843
MR		 10158 if (state == pci_channel_io_perm_failure) {
10159 rtnl_unlock();
3044b8d1 10160 return PCI_ERS_RESULT_DISCONNECT;
41c62843 10161 }
3044b8d1 10162
9a799d71 10163 if (netif_running(netdev))
126db13f 10164 ixgbe_close_suspend(adapter);
41c62843
MR		 10165
10166 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))
10167 pci_disable_device(pdev);
10168 rtnl_unlock();
9a799d71 10169
b4617240 10170 /* Request a slot reset. */
9a799d71
AK		 10171 return PCI_ERS_RESULT_NEED_RESET;
10172}
10173
10174/**
10175 * ixgbe_io_slot_reset - called after the pci bus has been reset.
10176 * @pdev: Pointer to PCI device
10177 *
10178 * Restart the card from scratch, as if from a cold-boot.
10179 */
10180static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
10181{
c60fbb00 10182 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
6fabd715
PWJ		 10183 pci_ers_result_t result;
10184 int err;
9a799d71 10185
9ce77666 10186 if (pci_enable_device_mem(pdev)) {
396e799c 10187 e_err(probe, "Cannot re-enable PCI device after reset.\n");
6fabd715
PWJ		 10188 result = PCI_ERS_RESULT_DISCONNECT;
10189 } else {
4e857c58 10190 smp_mb__before_atomic();
41c62843 10191 clear_bit(__IXGBE_DISABLED, &adapter->state);
0391bbe3 10192 adapter->hw.hw_addr = adapter->io_addr;
6fabd715
PWJ		 10193 pci_set_master(pdev);
10194 pci_restore_state(pdev);
c0e1f68b 10195 pci_save_state(pdev);
9a799d71 10196
dd4d8ca6 10197 pci_wake_from_d3(pdev, false);
9a799d71 10198
6fabd715 10199 ixgbe_reset(adapter);
88512539 10200 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
6fabd715
PWJ		 10201 result = PCI_ERS_RESULT_RECOVERED;
10202 }
10203
10204 err = pci_cleanup_aer_uncorrect_error_status(pdev);
10205 if (err) {
849c4542
ET		 10206 e_dev_err("pci_cleanup_aer_uncorrect_error_status "
10207 "failed 0x%0x\n", err);
6fabd715
PWJ		 10208 /* non-fatal, continue */
10209 }
9a799d71 10210
6fabd715 10211 return result;
9a799d71
AK		 10212}
10213
10214/**
10215 * ixgbe_io_resume - called when traffic can start flowing again.
10216 * @pdev: Pointer to PCI device
10217 *
10218 * This callback is called when the error recovery driver tells us that
10219 * its OK to resume normal operation.
10220 */
10221static void ixgbe_io_resume(struct pci_dev *pdev)
10222{
c60fbb00
AD		 10223 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
10224 struct net_device *netdev = adapter->netdev;
9a799d71 10225
83c61fa9
GR		 10226#ifdef CONFIG_PCI_IOV
10227 if (adapter->vferr_refcount) {
10228 e_info(drv, "Resuming after VF err\n");
10229 adapter->vferr_refcount--;
10230 return;
10231 }
10232
10233#endif
126db13f 10234 rtnl_lock();
c7ccde0f 10235 if (netif_running(netdev))
126db13f 10236 ixgbe_open(netdev);
9a799d71
AK		 10237
10238 netif_device_attach(netdev);
126db13f 10239 rtnl_unlock();
9a799d71
AK		 10240}
10241
3646f0e5 10242static const struct pci_error_handlers ixgbe_err_handler = {
9a799d71
AK		 10243 .error_detected = ixgbe_io_error_detected,
10244 .slot_reset = ixgbe_io_slot_reset,
10245 .resume = ixgbe_io_resume,
10246};
10247
10248static struct pci_driver ixgbe_driver = {
10249 .name = ixgbe_driver_name,
10250 .id_table = ixgbe_pci_tbl,
10251 .probe = ixgbe_probe,
9f9a12f8 10252 .remove = ixgbe_remove,
9a799d71
AK		 10253#ifdef CONFIG_PM
10254 .suspend = ixgbe_suspend,
10255 .resume = ixgbe_resume,
10256#endif
10257 .shutdown = ixgbe_shutdown,
da36b647 10258 .sriov_configure = ixgbe_pci_sriov_configure,
9a799d71
AK		 10259 .err_handler = &ixgbe_err_handler
10260};
10261
10262/**
10263 * ixgbe_init_module - Driver Registration Routine
10264 *
10265 * ixgbe_init_module is the first routine called when the driver is
10266 * loaded. All it does is register with the PCI subsystem.
10267 **/
10268static int __init ixgbe_init_module(void)
10269{
10270 int ret;
c7689578 10271 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
849c4542 10272 pr_info("%s\n", ixgbe_copyright);
9a799d71 10273
780484d8
MR		 10274 ixgbe_wq = create_singlethread_workqueue(ixgbe_driver_name);
10275 if (!ixgbe_wq) {
10276 pr_err("%s: Failed to create workqueue\n", ixgbe_driver_name);
10277 return -ENOMEM;
10278 }
10279
00949167 10280 ixgbe_dbg_init();
00949167 10281
f01fc1a8
JK		 10282 ret = pci_register_driver(&ixgbe_driver);
10283 if (ret) {
6b836879 10284 destroy_workqueue(ixgbe_wq);
f01fc1a8 10285 ixgbe_dbg_exit();
f01fc1a8
JK		 10286 return ret;
10287 }
10288
5dd2d332 10289#ifdef CONFIG_IXGBE_DCA
bd0362dd 10290 dca_register_notify(&dca_notifier);
bd0362dd 10291#endif
5dd2d332 10292
f01fc1a8 10293 return 0;
9a799d71 10294}
b4617240 10295
9a799d71
AK		 10296module_init(ixgbe_init_module);
10297
10298/**
10299 * ixgbe_exit_module - Driver Exit Cleanup Routine
10300 *
10301 * ixgbe_exit_module is called just before the driver is removed
10302 * from memory.
10303 **/
10304static void __exit ixgbe_exit_module(void)
10305{
5dd2d332 10306#ifdef CONFIG_IXGBE_DCA
bd0362dd
JC		 10307 dca_unregister_notify(&dca_notifier);
10308#endif
9a799d71 10309 pci_unregister_driver(&ixgbe_driver);
00949167 10310
00949167 10311 ixgbe_dbg_exit();
780484d8
MR		 10312 if (ixgbe_wq) {
10313 destroy_workqueue(ixgbe_wq);
10314 ixgbe_wq = NULL;
10315 }
9a799d71 10316}
bd0362dd 10317
5dd2d332 10318#ifdef CONFIG_IXGBE_DCA
bd0362dd 10319static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
e8e9f696 10320 void *p)
bd0362dd
JC		 10321{
10322 int ret_val;
10323
10324 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
e8e9f696 10325 __ixgbe_notify_dca);
bd0362dd
JC		 10326
10327 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
10328}
b453368d 10329
5dd2d332 10330#endif /* CONFIG_IXGBE_DCA */
849c4542 10331
9a799d71
AK		 10332module_exit(ixgbe_exit_module);
10333
10334/* ixgbe_main.c */
Linux 6.x block layer and io_uring tree(s)