projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbe: Address fact that RSC was not setting GSO size for incoming frames
[linux-2.6-block.git] / drivers / net / ethernet / intel / ixgbe / ixgbe_main.c
CommitLineData
9a799d71
AK		 1/*******************************************************************************
2
3 Intel 10 Gigabit PCI Express Linux driver
94971820 4 Copyright(c) 1999 - 2012 Intel Corporation.
9a799d71
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
9a799d71
AK		 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/types.h>
29#include <linux/module.h>
30#include <linux/pci.h>
31#include <linux/netdevice.h>
32#include <linux/vmalloc.h>
33#include <linux/string.h>
34#include <linux/in.h>
a6b7a407 35#include <linux/interrupt.h>
9a799d71
AK		 36#include <linux/ip.h>
37#include <linux/tcp.h>
897ab156 38#include <linux/sctp.h>
60127865 39#include <linux/pkt_sched.h>
9a799d71 40#include <linux/ipv6.h>
5a0e3ad6 41#include <linux/slab.h>
9a799d71
AK		 42#include <net/checksum.h>
43#include <net/ip6_checksum.h>
44#include <linux/ethtool.h>
01789349 45#include <linux/if.h>
9a799d71 46#include <linux/if_vlan.h>
70c71606 47#include <linux/prefetch.h>
eacd73f7 48#include <scsi/fc/fc_fcoe.h>
9a799d71
AK		 49
50#include "ixgbe.h"
51#include "ixgbe_common.h"
ee5f784a 52#include "ixgbe_dcb_82599.h"
1cdd1ec8 53#include "ixgbe_sriov.h"
9a799d71
AK		 54
55char ixgbe_driver_name[] = "ixgbe";
9c8eb720 56static const char ixgbe_driver_string[] =
e8e9f696 57 "Intel(R) 10 Gigabit PCI Express Network Driver";
ea81875a
NP		 58char ixgbe_default_device_descr[] =
59 "Intel(R) 10 Gigabit Network Connection";
75e3d3c6 60#define MAJ 3
19d478bb
DS		 61#define MIN 6
62#define BUILD 7
75e3d3c6 63#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
a38a104d 64 __stringify(BUILD) "-k"
9c8eb720 65const char ixgbe_driver_version[] = DRV_VERSION;
a52055e0 66static const char ixgbe_copyright[] =
94971820 67 "Copyright (c) 1999-2012 Intel Corporation.";
9a799d71
AK		 68
69static const struct ixgbe_info *ixgbe_info_tbl[] = {
b4617240 70 [board_82598] = &ixgbe_82598_info,
e8e26350 71 [board_82599] = &ixgbe_82599_info,
fe15e8e1 72 [board_X540] = &ixgbe_X540_info,
9a799d71
AK		 73};
74
75/* ixgbe_pci_tbl - PCI Device ID Table
76 *
77 * Wildcard entries (PCI_ANY_ID) should come last
78 * Last entry must be all 0s
79 *
80 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
81 * Class, Class Mask, private data (not used) }
82 */
a3aa1884 83static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
54239c67
AD		 84 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
85 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
86 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
87 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
88 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
7d145282 110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
9e791e4a 111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
9a799d71
AK		 112 /* required last entry */
113 {0, }
114};
115MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
116
5dd2d332 117#ifdef CONFIG_IXGBE_DCA
bd0362dd 118static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
e8e9f696 119 void *p);
bd0362dd
JC		 120static struct notifier_block dca_notifier = {
121 .notifier_call = ixgbe_notify_dca,
122 .next = NULL,
123 .priority = 0
124};
125#endif
126
1cdd1ec8
GR		 127#ifdef CONFIG_PCI_IOV
128static unsigned int max_vfs;
129module_param(max_vfs, uint, 0);
e8e9f696
JP		 130MODULE_PARM_DESC(max_vfs,
131 "Maximum number of virtual functions to allocate per physical function");
1cdd1ec8
GR		 132#endif /* CONFIG_PCI_IOV */
133
8ef78adc
PWJ		 134static unsigned int allow_unsupported_sfp;
135module_param(allow_unsupported_sfp, uint, 0);
136MODULE_PARM_DESC(allow_unsupported_sfp,
137 "Allow unsupported and untested SFP+ modules on 82599-based adapters");
138
9a799d71
AK		 139MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
140MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
141MODULE_LICENSE("GPL");
142MODULE_VERSION(DRV_VERSION);
143
144#define DEFAULT_DEBUG_LEVEL_SHIFT 3
145
7086400d
AD		 146static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
147{
148 if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
149 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
150 schedule_work(&adapter->service_task);
151}
152
153static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
154{
155 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
156
52f33af8 157 /* flush memory to make sure state is correct before next watchdog */
7086400d
AD		 158 smp_mb__before_clear_bit();
159 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
160}
161
dcd79aeb
TI		 162struct ixgbe_reg_info {
163 u32 ofs;
164 char *name;
165};
166
167static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
168
169 /* General Registers */
170 {IXGBE_CTRL, "CTRL"},
171 {IXGBE_STATUS, "STATUS"},
172 {IXGBE_CTRL_EXT, "CTRL_EXT"},
173
174 /* Interrupt Registers */
175 {IXGBE_EICR, "EICR"},
176
177 /* RX Registers */
178 {IXGBE_SRRCTL(0), "SRRCTL"},
179 {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
180 {IXGBE_RDLEN(0), "RDLEN"},
181 {IXGBE_RDH(0), "RDH"},
182 {IXGBE_RDT(0), "RDT"},
183 {IXGBE_RXDCTL(0), "RXDCTL"},
184 {IXGBE_RDBAL(0), "RDBAL"},
185 {IXGBE_RDBAH(0), "RDBAH"},
186
187 /* TX Registers */
188 {IXGBE_TDBAL(0), "TDBAL"},
189 {IXGBE_TDBAH(0), "TDBAH"},
190 {IXGBE_TDLEN(0), "TDLEN"},
191 {IXGBE_TDH(0), "TDH"},
192 {IXGBE_TDT(0), "TDT"},
193 {IXGBE_TXDCTL(0), "TXDCTL"},
194
195 /* List Terminator */
196 {}
197};
198
199
200/*
201 * ixgbe_regdump - register printout routine
202 */
203static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
204{
205 int i = 0, j = 0;
206 char rname[16];
207 u32 regs[64];
208
209 switch (reginfo->ofs) {
210 case IXGBE_SRRCTL(0):
211 for (i = 0; i < 64; i++)
212 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
213 break;
214 case IXGBE_DCA_RXCTRL(0):
215 for (i = 0; i < 64; i++)
216 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
217 break;
218 case IXGBE_RDLEN(0):
219 for (i = 0; i < 64; i++)
220 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
221 break;
222 case IXGBE_RDH(0):
223 for (i = 0; i < 64; i++)
224 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
225 break;
226 case IXGBE_RDT(0):
227 for (i = 0; i < 64; i++)
228 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
229 break;
230 case IXGBE_RXDCTL(0):
231 for (i = 0; i < 64; i++)
232 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
233 break;
234 case IXGBE_RDBAL(0):
235 for (i = 0; i < 64; i++)
236 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
237 break;
238 case IXGBE_RDBAH(0):
239 for (i = 0; i < 64; i++)
240 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
241 break;
242 case IXGBE_TDBAL(0):
243 for (i = 0; i < 64; i++)
244 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
245 break;
246 case IXGBE_TDBAH(0):
247 for (i = 0; i < 64; i++)
248 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
249 break;
250 case IXGBE_TDLEN(0):
251 for (i = 0; i < 64; i++)
252 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
253 break;
254 case IXGBE_TDH(0):
255 for (i = 0; i < 64; i++)
256 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
257 break;
258 case IXGBE_TDT(0):
259 for (i = 0; i < 64; i++)
260 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
261 break;
262 case IXGBE_TXDCTL(0):
263 for (i = 0; i < 64; i++)
264 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
265 break;
266 default:
c7689578 267 pr_info("%-15s %08x\n", reginfo->name,
dcd79aeb
TI		 268 IXGBE_READ_REG(hw, reginfo->ofs));
269 return;
270 }
271
272 for (i = 0; i < 8; i++) {
273 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
c7689578 274 pr_err("%-15s", rname);
dcd79aeb 275 for (j = 0; j < 8; j++)
c7689578
JP		 276 pr_cont(" %08x", regs[i*8+j]);
277 pr_cont("\n");
dcd79aeb
TI		 278 }
279
280}
281
282/*
283 * ixgbe_dump - Print registers, tx-rings and rx-rings
284 */
285static void ixgbe_dump(struct ixgbe_adapter *adapter)
286{
287 struct net_device *netdev = adapter->netdev;
288 struct ixgbe_hw *hw = &adapter->hw;
289 struct ixgbe_reg_info *reginfo;
290 int n = 0;
291 struct ixgbe_ring *tx_ring;
292 struct ixgbe_tx_buffer *tx_buffer_info;
293 union ixgbe_adv_tx_desc *tx_desc;
294 struct my_u0 { u64 a; u64 b; } *u0;
295 struct ixgbe_ring *rx_ring;
296 union ixgbe_adv_rx_desc *rx_desc;
297 struct ixgbe_rx_buffer *rx_buffer_info;
298 u32 staterr;
299 int i = 0;
300
301 if (!netif_msg_hw(adapter))
302 return;
303
304 /* Print netdevice Info */
305 if (netdev) {
306 dev_info(&adapter->pdev->dev, "Net device Info\n");
c7689578 307 pr_info("Device Name state "
dcd79aeb 308 "trans_start last_rx\n");
c7689578
JP		 309 pr_info("%-15s %016lX %016lX %016lX\n",
310 netdev->name,
311 netdev->state,
312 netdev->trans_start,
313 netdev->last_rx);
dcd79aeb
TI		 314 }
315
316 /* Print Registers */
317 dev_info(&adapter->pdev->dev, "Register Dump\n");
c7689578 318 pr_info(" Register Name Value\n");
dcd79aeb
TI		 319 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
320 reginfo->name; reginfo++) {
321 ixgbe_regdump(hw, reginfo);
322 }
323
324 /* Print TX Ring Summary */
325 if (!netdev || !netif_running(netdev))
326 goto exit;
327
328 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
c7689578 329 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
dcd79aeb
TI		 330 for (n = 0; n < adapter->num_tx_queues; n++) {
331 tx_ring = adapter->tx_ring[n];
332 tx_buffer_info =
333 &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
d3d00239 334 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
dcd79aeb
TI		 335 n, tx_ring->next_to_use, tx_ring->next_to_clean,
336 (u64)tx_buffer_info->dma,
337 tx_buffer_info->length,
338 tx_buffer_info->next_to_watch,
339 (u64)tx_buffer_info->time_stamp);
340 }
341
342 /* Print TX Rings */
343 if (!netif_msg_tx_done(adapter))
344 goto rx_ring_summary;
345
346 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
347
348 /* Transmit Descriptor Formats
349 *
350 * Advanced Transmit Descriptor
351 * +--------------------------------------------------------------+
352 * 0 | Buffer Address [63:0] |
353 * +--------------------------------------------------------------+
354 * 8 | PAYLEN | PORTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
355 * +--------------------------------------------------------------+
356 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
357 */
358
359 for (n = 0; n < adapter->num_tx_queues; n++) {
360 tx_ring = adapter->tx_ring[n];
c7689578
JP		 361 pr_info("------------------------------------\n");
362 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
363 pr_info("------------------------------------\n");
364 pr_info("T [desc] [address 63:0 ] "
dcd79aeb
TI		 365 "[PlPOIdStDDt Ln] [bi->dma ] "
366 "leng ntw timestamp bi->skb\n");
367
368 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
31f05a2d 369 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
dcd79aeb
TI		 370 tx_buffer_info = &tx_ring->tx_buffer_info[i];
371 u0 = (struct my_u0 *)tx_desc;
c7689578 372 pr_info("T [0x%03X] %016llX %016llX %016llX"
d3d00239 373 " %04X %p %016llX %p", i,
dcd79aeb
TI		 374 le64_to_cpu(u0->a),
375 le64_to_cpu(u0->b),
376 (u64)tx_buffer_info->dma,
377 tx_buffer_info->length,
378 tx_buffer_info->next_to_watch,
379 (u64)tx_buffer_info->time_stamp,
380 tx_buffer_info->skb);
381 if (i == tx_ring->next_to_use &&
382 i == tx_ring->next_to_clean)
c7689578 383 pr_cont(" NTC/U\n");
dcd79aeb 384 else if (i == tx_ring->next_to_use)
c7689578 385 pr_cont(" NTU\n");
dcd79aeb 386 else if (i == tx_ring->next_to_clean)
c7689578 387 pr_cont(" NTC\n");
dcd79aeb 388 else
c7689578 389 pr_cont("\n");
dcd79aeb
TI		 390
391 if (netif_msg_pktdata(adapter) &&
392 tx_buffer_info->dma != 0)
393 print_hex_dump(KERN_INFO, "",
394 DUMP_PREFIX_ADDRESS, 16, 1,
395 phys_to_virt(tx_buffer_info->dma),
396 tx_buffer_info->length, true);
397 }
398 }
399
400 /* Print RX Rings Summary */
401rx_ring_summary:
402 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
c7689578 403 pr_info("Queue [NTU] [NTC]\n");
dcd79aeb
TI		 404 for (n = 0; n < adapter->num_rx_queues; n++) {
405 rx_ring = adapter->rx_ring[n];
c7689578
JP		 406 pr_info("%5d %5X %5X\n",
407 n, rx_ring->next_to_use, rx_ring->next_to_clean);
dcd79aeb
TI		 408 }
409
410 /* Print RX Rings */
411 if (!netif_msg_rx_status(adapter))
412 goto exit;
413
414 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
415
416 /* Advanced Receive Descriptor (Read) Format
417 * 63 1 0
418 * +-----------------------------------------------------+
419 * 0 | Packet Buffer Address [63:1] |A0/NSE|
420 * +----------------------------------------------+------+
421 * 8 | Header Buffer Address [63:1] | DD |
422 * +-----------------------------------------------------+
423 *
424 *
425 * Advanced Receive Descriptor (Write-Back) Format
426 *
427 * 63 48 47 32 31 30 21 20 16 15 4 3 0
428 * +------------------------------------------------------+
429 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
430 * | Checksum Ident | | | | Type | Type |
431 * +------------------------------------------------------+
432 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
433 * +------------------------------------------------------+
434 * 63 48 47 32 31 20 19 0
435 */
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
c7689578
JP		 438 pr_info("------------------------------------\n");
439 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
440 pr_info("------------------------------------\n");
441 pr_info("R [desc] [ PktBuf A0] "
dcd79aeb
TI		 442 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
443 "<-- Adv Rx Read format\n");
c7689578 444 pr_info("RWB[desc] [PcsmIpSHl PtRs] "
dcd79aeb
TI		 445 "[vl er S cks ln] ---------------- [bi->skb] "
446 "<-- Adv Rx Write-Back format\n");
447
448 for (i = 0; i < rx_ring->count; i++) {
449 rx_buffer_info = &rx_ring->rx_buffer_info[i];
31f05a2d 450 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
dcd79aeb
TI		 451 u0 = (struct my_u0 *)rx_desc;
452 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
453 if (staterr & IXGBE_RXD_STAT_DD) {
454 /* Descriptor Done */
c7689578 455 pr_info("RWB[0x%03X] %016llX "
dcd79aeb
TI		 456 "%016llX ---------------- %p", i,
457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 rx_buffer_info->skb);
460 } else {
c7689578 461 pr_info("R [0x%03X] %016llX "
dcd79aeb
TI		 462 "%016llX %016llX %p", i,
463 le64_to_cpu(u0->a),
464 le64_to_cpu(u0->b),
465 (u64)rx_buffer_info->dma,
466 rx_buffer_info->skb);
467
468 if (netif_msg_pktdata(adapter)) {
469 print_hex_dump(KERN_INFO, "",
470 DUMP_PREFIX_ADDRESS, 16, 1,
471 phys_to_virt(rx_buffer_info->dma),
472 rx_ring->rx_buf_len, true);
473
474 if (rx_ring->rx_buf_len
919e78a6 475 < IXGBE_RXBUFFER_2K)
dcd79aeb
TI		 476 print_hex_dump(KERN_INFO, "",
477 DUMP_PREFIX_ADDRESS, 16, 1,
478 phys_to_virt(
479 rx_buffer_info->page_dma +
480 rx_buffer_info->page_offset
481 ),
482 PAGE_SIZE/2, true);
483 }
484 }
485
486 if (i == rx_ring->next_to_use)
c7689578 487 pr_cont(" NTU\n");
dcd79aeb 488 else if (i == rx_ring->next_to_clean)
c7689578 489 pr_cont(" NTC\n");
dcd79aeb 490 else
c7689578 491 pr_cont("\n");
dcd79aeb
TI		 492
493 }
494 }
495
496exit:
497 return;
498}
499
5eba3699
AV		 500static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
501{
502 u32 ctrl_ext;
503
504 /* Let firmware take over control of h/w */
505 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
506 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
e8e9f696 507 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
5eba3699
AV		 508}
509
510static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
511{
512 u32 ctrl_ext;
513
514 /* Let firmware know the driver has taken over */
515 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
516 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
e8e9f696 517 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
5eba3699 518}
9a799d71 519
e8e26350
PW		 520/*
521 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
522 * @adapter: pointer to adapter struct
523 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
524 * @queue: queue to map the corresponding interrupt to
525 * @msix_vector: the vector to map to the corresponding queue
526 *
527 */
528static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
e8e9f696 529 u8 queue, u8 msix_vector)
9a799d71
AK		 530{
531 u32 ivar, index;
e8e26350
PW		 532 struct ixgbe_hw *hw = &adapter->hw;
533 switch (hw->mac.type) {
534 case ixgbe_mac_82598EB:
535 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
536 if (direction == -1)
537 direction = 0;
538 index = (((direction * 64) + queue) >> 2) & 0x1F;
539 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
540 ivar &= ~(0xFF << (8 * (queue & 0x3)));
541 ivar |= (msix_vector << (8 * (queue & 0x3)));
542 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
543 break;
544 case ixgbe_mac_82599EB:
b93a2226 545 case ixgbe_mac_X540:
e8e26350
PW		 546 if (direction == -1) {
547 /* other causes */
548 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
549 index = ((queue & 1) * 8);
550 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
551 ivar &= ~(0xFF << index);
552 ivar |= (msix_vector << index);
553 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
554 break;
555 } else {
556 /* tx or rx causes */
557 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
558 index = ((16 * (queue & 1)) + (8 * direction));
559 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
560 ivar &= ~(0xFF << index);
561 ivar |= (msix_vector << index);
562 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
563 break;
564 }
565 default:
566 break;
567 }
9a799d71
AK		 568}
569
fe49f04a 570static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
e8e9f696 571 u64 qmask)
fe49f04a
AD		 572{
573 u32 mask;
574
bd508178
AD		 575 switch (adapter->hw.mac.type) {
576 case ixgbe_mac_82598EB:
fe49f04a
AD		 577 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
578 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
bd508178
AD		 579 break;
580 case ixgbe_mac_82599EB:
b93a2226 581 case ixgbe_mac_X540:
fe49f04a
AD		 582 mask = (qmask & 0xFFFFFFFF);
583 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
584 mask = (qmask >> 32);
585 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
bd508178
AD		 586 break;
587 default:
588 break;
fe49f04a
AD		 589 }
590}
591
d3d00239
AD		 592static inline void ixgbe_unmap_tx_resource(struct ixgbe_ring *ring,
593 struct ixgbe_tx_buffer *tx_buffer)
9a799d71 594{
d3d00239
AD		 595 if (tx_buffer->dma) {
596 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_MAPPED_AS_PAGE)
597 dma_unmap_page(ring->dev,
598 tx_buffer->dma,
599 tx_buffer->length,
600 DMA_TO_DEVICE);
e5a43549 601 else
d3d00239
AD		 602 dma_unmap_single(ring->dev,
603 tx_buffer->dma,
604 tx_buffer->length,
605 DMA_TO_DEVICE);
e5a43549 606 }
d3d00239
AD		 607 tx_buffer->dma = 0;
608}
609
610void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
611 struct ixgbe_tx_buffer *tx_buffer_info)
612{
613 ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
614 if (tx_buffer_info->skb)
9a799d71 615 dev_kfree_skb_any(tx_buffer_info->skb);
d3d00239 616 tx_buffer_info->skb = NULL;
9a799d71
AK		 617 /* tx_buffer_info must be completely set up in the transmit path */
618}
619
c84d324c
JF		 620static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
621{
622 struct ixgbe_hw *hw = &adapter->hw;
623 struct ixgbe_hw_stats *hwstats = &adapter->stats;
624 u32 data = 0;
625 u32 xoff[8] = {0};
626 int i;
627
628 if ((hw->fc.current_mode == ixgbe_fc_full) ||
629 (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
630 switch (hw->mac.type) {
631 case ixgbe_mac_82598EB:
632 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
6837e895
PW		 633 break;
634 default:
c84d324c
JF		 635 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
636 }
637 hwstats->lxoffrxc += data;
638
639 /* refill credits (no tx hang) if we received xoff */
640 if (!data)
641 return;
642
643 for (i = 0; i < adapter->num_tx_queues; i++)
644 clear_bit(__IXGBE_HANG_CHECK_ARMED,
645 &adapter->tx_ring[i]->state);
646 return;
647 } else if (!(adapter->dcb_cfg.pfc_mode_enable))
648 return;
649
650 /* update stats for each tc, only valid with PFC enabled */
651 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
652 switch (hw->mac.type) {
653 case ixgbe_mac_82598EB:
654 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
bd508178 655 break;
c84d324c
JF		 656 default:
657 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
26f23d82 658 }
c84d324c
JF		 659 hwstats->pxoffrxc[i] += xoff[i];
660 }
661
662 /* disarm tx queues that have received xoff frames */
663 for (i = 0; i < adapter->num_tx_queues; i++) {
664 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
fb5475ff 665 u8 tc = tx_ring->dcb_tc;
c84d324c
JF		 666
667 if (xoff[tc])
668 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
26f23d82 669 }
26f23d82
YZ		 670}
671
c84d324c 672static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
9a799d71 673{
c84d324c
JF		 674 return ring->tx_stats.completed;
675}
676
677static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
678{
679 struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
e01c31a5 680 struct ixgbe_hw *hw = &adapter->hw;
e01c31a5 681
c84d324c
JF		 682 u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
683 u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
684
685 if (head != tail)
686 return (head < tail) ?
687 tail - head : (tail + ring->count - head);
688
689 return 0;
690}
691
692static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
693{
694 u32 tx_done = ixgbe_get_tx_completed(tx_ring);
695 u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
696 u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
697 bool ret = false;
698
7d637bcc 699 clear_check_for_tx_hang(tx_ring);
c84d324c
JF		 700
701 /*
702 * Check for a hung queue, but be thorough. This verifies
703 * that a transmit has been completed since the previous
704 * check AND there is at least one packet pending. The
705 * ARMED bit is set to indicate a potential hang. The
706 * bit is cleared if a pause frame is received to remove
707 * false hang detection due to PFC or 802.3x frames. By
708 * requiring this to fail twice we avoid races with
709 * pfc clearing the ARMED bit and conditions where we
710 * run the check_tx_hang logic with a transmit completion
711 * pending but without time to complete it yet.
712 */
713 if ((tx_done_old == tx_done) && tx_pending) {
714 /* make sure it is true for two checks in a row */
715 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
716 &tx_ring->state);
717 } else {
718 /* update completed stats and continue */
719 tx_ring->tx_stats.tx_done_old = tx_done;
720 /* reset the countdown */
721 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
9a799d71
AK		 722 }
723
c84d324c 724 return ret;
9a799d71
AK		 725}
726
c83c6cbd
AD		 727/**
728 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
729 * @adapter: driver private struct
730 **/
731static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
732{
733
734 /* Do the reset outside of interrupt context */
735 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
736 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
737 ixgbe_service_event_schedule(adapter);
738 }
739}
e01c31a5 740
9a799d71
AK		 741/**
742 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
fe49f04a 743 * @q_vector: structure containing interrupt and ring information
e01c31a5 744 * @tx_ring: tx ring to clean
9a799d71 745 **/
fe49f04a 746static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
e8e9f696 747 struct ixgbe_ring *tx_ring)
9a799d71 748{
fe49f04a 749 struct ixgbe_adapter *adapter = q_vector->adapter;
d3d00239
AD		 750 struct ixgbe_tx_buffer *tx_buffer;
751 union ixgbe_adv_tx_desc *tx_desc;
e01c31a5 752 unsigned int total_bytes = 0, total_packets = 0;
59224555 753 unsigned int budget = q_vector->tx.work_limit;
d3d00239 754 u16 i = tx_ring->next_to_clean;
9a799d71 755
d3d00239
AD		 756 tx_buffer = &tx_ring->tx_buffer_info[i];
757 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
12207e49 758
30065e63 759 for (; budget; budget--) {
d3d00239
AD		 760 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
761
762 /* if next_to_watch is not set then there is no work pending */
763 if (!eop_desc)
764 break;
765
766 /* if DD is not set pending work has not been completed */
767 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
768 break;
8ad494b0 769
d3d00239
AD		 770 /* count the packet as being completed */
771 tx_ring->tx_stats.completed++;
772
773 /* clear next_to_watch to prevent false hangs */
774 tx_buffer->next_to_watch = NULL;
8ad494b0 775
d3d00239
AD		 776 /* prevent any other reads prior to eop_desc being verified */
777 rmb();
778
779 do {
780 ixgbe_unmap_tx_resource(tx_ring, tx_buffer);
8ad494b0 781 tx_desc->wb.status = 0;
d3d00239
AD		 782 if (likely(tx_desc == eop_desc)) {
783 eop_desc = NULL;
784 dev_kfree_skb_any(tx_buffer->skb);
785 tx_buffer->skb = NULL;
786
787 total_bytes += tx_buffer->bytecount;
788 total_packets += tx_buffer->gso_segs;
789 }
9a799d71 790
d3d00239
AD		 791 tx_buffer++;
792 tx_desc++;
8ad494b0 793 i++;
d3d00239 794 if (unlikely(i == tx_ring->count)) {
8ad494b0 795 i = 0;
e01c31a5 796
d3d00239
AD		 797 tx_buffer = tx_ring->tx_buffer_info;
798 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
e092be60 799 }
e01c31a5 800
d3d00239 801 } while (eop_desc);
12207e49
PWJ		 802 }
803
9a799d71 804 tx_ring->next_to_clean = i;
d3d00239 805 u64_stats_update_begin(&tx_ring->syncp);
b953799e 806 tx_ring->stats.bytes += total_bytes;
bd198058 807 tx_ring->stats.packets += total_packets;
d3d00239 808 u64_stats_update_end(&tx_ring->syncp);
bd198058
AD		 809 q_vector->tx.total_bytes += total_bytes;
810 q_vector->tx.total_packets += total_packets;
b953799e 811
c84d324c
JF		 812 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
813 /* schedule immediate reset if we believe we hung */
814 struct ixgbe_hw *hw = &adapter->hw;
d3d00239 815 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
c84d324c
JF		 816 e_err(drv, "Detected Tx Unit Hang\n"
817 " Tx Queue <%d>\n"
818 " TDH, TDT <%x>, <%x>\n"
819 " next_to_use <%x>\n"
820 " next_to_clean <%x>\n"
821 "tx_buffer_info[next_to_clean]\n"
822 " time_stamp <%lx>\n"
823 " jiffies <%lx>\n",
824 tx_ring->queue_index,
825 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
826 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
d3d00239
AD		 827 tx_ring->next_to_use, i,
828 tx_ring->tx_buffer_info[i].time_stamp, jiffies);
c84d324c
JF		 829
830 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
831
832 e_info(probe,
833 "tx hang %d detected on queue %d, resetting adapter\n",
834 adapter->tx_timeout_count + 1, tx_ring->queue_index);
835
b953799e 836 /* schedule immediate reset if we believe we hung */
c83c6cbd 837 ixgbe_tx_timeout_reset(adapter);
b953799e
AD		 838
839 /* the adapter is about to reset, no point in enabling stuff */
59224555 840 return true;
b953799e 841 }
9a799d71 842
e092be60 843#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
30065e63 844 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
7d4987de 845 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
e092be60
AV		 846 /* Make sure that anybody stopping the queue after this
847 * sees the new next_to_clean.
848 */
849 smp_mb();
fc77dc3c 850 if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
30eba97a 851 !test_bit(__IXGBE_DOWN, &adapter->state)) {
fc77dc3c 852 netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
5b7da515 853 ++tx_ring->tx_stats.restart_queue;
30eba97a 854 }
e092be60 855 }
9a799d71 856
59224555 857 return !!budget;
9a799d71
AK		 858}
859
5dd2d332 860#ifdef CONFIG_IXGBE_DCA
bd0362dd 861static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
33cf09c9
AD		 862 struct ixgbe_ring *rx_ring,
863 int cpu)
bd0362dd 864{
33cf09c9 865 struct ixgbe_hw *hw = &adapter->hw;
bd0362dd 866 u32 rxctrl;
33cf09c9
AD		 867 u8 reg_idx = rx_ring->reg_idx;
868
869 rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
870 switch (hw->mac.type) {
871 case ixgbe_mac_82598EB:
872 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
263a84e7 873 rxctrl |= dca3_get_tag(rx_ring->dev, cpu);
33cf09c9
AD		 874 break;
875 case ixgbe_mac_82599EB:
b93a2226 876 case ixgbe_mac_X540:
33cf09c9 877 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
263a84e7 878 rxctrl |= (dca3_get_tag(rx_ring->dev, cpu) <<
33cf09c9
AD		 879 IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
880 break;
881 default:
882 break;
bd0362dd 883 }
33cf09c9
AD		 884 rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
885 rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
886 rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
33cf09c9 887 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
bd0362dd
JC		 888}
889
890static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
33cf09c9
AD		 891 struct ixgbe_ring *tx_ring,
892 int cpu)
bd0362dd 893{
33cf09c9 894 struct ixgbe_hw *hw = &adapter->hw;
bd0362dd 895 u32 txctrl;
33cf09c9
AD		 896 u8 reg_idx = tx_ring->reg_idx;
897
898 switch (hw->mac.type) {
899 case ixgbe_mac_82598EB:
900 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
901 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
263a84e7 902 txctrl |= dca3_get_tag(tx_ring->dev, cpu);
33cf09c9 903 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
33cf09c9
AD		 904 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
905 break;
906 case ixgbe_mac_82599EB:
b93a2226 907 case ixgbe_mac_X540:
33cf09c9
AD		 908 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
909 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
263a84e7 910 txctrl |= (dca3_get_tag(tx_ring->dev, cpu) <<
33cf09c9
AD		 911 IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
912 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
33cf09c9
AD		 913 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
914 break;
915 default:
916 break;
917 }
918}
919
920static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
921{
922 struct ixgbe_adapter *adapter = q_vector->adapter;
efe3d3c8 923 struct ixgbe_ring *ring;
bd0362dd 924 int cpu = get_cpu();
bd0362dd 925
33cf09c9
AD		 926 if (q_vector->cpu == cpu)
927 goto out_no_update;
928
efe3d3c8
AD		 929 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
930 ixgbe_update_tx_dca(adapter, ring, cpu);
33cf09c9 931
efe3d3c8
AD		 932 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
933 ixgbe_update_rx_dca(adapter, ring, cpu);
33cf09c9
AD		 934
935 q_vector->cpu = cpu;
936out_no_update:
bd0362dd
JC		 937 put_cpu();
938}
939
940static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
941{
33cf09c9 942 int num_q_vectors;
bd0362dd
JC		 943 int i;
944
945 if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
946 return;
947
e35ec126
AD		 948 /* always use CB2 mode, difference is masked in the CB driver */
949 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
950
33cf09c9
AD		 951 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
952 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
953 else
954 num_q_vectors = 1;
955
956 for (i = 0; i < num_q_vectors; i++) {
957 adapter->q_vector[i]->cpu = -1;
958 ixgbe_update_dca(adapter->q_vector[i]);
bd0362dd
JC		 959 }
960}
961
962static int __ixgbe_notify_dca(struct device *dev, void *data)
963{
c60fbb00 964 struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
bd0362dd
JC		 965 unsigned long event = *(unsigned long *)data;
966
2a72c31e 967 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
33cf09c9
AD		 968 return 0;
969
bd0362dd
JC		 970 switch (event) {
971 case DCA_PROVIDER_ADD:
96b0e0f6
JB		 972 /* if we're already enabled, don't do it again */
973 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
974 break;
652f093f 975 if (dca_add_requester(dev) == 0) {
96b0e0f6 976 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
bd0362dd
JC		 977 ixgbe_setup_dca(adapter);
978 break;
979 }
980 /* Fall Through since DCA is disabled. */
981 case DCA_PROVIDER_REMOVE:
982 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
983 dca_remove_requester(dev);
984 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
985 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
986 }
987 break;
988 }
989
652f093f 990 return 0;
bd0362dd 991}
5dd2d332 992#endif /* CONFIG_IXGBE_DCA */
67a74ee2
ET		 993
994static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
995 struct sk_buff *skb)
996{
997 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
998}
999
ff886dfc
AD		 1000/**
1001 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
1002 * @adapter: address of board private structure
1003 * @rx_desc: advanced rx descriptor
1004 *
1005 * Returns : true if it is FCoE pkt
1006 */
1007static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
1008 union ixgbe_adv_rx_desc *rx_desc)
1009{
1010 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1011
1012 return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
1013 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1014 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1015 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1016}
1017
9a799d71
AK		 1018/**
1019 * ixgbe_receive_skb - Send a completed packet up the stack
1020 * @adapter: board private structure
1021 * @skb: packet to send up
177db6ff
MC		 1022 * @status: hardware indication of status of receive
1023 * @rx_ring: rx descriptor ring (for a specific queue) to setup
1024 * @rx_desc: rx descriptor
9a799d71 1025 **/
78b6f4ce 1026static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
e8e9f696
JP		 1027 struct sk_buff *skb, u8 status,
1028 struct ixgbe_ring *ring,
1029 union ixgbe_adv_rx_desc *rx_desc)
9a799d71 1030{
78b6f4ce
HX		 1031 struct ixgbe_adapter *adapter = q_vector->adapter;
1032 struct napi_struct *napi = &q_vector->napi;
177db6ff
MC		 1033 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
1034 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
9a799d71 1035
f62bbb5e
JG		 1036 if (is_vlan && (tag & VLAN_VID_MASK))
1037 __vlan_hwaccel_put_tag(skb, tag);
1038
1039 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1040 napi_gro_receive(napi, skb);
1041 else
1042 netif_rx(skb);
9a799d71
AK		 1043}
1044
e59bd25d
AV		 1045/**
1046 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1047 * @adapter: address of board private structure
1048 * @status_err: hardware indication of status of receive
1049 * @skb: skb currently being received and modified
ff886dfc 1050 * @status_err: status error value of last descriptor in packet
e59bd25d 1051 **/
9a799d71 1052static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
8bae1b2b 1053 union ixgbe_adv_rx_desc *rx_desc,
ff886dfc
AD		 1054 struct sk_buff *skb,
1055 u32 status_err)
9a799d71 1056{
ff886dfc 1057 skb->ip_summed = CHECKSUM_NONE;
9a799d71 1058
712744be
JB		 1059 /* Rx csum disabled */
1060 if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
9a799d71 1061 return;
e59bd25d
AV		 1062
1063 /* if IP and error */
1064 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
1065 (status_err & IXGBE_RXDADV_ERR_IPE)) {
9a799d71
AK		 1066 adapter->hw_csum_rx_error++;
1067 return;
1068 }
e59bd25d
AV		 1069
1070 if (!(status_err & IXGBE_RXD_STAT_L4CS))
1071 return;
1072
1073 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
8bae1b2b
DS		 1074 u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1075
1076 /*
1077 * 82599 errata, UDP frames with a 0 checksum can be marked as
1078 * checksum errors.
1079 */
1080 if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1081 (adapter->hw.mac.type == ixgbe_mac_82599EB))
1082 return;
1083
e59bd25d
AV		 1084 adapter->hw_csum_rx_error++;
1085 return;
1086 }
1087
9a799d71 1088 /* It must be a TCP or UDP packet with a valid checksum */
e59bd25d 1089 skb->ip_summed = CHECKSUM_UNNECESSARY;
9a799d71
AK		 1090}
1091
84ea2591 1092static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
e8e26350
PW		 1093{
1094 /*
1095 * Force memory writes to complete before letting h/w
1096 * know there are new descriptors to fetch. (Only
1097 * applicable for weak-ordered memory model archs,
1098 * such as IA-64).
1099 */
1100 wmb();
84ea2591 1101 writel(val, rx_ring->tail);
e8e26350
PW		 1102}
1103
9a799d71
AK		 1104/**
1105 * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
fc77dc3c
AD		 1106 * @rx_ring: ring to place buffers on
1107 * @cleaned_count: number of buffers to replace
9a799d71 1108 **/
fc77dc3c 1109void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
9a799d71 1110{
9a799d71 1111 union ixgbe_adv_rx_desc *rx_desc;
3a581073 1112 struct ixgbe_rx_buffer *bi;
d5f398ed
AD		 1113 struct sk_buff *skb;
1114 u16 i = rx_ring->next_to_use;
9a799d71 1115
fc77dc3c
AD		 1116 /* do nothing if no valid netdev defined */
1117 if (!rx_ring->netdev)
1118 return;
1119
9a799d71 1120 while (cleaned_count--) {
31f05a2d 1121 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
d5f398ed
AD		 1122 bi = &rx_ring->rx_buffer_info[i];
1123 skb = bi->skb;
9a799d71 1124
d5f398ed 1125 if (!skb) {
fc77dc3c 1126 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
d5f398ed 1127 rx_ring->rx_buf_len);
9a799d71 1128 if (!skb) {
5b7da515 1129 rx_ring->rx_stats.alloc_rx_buff_failed++;
9a799d71
AK		 1130 goto no_buffers;
1131 }
d716a7d8
AD		 1132 /* initialize queue mapping */
1133 skb_record_rx_queue(skb, rx_ring->queue_index);
d5f398ed 1134 bi->skb = skb;
d716a7d8 1135 }
9a799d71 1136
d716a7d8 1137 if (!bi->dma) {
b6ec895e 1138 bi->dma = dma_map_single(rx_ring->dev,
d5f398ed 1139 skb->data,
e8e9f696 1140 rx_ring->rx_buf_len,
1b507730 1141 DMA_FROM_DEVICE);
b6ec895e 1142 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
5b7da515 1143 rx_ring->rx_stats.alloc_rx_buff_failed++;
d5f398ed
AD		 1144 bi->dma = 0;
1145 goto no_buffers;
1146 }
9a799d71 1147 }
d5f398ed 1148
7d637bcc 1149 if (ring_is_ps_enabled(rx_ring)) {
d5f398ed 1150 if (!bi->page) {
1f2149c1 1151 bi->page = alloc_page(GFP_ATOMIC | __GFP_COLD);
d5f398ed 1152 if (!bi->page) {
5b7da515 1153 rx_ring->rx_stats.alloc_rx_page_failed++;
d5f398ed
AD		 1154 goto no_buffers;
1155 }
1156 }
1157
1158 if (!bi->page_dma) {
1159 /* use a half page if we're re-using */
1160 bi->page_offset ^= PAGE_SIZE / 2;
b6ec895e 1161 bi->page_dma = dma_map_page(rx_ring->dev,
d5f398ed
AD		 1162 bi->page,
1163 bi->page_offset,
1164 PAGE_SIZE / 2,
1165 DMA_FROM_DEVICE);
b6ec895e 1166 if (dma_mapping_error(rx_ring->dev,
d5f398ed 1167 bi->page_dma)) {
5b7da515 1168 rx_ring->rx_stats.alloc_rx_page_failed++;
d5f398ed
AD		 1169 bi->page_dma = 0;
1170 goto no_buffers;
1171 }
1172 }
1173
1174 /* Refresh the desc even if buffer_addrs didn't change
1175 * because each write-back erases this info. */
3a581073
JB		 1176 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1177 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9a799d71 1178 } else {
3a581073 1179 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
84418e3b 1180 rx_desc->read.hdr_addr = 0;
9a799d71
AK		 1181 }
1182
1183 i++;
1184 if (i == rx_ring->count)
1185 i = 0;
9a799d71 1186 }
7c6e0a43 1187
9a799d71
AK		 1188no_buffers:
1189 if (rx_ring->next_to_use != i) {
1190 rx_ring->next_to_use = i;
84ea2591 1191 ixgbe_release_rx_desc(rx_ring, i);
9a799d71
AK		 1192 }
1193}
1194
c267fc16 1195static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
7c6e0a43 1196{
c267fc16
AD		 1197 /* HW will not DMA in data larger than the given buffer, even if it
1198 * parses the (NFS, of course) header to be larger. In that case, it
1199 * fills the header buffer and spills the rest into the page.
1200 */
1201 u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
1202 u16 hlen = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
1203 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
1204 if (hlen > IXGBE_RX_HDR_SIZE)
1205 hlen = IXGBE_RX_HDR_SIZE;
1206 return hlen;
7c6e0a43
JB		 1207}
1208
f8212f97 1209/**
4c1975d7
AD		 1210 * ixgbe_merge_active_tail - merge active tail into lro skb
1211 * @tail: pointer to active tail in frag_list
f8212f97 1212 *
4c1975d7
AD		 1213 * This function merges the length and data of an active tail into the
1214 * skb containing the frag_list. It resets the tail's pointer to the head,
1215 * but it leaves the heads pointer to tail intact.
f8212f97 1216 **/
4c1975d7 1217static inline struct sk_buff *ixgbe_merge_active_tail(struct sk_buff *tail)
f8212f97 1218{
4c1975d7 1219 struct sk_buff *head = IXGBE_CB(tail)->head;
f8212f97 1220
4c1975d7
AD		 1221 if (!head)
1222 return tail;
1223
1224 head->len += tail->len;
1225 head->data_len += tail->len;
1226 head->truesize += tail->len;
1227
1228 IXGBE_CB(tail)->head = NULL;
1229
1230 return head;
1231}
1232
1233/**
1234 * ixgbe_add_active_tail - adds an active tail into the skb frag_list
1235 * @head: pointer to the start of the skb
1236 * @tail: pointer to active tail to add to frag_list
1237 *
1238 * This function adds an active tail to the end of the frag list. This tail
1239 * will still be receiving data so we cannot yet ad it's stats to the main
1240 * skb. That is done via ixgbe_merge_active_tail.
1241 **/
1242static inline void ixgbe_add_active_tail(struct sk_buff *head,
1243 struct sk_buff *tail)
1244{
1245 struct sk_buff *old_tail = IXGBE_CB(head)->tail;
1246
1247 if (old_tail) {
1248 ixgbe_merge_active_tail(old_tail);
1249 old_tail->next = tail;
1250 } else {
1251 skb_shinfo(head)->frag_list = tail;
f8212f97
AD		 1252 }
1253
4c1975d7
AD		 1254 IXGBE_CB(tail)->head = head;
1255 IXGBE_CB(head)->tail = tail;
1256}
1257
1258/**
1259 * ixgbe_close_active_frag_list - cleanup pointers on a frag_list skb
1260 * @head: pointer to head of an active frag list
1261 *
1262 * This function will clear the frag_tail_tracker pointer on an active
1263 * frag_list and returns true if the pointer was actually set
1264 **/
1265static inline bool ixgbe_close_active_frag_list(struct sk_buff *head)
1266{
1267 struct sk_buff *tail = IXGBE_CB(head)->tail;
1268
1269 if (!tail)
1270 return false;
1271
1272 ixgbe_merge_active_tail(tail);
1273
1274 IXGBE_CB(head)->tail = NULL;
aa80175a 1275
4c1975d7 1276 return true;
f8212f97
AD		 1277}
1278
1d2024f6
AD		 1279/**
1280 * ixgbe_get_headlen - determine size of header for RSC/LRO/GRO/FCOE
1281 * @data: pointer to the start of the headers
1282 * @max_len: total length of section to find headers in
1283 *
1284 * This function is meant to determine the length of headers that will
1285 * be recognized by hardware for LRO, GRO, and RSC offloads. The main
1286 * motivation of doing this is to only perform one pull for IPv4 TCP
1287 * packets so that we can do basic things like calculating the gso_size
1288 * based on the average data per packet.
1289 **/
1290static unsigned int ixgbe_get_headlen(unsigned char *data,
1291 unsigned int max_len)
1292{
1293 union {
1294 unsigned char *network;
1295 /* l2 headers */
1296 struct ethhdr *eth;
1297 struct vlan_hdr *vlan;
1298 /* l3 headers */
1299 struct iphdr *ipv4;
1300 } hdr;
1301 __be16 protocol;
1302 u8 nexthdr = 0; /* default to not TCP */
1303 u8 hlen;
1304
1305 /* this should never happen, but better safe than sorry */
1306 if (max_len < ETH_HLEN)
1307 return max_len;
1308
1309 /* initialize network frame pointer */
1310 hdr.network = data;
1311
1312 /* set first protocol and move network header forward */
1313 protocol = hdr.eth->h_proto;
1314 hdr.network += ETH_HLEN;
1315
1316 /* handle any vlan tag if present */
1317 if (protocol == __constant_htons(ETH_P_8021Q)) {
1318 if ((hdr.network - data) > (max_len - VLAN_HLEN))
1319 return max_len;
1320
1321 protocol = hdr.vlan->h_vlan_encapsulated_proto;
1322 hdr.network += VLAN_HLEN;
1323 }
1324
1325 /* handle L3 protocols */
1326 if (protocol == __constant_htons(ETH_P_IP)) {
1327 if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
1328 return max_len;
1329
1330 /* access ihl as a u8 to avoid unaligned access on ia64 */
1331 hlen = (hdr.network[0] & 0x0F) << 2;
1332
1333 /* verify hlen meets minimum size requirements */
1334 if (hlen < sizeof(struct iphdr))
1335 return hdr.network - data;
1336
1337 /* record next protocol */
1338 nexthdr = hdr.ipv4->protocol;
1339 hdr.network += hlen;
1340#ifdef CONFIG_FCOE
1341 } else if (protocol == __constant_htons(ETH_P_FCOE)) {
1342 if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN))
1343 return max_len;
1344 hdr.network += FCOE_HEADER_LEN;
1345#endif
1346 } else {
1347 return hdr.network - data;
1348 }
1349
1350 /* finally sort out TCP */
1351 if (nexthdr == IPPROTO_TCP) {
1352 if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
1353 return max_len;
1354
1355 /* access doff as a u8 to avoid unaligned access on ia64 */
1356 hlen = (hdr.network[12] & 0xF0) >> 2;
1357
1358 /* verify hlen meets minimum size requirements */
1359 if (hlen < sizeof(struct tcphdr))
1360 return hdr.network - data;
1361
1362 hdr.network += hlen;
1363 }
1364
1365 /*
1366 * If everything has gone correctly hdr.network should be the
1367 * data section of the packet and will be the end of the header.
1368 * If not then it probably represents the end of the last recognized
1369 * header.
1370 */
1371 if ((hdr.network - data) < max_len)
1372 return hdr.network - data;
1373 else
1374 return max_len;
1375}
1376
4c1975d7
AD		 1377static void ixgbe_get_rsc_cnt(struct ixgbe_ring *rx_ring,
1378 union ixgbe_adv_rx_desc *rx_desc,
1379 struct sk_buff *skb)
aa80175a 1380{
4c1975d7
AD		 1381 __le32 rsc_enabled;
1382 u32 rsc_cnt;
1383
1384 if (!ring_is_rsc_enabled(rx_ring))
1385 return;
1386
1387 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1388 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
1389
1390 /* If this is an RSC frame rsc_cnt should be non-zero */
1391 if (!rsc_enabled)
1392 return;
1393
1394 rsc_cnt = le32_to_cpu(rsc_enabled);
1395 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1396
1397 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
aa80175a 1398}
43634e82 1399
1d2024f6
AD		 1400static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1401 struct sk_buff *skb)
1402{
1403 u16 hdr_len = ixgbe_get_headlen(skb->data, skb_headlen(skb));
1404
1405 /* set gso_size to avoid messing up TCP MSS */
1406 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1407 IXGBE_CB(skb)->append_cnt);
1408}
1409
1410static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1411 struct sk_buff *skb)
1412{
1413 /* if append_cnt is 0 then frame is not RSC */
1414 if (!IXGBE_CB(skb)->append_cnt)
1415 return;
1416
1417 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1418 rx_ring->rx_stats.rsc_flush++;
1419
1420 ixgbe_set_rsc_gso_size(rx_ring, skb);
1421
1422 /* gso_size is computed using append_cnt so always clear it last */
1423 IXGBE_CB(skb)->append_cnt = 0;
1424}
1425
4ff7fb12 1426static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
e8e9f696 1427 struct ixgbe_ring *rx_ring,
4ff7fb12 1428 int budget)
9a799d71 1429{
78b6f4ce 1430 struct ixgbe_adapter *adapter = q_vector->adapter;
9a799d71 1431 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
4c1975d7 1432 struct ixgbe_rx_buffer *rx_buffer_info;
9a799d71 1433 struct sk_buff *skb;
d2f4fbe2 1434 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
c267fc16 1435 const int current_node = numa_node_id();
3d8fd385
YZ		 1436#ifdef IXGBE_FCOE
1437 int ddp_bytes = 0;
1438#endif /* IXGBE_FCOE */
c267fc16
AD		 1439 u32 staterr;
1440 u16 i;
1441 u16 cleaned_count = 0;
9a799d71
AK		 1442
1443 i = rx_ring->next_to_clean;
31f05a2d 1444 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
9a799d71 1445 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
9a799d71
AK		 1446
1447 while (staterr & IXGBE_RXD_STAT_DD) {
7c6e0a43 1448 u32 upper_len = 0;
9a799d71 1449
3c945e5b 1450 rmb(); /* read descriptor and rx_buffer_info after status DD */
9a799d71 1451
c267fc16
AD		 1452 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1453
9a799d71 1454 skb = rx_buffer_info->skb;
9a799d71 1455 rx_buffer_info->skb = NULL;
c267fc16 1456 prefetch(skb->data);
9a799d71 1457
b811ce91
JB		 1458 /* linear means we are building an skb from multiple pages */
1459 if (!skb_is_nonlinear(skb)) {
c267fc16 1460 u16 hlen;
c267fc16
AD		 1461 if (ring_is_ps_enabled(rx_ring)) {
1462 hlen = ixgbe_get_hlen(rx_desc);
1463 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1464 } else {
1465 hlen = le16_to_cpu(rx_desc->wb.upper.length);
1466 }
1467
1468 skb_put(skb, hlen);
4c1975d7
AD		 1469
1470 /*
1471 * Delay unmapping of the first packet. It carries the
1472 * header information, HW may still access the header
1473 * after writeback. Only unmap it when EOP is reached
1474 */
1475 if (!IXGBE_CB(skb)->head) {
1476 IXGBE_CB(skb)->delay_unmap = true;
1477 IXGBE_CB(skb)->dma = rx_buffer_info->dma;
1478 } else {
1479 skb = ixgbe_merge_active_tail(skb);
1480 dma_unmap_single(rx_ring->dev,
1481 rx_buffer_info->dma,
1482 rx_ring->rx_buf_len,
1483 DMA_FROM_DEVICE);
1484 }
1485 rx_buffer_info->dma = 0;
c267fc16
AD		 1486 } else {
1487 /* assume packet split since header is unmapped */
1488 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
9a799d71
AK		 1489 }
1490
1491 if (upper_len) {
b6ec895e
AD		 1492 dma_unmap_page(rx_ring->dev,
1493 rx_buffer_info->page_dma,
1494 PAGE_SIZE / 2,
1495 DMA_FROM_DEVICE);
9a799d71
AK		 1496 rx_buffer_info->page_dma = 0;
1497 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
e8e9f696
JP		 1498 rx_buffer_info->page,
1499 rx_buffer_info->page_offset,
1500 upper_len);
762f4c57 1501
c267fc16
AD		 1502 if ((page_count(rx_buffer_info->page) == 1) &&
1503 (page_to_nid(rx_buffer_info->page) == current_node))
762f4c57 1504 get_page(rx_buffer_info->page);
c267fc16
AD		 1505 else
1506 rx_buffer_info->page = NULL;
9a799d71
AK		 1507
1508 skb->len += upper_len;
1509 skb->data_len += upper_len;
98130646 1510 skb->truesize += PAGE_SIZE / 2;
9a799d71
AK		 1511 }
1512
4c1975d7
AD		 1513 ixgbe_get_rsc_cnt(rx_ring, rx_desc, skb);
1514
9a799d71
AK		 1515 i++;
1516 if (i == rx_ring->count)
1517 i = 0;
9a799d71 1518
31f05a2d 1519 next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
9a799d71 1520 prefetch(next_rxd);
9a799d71 1521 cleaned_count++;
f8212f97 1522
4c1975d7
AD		 1523 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1524 struct ixgbe_rx_buffer *next_buffer;
1525 u32 nextp;
1526
1527 if (IXGBE_CB(skb)->append_cnt) {
1528 nextp = staterr & IXGBE_RXDADV_NEXTP_MASK;
1529 nextp >>= IXGBE_RXDADV_NEXTP_SHIFT;
1530 } else {
1531 nextp = i;
1532 }
1533
f8212f97 1534 next_buffer = &rx_ring->rx_buffer_info[nextp];
f8212f97 1535
7d637bcc 1536 if (ring_is_ps_enabled(rx_ring)) {
f8212f97
AD		 1537 rx_buffer_info->skb = next_buffer->skb;
1538 rx_buffer_info->dma = next_buffer->dma;
1539 next_buffer->skb = skb;
1540 next_buffer->dma = 0;
1541 } else {
4c1975d7
AD		 1542 struct sk_buff *next_skb = next_buffer->skb;
1543 ixgbe_add_active_tail(skb, next_skb);
1544 IXGBE_CB(next_skb)->head = skb;
f8212f97 1545 }
5b7da515 1546 rx_ring->rx_stats.non_eop_descs++;
9a799d71
AK		 1547 goto next_desc;
1548 }
1549
4c1975d7
AD		 1550 dma_unmap_single(rx_ring->dev,
1551 IXGBE_CB(skb)->dma,
1552 rx_ring->rx_buf_len,
1553 DMA_FROM_DEVICE);
1554 IXGBE_CB(skb)->dma = 0;
1555 IXGBE_CB(skb)->delay_unmap = false;
1556
1557 if (ixgbe_close_active_frag_list(skb) &&
1558 !IXGBE_CB(skb)->append_cnt) {
aa80175a 1559 /* if we got here without RSC the packet is invalid */
4c1975d7
AD		 1560 dev_kfree_skb_any(skb);
1561 goto next_desc;
aa80175a 1562 }
c267fc16 1563
1d2024f6 1564 ixgbe_update_rsc_stats(rx_ring, skb);
c267fc16
AD		 1565
1566 /* ERR_MASK will only have valid bits if EOP set */
ff886dfc
AD		 1567 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
1568 dev_kfree_skb_any(skb);
9a799d71
AK		 1569 goto next_desc;
1570 }
1571
ff886dfc 1572 ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
67a74ee2
ET		 1573 if (adapter->netdev->features & NETIF_F_RXHASH)
1574 ixgbe_rx_hash(rx_desc, skb);
d2f4fbe2
AV		 1575
1576 /* probably a little skewed due to removing CRC */
1577 total_rx_bytes += skb->len;
1578 total_rx_packets++;
1579
fc77dc3c 1580 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
332d4a7d
YZ		 1581#ifdef IXGBE_FCOE
1582 /* if ddp, not passing to ULD unless for FCP_RSP or error */
ff886dfc
AD		 1583 if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
1584 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
1585 staterr);
63d635b2
AD		 1586 if (!ddp_bytes) {
1587 dev_kfree_skb_any(skb);
332d4a7d 1588 goto next_desc;
63d635b2 1589 }
3d8fd385 1590 }
332d4a7d 1591#endif /* IXGBE_FCOE */
fdaff1ce 1592 ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
9a799d71 1593
4ff7fb12 1594 budget--;
9a799d71
AK		 1595next_desc:
1596 rx_desc->wb.upper.status_error = 0;
1597
4ff7fb12 1598 if (!budget)
c267fc16
AD		 1599 break;
1600
9a799d71
AK		 1601 /* return some buffers to hardware, one at a time is too slow */
1602 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
fc77dc3c 1603 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
9a799d71
AK		 1604 cleaned_count = 0;
1605 }
1606
1607 /* use prefetched values */
1608 rx_desc = next_rxd;
9a799d71 1609 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
177db6ff
MC		 1610 }
1611
9a799d71 1612 rx_ring->next_to_clean = i;
7d4987de 1613 cleaned_count = ixgbe_desc_unused(rx_ring);
9a799d71
AK		 1614
1615 if (cleaned_count)
fc77dc3c 1616 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
9a799d71 1617
3d8fd385
YZ		 1618#ifdef IXGBE_FCOE
1619 /* include DDPed FCoE data */
1620 if (ddp_bytes > 0) {
1621 unsigned int mss;
1622
fc77dc3c 1623 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
3d8fd385
YZ		 1624 sizeof(struct fc_frame_header) -
1625 sizeof(struct fcoe_crc_eof);
1626 if (mss > 512)
1627 mss &= ~511;
1628 total_rx_bytes += ddp_bytes;
1629 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
1630 }
1631#endif /* IXGBE_FCOE */
1632
c267fc16
AD		 1633 u64_stats_update_begin(&rx_ring->syncp);
1634 rx_ring->stats.packets += total_rx_packets;
1635 rx_ring->stats.bytes += total_rx_bytes;
1636 u64_stats_update_end(&rx_ring->syncp);
bd198058
AD		 1637 q_vector->rx.total_packets += total_rx_packets;
1638 q_vector->rx.total_bytes += total_rx_bytes;
4ff7fb12
AD		 1639
1640 return !!budget;
9a799d71
AK		 1641}
1642
9a799d71
AK		 1643/**
1644 * ixgbe_configure_msix - Configure MSI-X hardware
1645 * @adapter: board private structure
1646 *
1647 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1648 * interrupts.
1649 **/
1650static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1651{
021230d4 1652 struct ixgbe_q_vector *q_vector;
efe3d3c8 1653 int q_vectors, v_idx;
021230d4 1654 u32 mask;
9a799d71 1655
021230d4 1656 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
9a799d71 1657
8e34d1aa
AD		 1658 /* Populate MSIX to EITR Select */
1659 if (adapter->num_vfs > 32) {
1660 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
1661 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
1662 }
1663
4df10466
JB		 1664 /*
1665 * Populate the IVAR table and set the ITR values to the
021230d4
AV		 1666 * corresponding register.
1667 */
1668 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
efe3d3c8 1669 struct ixgbe_ring *ring;
7a921c93 1670 q_vector = adapter->q_vector[v_idx];
021230d4 1671
efe3d3c8
AD		 1672 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
1673 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1674
1675 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
1676 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1677
d5bf4f67
ET		 1678 if (q_vector->tx.ring && !q_vector->rx.ring) {
1679 /* tx only vector */
1680 if (adapter->tx_itr_setting == 1)
1681 q_vector->itr = IXGBE_10K_ITR;
1682 else
1683 q_vector->itr = adapter->tx_itr_setting;
1684 } else {
1685 /* rx or rx/tx vector */
1686 if (adapter->rx_itr_setting == 1)
1687 q_vector->itr = IXGBE_20K_ITR;
1688 else
1689 q_vector->itr = adapter->rx_itr_setting;
1690 }
021230d4 1691
fe49f04a 1692 ixgbe_write_eitr(q_vector);
9a799d71
AK		 1693 }
1694
bd508178
AD		 1695 switch (adapter->hw.mac.type) {
1696 case ixgbe_mac_82598EB:
e8e26350 1697 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
e8e9f696 1698 v_idx);
bd508178
AD		 1699 break;
1700 case ixgbe_mac_82599EB:
b93a2226 1701 case ixgbe_mac_X540:
e8e26350 1702 ixgbe_set_ivar(adapter, -1, 1, v_idx);
bd508178 1703 break;
bd508178
AD		 1704 default:
1705 break;
1706 }
021230d4
AV		 1707 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1708
41fb9248 1709 /* set up to autoclear timer, and the vectors */
021230d4 1710 mask = IXGBE_EIMS_ENABLE_MASK;
d5bf4f67
ET		 1711 mask &= ~(IXGBE_EIMS_OTHER |
1712 IXGBE_EIMS_MAILBOX |
1713 IXGBE_EIMS_LSC);
1714
021230d4 1715 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
9a799d71
AK		 1716}
1717
f494e8fa
AV		 1718enum latency_range {
1719 lowest_latency = 0,
1720 low_latency = 1,
1721 bulk_latency = 2,
1722 latency_invalid = 255
1723};
1724
1725/**
1726 * ixgbe_update_itr - update the dynamic ITR value based on statistics
bd198058
AD		 1727 * @q_vector: structure containing interrupt and ring information
1728 * @ring_container: structure containing ring performance data
f494e8fa
AV		 1729 *
1730 * Stores a new ITR value based on packets and byte
1731 * counts during the last interrupt. The advantage of per interrupt
1732 * computation is faster updates and more accurate ITR for the current
1733 * traffic pattern. Constants in this function were computed
1734 * based on theoretical maximum wire speed and thresholds were set based
1735 * on testing data as well as attempting to minimize response time
1736 * while increasing bulk throughput.
1737 * this functionality is controlled by the InterruptThrottleRate module
1738 * parameter (see ixgbe_param.c)
1739 **/
bd198058
AD		 1740static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
1741 struct ixgbe_ring_container *ring_container)
f494e8fa 1742{
f494e8fa 1743 u64 bytes_perint;
bd198058
AD		 1744 struct ixgbe_adapter *adapter = q_vector->adapter;
1745 int bytes = ring_container->total_bytes;
1746 int packets = ring_container->total_packets;
1747 u32 timepassed_us;
1748 u8 itr_setting = ring_container->itr;
f494e8fa
AV		 1749
1750 if (packets == 0)
bd198058 1751 return;
f494e8fa
AV		 1752
1753 /* simple throttlerate management
1754 * 0-20MB/s lowest (100000 ints/s)
1755 * 20-100MB/s low (20000 ints/s)
1756 * 100-1249MB/s bulk (8000 ints/s)
1757 */
1758 /* what was last interrupt timeslice? */
d5bf4f67 1759 timepassed_us = q_vector->itr >> 2;
f494e8fa
AV		 1760 bytes_perint = bytes / timepassed_us; /* bytes/usec */
1761
1762 switch (itr_setting) {
1763 case lowest_latency:
1764 if (bytes_perint > adapter->eitr_low)
bd198058 1765 itr_setting = low_latency;
f494e8fa
AV		 1766 break;
1767 case low_latency:
1768 if (bytes_perint > adapter->eitr_high)
bd198058 1769 itr_setting = bulk_latency;
f494e8fa 1770 else if (bytes_perint <= adapter->eitr_low)
bd198058 1771 itr_setting = lowest_latency;
f494e8fa
AV		 1772 break;
1773 case bulk_latency:
1774 if (bytes_perint <= adapter->eitr_high)
bd198058 1775 itr_setting = low_latency;
f494e8fa
AV		 1776 break;
1777 }
1778
bd198058
AD		 1779 /* clear work counters since we have the values we need */
1780 ring_container->total_bytes = 0;
1781 ring_container->total_packets = 0;
1782
1783 /* write updated itr to ring container */
1784 ring_container->itr = itr_setting;
f494e8fa
AV		 1785}
1786
509ee935
JB		 1787/**
1788 * ixgbe_write_eitr - write EITR register in hardware specific way
fe49f04a 1789 * @q_vector: structure containing interrupt and ring information
509ee935
JB		 1790 *
1791 * This function is made to be called by ethtool and by the driver
1792 * when it needs to update EITR registers at runtime. Hardware
1793 * specific quirks/differences are taken care of here.
1794 */
fe49f04a 1795void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
509ee935 1796{
fe49f04a 1797 struct ixgbe_adapter *adapter = q_vector->adapter;
509ee935 1798 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 1799 int v_idx = q_vector->v_idx;
d5bf4f67 1800 u32 itr_reg = q_vector->itr;
fe49f04a 1801
bd508178
AD		 1802 switch (adapter->hw.mac.type) {
1803 case ixgbe_mac_82598EB:
509ee935
JB		 1804 /* must write high and low 16 bits to reset counter */
1805 itr_reg |= (itr_reg << 16);
bd508178
AD		 1806 break;
1807 case ixgbe_mac_82599EB:
b93a2226 1808 case ixgbe_mac_X540:
509ee935
JB		 1809 /*
1810 * set the WDIS bit to not clear the timer bits and cause an
1811 * immediate assertion of the interrupt
1812 */
1813 itr_reg |= IXGBE_EITR_CNT_WDIS;
bd508178
AD		 1814 break;
1815 default:
1816 break;
509ee935
JB		 1817 }
1818 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
1819}
1820
bd198058 1821static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
f494e8fa 1822{
d5bf4f67 1823 u32 new_itr = q_vector->itr;
bd198058 1824 u8 current_itr;
f494e8fa 1825
bd198058
AD		 1826 ixgbe_update_itr(q_vector, &q_vector->tx);
1827 ixgbe_update_itr(q_vector, &q_vector->rx);
f494e8fa 1828
08c8833b 1829 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
f494e8fa
AV		 1830
1831 switch (current_itr) {
1832 /* counts and packets in update_itr are dependent on these numbers */
1833 case lowest_latency:
d5bf4f67 1834 new_itr = IXGBE_100K_ITR;
f494e8fa
AV		 1835 break;
1836 case low_latency:
d5bf4f67 1837 new_itr = IXGBE_20K_ITR;
f494e8fa
AV		 1838 break;
1839 case bulk_latency:
d5bf4f67 1840 new_itr = IXGBE_8K_ITR;
f494e8fa 1841 break;
bd198058
AD		 1842 default:
1843 break;
f494e8fa
AV		 1844 }
1845
d5bf4f67 1846 if (new_itr != q_vector->itr) {
fe49f04a 1847 /* do an exponential smoothing */
d5bf4f67
ET		 1848 new_itr = (10 * new_itr * q_vector->itr) /
1849 ((9 * new_itr) + q_vector->itr);
509ee935 1850
bd198058 1851 /* save the algorithm value here */
d5bf4f67 1852 q_vector->itr = new_itr & IXGBE_MAX_EITR;
fe49f04a
AD		 1853
1854 ixgbe_write_eitr(q_vector);
f494e8fa 1855 }
f494e8fa
AV		 1856}
1857
119fc60a 1858/**
f0f9778d
AD		 1859 * ixgbe_check_overtemp_subtask - check for over tempurature
1860 * @adapter: pointer to adapter
119fc60a 1861 **/
f0f9778d 1862static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
119fc60a 1863{
119fc60a
MC		 1864 struct ixgbe_hw *hw = &adapter->hw;
1865 u32 eicr = adapter->interrupt_event;
1866
f0f9778d 1867 if (test_bit(__IXGBE_DOWN, &adapter->state))
7ca647bd
JP		 1868 return;
1869
f0f9778d
AD		 1870 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1871 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
1872 return;
1873
1874 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1875
7ca647bd 1876 switch (hw->device_id) {
f0f9778d
AD		 1877 case IXGBE_DEV_ID_82599_T3_LOM:
1878 /*
1879 * Since the warning interrupt is for both ports
1880 * we don't have to check if:
1881 * - This interrupt wasn't for our port.
1882 * - We may have missed the interrupt so always have to
1883 * check if we got a LSC
1884 */
1885 if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
1886 !(eicr & IXGBE_EICR_LSC))
1887 return;
1888
1889 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
1890 u32 autoneg;
1891 bool link_up = false;
7ca647bd 1892
7ca647bd
JP		 1893 hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
1894
f0f9778d
AD		 1895 if (link_up)
1896 return;
1897 }
1898
1899 /* Check if this is not due to overtemp */
1900 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
1901 return;
1902
1903 break;
7ca647bd
JP		 1904 default:
1905 if (!(eicr & IXGBE_EICR_GPI_SDP0))
119fc60a 1906 return;
7ca647bd 1907 break;
119fc60a 1908 }
7ca647bd
JP		 1909 e_crit(drv,
1910 "Network adapter has been stopped because it has over heated. "
1911 "Restart the computer. If the problem persists, "
1912 "power off the system and replace the adapter\n");
f0f9778d
AD		 1913
1914 adapter->interrupt_event = 0;
119fc60a
MC		 1915}
1916
0befdb3e
JB		 1917static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
1918{
1919 struct ixgbe_hw *hw = &adapter->hw;
1920
1921 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
1922 (eicr & IXGBE_EICR_GPI_SDP1)) {
396e799c 1923 e_crit(probe, "Fan has stopped, replace the adapter\n");
0befdb3e
JB		 1924 /* write to clear the interrupt */
1925 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1926 }
1927}
cf8280ee 1928
4f51bf70
JK		 1929static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
1930{
1931 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
1932 return;
1933
1934 switch (adapter->hw.mac.type) {
1935 case ixgbe_mac_82599EB:
1936 /*
1937 * Need to check link state so complete overtemp check
1938 * on service task
1939 */
1940 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
1941 (!test_bit(__IXGBE_DOWN, &adapter->state))) {
1942 adapter->interrupt_event = eicr;
1943 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1944 ixgbe_service_event_schedule(adapter);
1945 return;
1946 }
1947 return;
1948 case ixgbe_mac_X540:
1949 if (!(eicr & IXGBE_EICR_TS))
1950 return;
1951 break;
1952 default:
1953 return;
1954 }
1955
1956 e_crit(drv,
1957 "Network adapter has been stopped because it has over heated. "
1958 "Restart the computer. If the problem persists, "
1959 "power off the system and replace the adapter\n");
1960}
1961
e8e26350
PW		 1962static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
1963{
1964 struct ixgbe_hw *hw = &adapter->hw;
1965
73c4b7cd
AD		 1966 if (eicr & IXGBE_EICR_GPI_SDP2) {
1967 /* Clear the interrupt */
1968 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
7086400d
AD		 1969 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1970 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
1971 ixgbe_service_event_schedule(adapter);
1972 }
73c4b7cd
AD		 1973 }
1974
e8e26350
PW		 1975 if (eicr & IXGBE_EICR_GPI_SDP1) {
1976 /* Clear the interrupt */
1977 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
7086400d
AD		 1978 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1979 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
1980 ixgbe_service_event_schedule(adapter);
1981 }
e8e26350
PW		 1982 }
1983}
1984
cf8280ee
JB		 1985static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
1986{
1987 struct ixgbe_hw *hw = &adapter->hw;
1988
1989 adapter->lsc_int++;
1990 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1991 adapter->link_check_timeout = jiffies;
1992 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1993 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
8a0717f3 1994 IXGBE_WRITE_FLUSH(hw);
93c52dd0 1995 ixgbe_service_event_schedule(adapter);
cf8280ee
JB		 1996 }
1997}
1998
fe49f04a
AD		 1999static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
2000 u64 qmask)
2001{
2002 u32 mask;
bd508178 2003 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 2004
bd508178
AD		 2005 switch (hw->mac.type) {
2006 case ixgbe_mac_82598EB:
fe49f04a 2007 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
bd508178
AD		 2008 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
2009 break;
2010 case ixgbe_mac_82599EB:
b93a2226 2011 case ixgbe_mac_X540:
fe49f04a 2012 mask = (qmask & 0xFFFFFFFF);
bd508178
AD		 2013 if (mask)
2014 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
fe49f04a 2015 mask = (qmask >> 32);
bd508178
AD		 2016 if (mask)
2017 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
2018 break;
2019 default:
2020 break;
fe49f04a
AD		 2021 }
2022 /* skip the flush */
2023}
2024
2025static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
e8e9f696 2026 u64 qmask)
fe49f04a
AD		 2027{
2028 u32 mask;
bd508178 2029 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 2030
bd508178
AD		 2031 switch (hw->mac.type) {
2032 case ixgbe_mac_82598EB:
fe49f04a 2033 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
bd508178
AD		 2034 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
2035 break;
2036 case ixgbe_mac_82599EB:
b93a2226 2037 case ixgbe_mac_X540:
fe49f04a 2038 mask = (qmask & 0xFFFFFFFF);
bd508178
AD		 2039 if (mask)
2040 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
fe49f04a 2041 mask = (qmask >> 32);
bd508178
AD		 2042 if (mask)
2043 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
2044 break;
2045 default:
2046 break;
fe49f04a
AD		 2047 }
2048 /* skip the flush */
2049}
2050
021230d4 2051/**
2c4af694
AD		 2052 * ixgbe_irq_enable - Enable default interrupt generation settings
2053 * @adapter: board private structure
021230d4 2054 **/
2c4af694
AD		 2055static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2056 bool flush)
9a799d71 2057{
2c4af694 2058 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
9a799d71 2059
2c4af694
AD		 2060 /* don't reenable LSC while waiting for link */
2061 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
2062 mask &= ~IXGBE_EIMS_LSC;
9a799d71 2063
2c4af694 2064 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
4f51bf70
JK		 2065 switch (adapter->hw.mac.type) {
2066 case ixgbe_mac_82599EB:
2067 mask |= IXGBE_EIMS_GPI_SDP0;
2068 break;
2069 case ixgbe_mac_X540:
2070 mask |= IXGBE_EIMS_TS;
2071 break;
2072 default:
2073 break;
2074 }
2c4af694
AD		 2075 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
2076 mask |= IXGBE_EIMS_GPI_SDP1;
2077 switch (adapter->hw.mac.type) {
2078 case ixgbe_mac_82599EB:
2c4af694
AD		 2079 mask |= IXGBE_EIMS_GPI_SDP1;
2080 mask |= IXGBE_EIMS_GPI_SDP2;
858bc081
DS		 2081 case ixgbe_mac_X540:
2082 mask |= IXGBE_EIMS_ECC;
2c4af694
AD		 2083 mask |= IXGBE_EIMS_MAILBOX;
2084 break;
2085 default:
2086 break;
9a799d71 2087 }
2c4af694
AD		 2088 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2089 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
2090 mask |= IXGBE_EIMS_FLOW_DIR;
9a799d71 2091
2c4af694
AD		 2092 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2093 if (queues)
2094 ixgbe_irq_enable_queues(adapter, ~0);
2095 if (flush)
2096 IXGBE_WRITE_FLUSH(&adapter->hw);
9a799d71
AK		 2097}
2098
2c4af694 2099static irqreturn_t ixgbe_msix_other(int irq, void *data)
f0848276 2100{
a65151ba 2101 struct ixgbe_adapter *adapter = data;
9a799d71 2102 struct ixgbe_hw *hw = &adapter->hw;
54037505 2103 u32 eicr;
91281fd3 2104
54037505
DS		 2105 /*
2106 * Workaround for Silicon errata. Use clear-by-write instead
2107 * of clear-by-read. Reading with EICS will return the
2108 * interrupt causes without clearing, which later be done
2109 * with the write to EICR.
2110 */
2111 eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
2112 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
33cf09c9 2113
cf8280ee
JB		 2114 if (eicr & IXGBE_EICR_LSC)
2115 ixgbe_check_lsc(adapter);
f0848276 2116
1cdd1ec8
GR		 2117 if (eicr & IXGBE_EICR_MAILBOX)
2118 ixgbe_msg_task(adapter);
efe3d3c8 2119
bd508178
AD		 2120 switch (hw->mac.type) {
2121 case ixgbe_mac_82599EB:
b93a2226 2122 case ixgbe_mac_X540:
2c4af694
AD		 2123 if (eicr & IXGBE_EICR_ECC)
2124 e_info(link, "Received unrecoverable ECC Err, please "
2125 "reboot\n");
c4cf55e5
PWJ		 2126 /* Handle Flow Director Full threshold interrupt */
2127 if (eicr & IXGBE_EICR_FLOW_DIR) {
d034acf1 2128 int reinit_count = 0;
c4cf55e5 2129 int i;
c4cf55e5 2130 for (i = 0; i < adapter->num_tx_queues; i++) {
d034acf1 2131 struct ixgbe_ring *ring = adapter->tx_ring[i];
7d637bcc 2132 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
d034acf1
AD		 2133 &ring->state))
2134 reinit_count++;
2135 }
2136 if (reinit_count) {
2137 /* no more flow director interrupts until after init */
2138 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
d034acf1
AD		 2139 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
2140 ixgbe_service_event_schedule(adapter);
c4cf55e5
PWJ		 2141 }
2142 }
f0f9778d 2143 ixgbe_check_sfp_event(adapter, eicr);
4f51bf70 2144 ixgbe_check_overtemp_event(adapter, eicr);
bd508178
AD		 2145 break;
2146 default:
2147 break;
c4cf55e5 2148 }
f0848276 2149
bd508178 2150 ixgbe_check_fan_failure(adapter, eicr);
efe3d3c8 2151
7086400d 2152 /* re-enable the original interrupt state, no lsc, no queues */
d4f80882 2153 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2c4af694 2154 ixgbe_irq_enable(adapter, false, false);
f0848276 2155
9a799d71 2156 return IRQ_HANDLED;
f0848276 2157}
91281fd3 2158
4ff7fb12 2159static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
91281fd3 2160{
021230d4 2161 struct ixgbe_q_vector *q_vector = data;
91281fd3 2162
9b471446 2163 /* EIAM disabled interrupts (on this vector) for us */
91281fd3 2164
4ff7fb12
AD		 2165 if (q_vector->rx.ring || q_vector->tx.ring)
2166 napi_schedule(&q_vector->napi);
91281fd3 2167
9a799d71 2168 return IRQ_HANDLED;
91281fd3
AD		 2169}
2170
021230d4 2171static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
e8e9f696 2172 int r_idx)
021230d4 2173{
7a921c93 2174 struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2274543f 2175 struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
7a921c93 2176
2274543f 2177 rx_ring->q_vector = q_vector;
efe3d3c8
AD		 2178 rx_ring->next = q_vector->rx.ring;
2179 q_vector->rx.ring = rx_ring;
2180 q_vector->rx.count++;
021230d4
AV		 2181}
2182
2183static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
e8e9f696 2184 int t_idx)
021230d4 2185{
7a921c93 2186 struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2274543f 2187 struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
7a921c93 2188
2274543f 2189 tx_ring->q_vector = q_vector;
efe3d3c8
AD		 2190 tx_ring->next = q_vector->tx.ring;
2191 q_vector->tx.ring = tx_ring;
2192 q_vector->tx.count++;
bd198058 2193 q_vector->tx.work_limit = a->tx_work_limit;
021230d4
AV		 2194}
2195
9a799d71 2196/**
021230d4
AV		 2197 * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
2198 * @adapter: board private structure to initialize
9a799d71 2199 *
021230d4
AV		 2200 * This function maps descriptor rings to the queue-specific vectors
2201 * we were allotted through the MSI-X enabling code. Ideally, we'd have
2202 * one vector per ring/queue, but on a constrained vector budget, we
2203 * group the rings as "efficiently" as possible. You would add new
2204 * mapping configurations in here.
9a799d71 2205 **/
4cc6df29 2206static void ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
021230d4 2207{
4cc6df29
AD		 2208 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2209 int rxr_remaining = adapter->num_rx_queues, rxr_idx = 0;
2210 int txr_remaining = adapter->num_tx_queues, txr_idx = 0;
021230d4 2211 int v_start = 0;
021230d4 2212
4cc6df29 2213 /* only one q_vector if MSI-X is disabled. */
021230d4 2214 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
4cc6df29 2215 q_vectors = 1;
d0759ebb 2216
021230d4 2217 /*
4cc6df29
AD		 2218 * If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2219 * group them so there are multiple queues per vector.
2220 *
2221 * Re-adjusting *qpv takes care of the remainder.
021230d4 2222 */
4cc6df29
AD		 2223 for (; v_start < q_vectors && rxr_remaining; v_start++) {
2224 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_start);
2225 for (; rqpv; rqpv--, rxr_idx++, rxr_remaining--)
021230d4 2226 map_vector_to_rxq(adapter, v_start, rxr_idx);
021230d4 2227 }
9a799d71 2228
021230d4 2229 /*
4cc6df29
AD		 2230 * If there are not enough q_vectors for each ring to have it's own
2231 * vector then we must pair up Rx/Tx on a each vector
021230d4 2232 */
4cc6df29
AD		 2233 if ((v_start + txr_remaining) > q_vectors)
2234 v_start = 0;
2235
2236 for (; v_start < q_vectors && txr_remaining; v_start++) {
2237 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_start);
2238 for (; tqpv; tqpv--, txr_idx++, txr_remaining--)
2239 map_vector_to_txq(adapter, v_start, txr_idx);
9a799d71 2240 }
021230d4
AV		 2241}
2242
2243/**
2244 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2245 * @adapter: board private structure
2246 *
2247 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2248 * interrupts from the kernel.
2249 **/
2250static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2251{
2252 struct net_device *netdev = adapter->netdev;
207867f5
AD		 2253 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2254 int vector, err;
e8e9f696 2255 int ri = 0, ti = 0;
021230d4 2256
021230d4 2257 for (vector = 0; vector < q_vectors; vector++) {
d0759ebb 2258 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
207867f5 2259 struct msix_entry *entry = &adapter->msix_entries[vector];
cb13fc20 2260
4ff7fb12 2261 if (q_vector->tx.ring && q_vector->rx.ring) {
9fe93afd 2262 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4ff7fb12
AD		 2263 "%s-%s-%d", netdev->name, "TxRx", ri++);
2264 ti++;
2265 } else if (q_vector->rx.ring) {
9fe93afd 2266 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4ff7fb12
AD		 2267 "%s-%s-%d", netdev->name, "rx", ri++);
2268 } else if (q_vector->tx.ring) {
9fe93afd 2269 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4ff7fb12 2270 "%s-%s-%d", netdev->name, "tx", ti++);
d0759ebb
AD		 2271 } else {
2272 /* skip this unused q_vector */
2273 continue;
32aa77a4 2274 }
207867f5
AD		 2275 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2276 q_vector->name, q_vector);
9a799d71 2277 if (err) {
396e799c 2278 e_err(probe, "request_irq failed for MSIX interrupt "
849c4542 2279 "Error: %d\n", err);
021230d4 2280 goto free_queue_irqs;
9a799d71 2281 }
207867f5
AD		 2282 /* If Flow Director is enabled, set interrupt affinity */
2283 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2284 /* assign the mask for this irq */
2285 irq_set_affinity_hint(entry->vector,
2286 q_vector->affinity_mask);
2287 }
9a799d71
AK		 2288 }
2289
021230d4 2290 err = request_irq(adapter->msix_entries[vector].vector,
2c4af694 2291 ixgbe_msix_other, 0, netdev->name, adapter);
9a799d71 2292 if (err) {
396e799c 2293 e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
021230d4 2294 goto free_queue_irqs;
9a799d71
AK		 2295 }
2296
9a799d71
AK		 2297 return 0;
2298
021230d4 2299free_queue_irqs:
207867f5
AD		 2300 while (vector) {
2301 vector--;
2302 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2303 NULL);
2304 free_irq(adapter->msix_entries[vector].vector,
2305 adapter->q_vector[vector]);
2306 }
021230d4
AV		 2307 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2308 pci_disable_msix(adapter->pdev);
9a799d71
AK		 2309 kfree(adapter->msix_entries);
2310 adapter->msix_entries = NULL;
9a799d71
AK		 2311 return err;
2312}
2313
2314/**
021230d4 2315 * ixgbe_intr - legacy mode Interrupt Handler
9a799d71
AK		 2316 * @irq: interrupt number
2317 * @data: pointer to a network interface device structure
9a799d71
AK		 2318 **/
2319static irqreturn_t ixgbe_intr(int irq, void *data)
2320{
a65151ba 2321 struct ixgbe_adapter *adapter = data;
9a799d71 2322 struct ixgbe_hw *hw = &adapter->hw;
7a921c93 2323 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
9a799d71
AK		 2324 u32 eicr;
2325
54037505 2326 /*
6af3b9eb 2327 * Workaround for silicon errata on 82598. Mask the interrupts
54037505
DS		 2328 * before the read of EICR.
2329 */
2330 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2331
021230d4 2332 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
52f33af8 2333 * therefore no explicit interrupt disable is necessary */
021230d4 2334 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
f47cf66e 2335 if (!eicr) {
6af3b9eb
ET		 2336 /*
2337 * shared interrupt alert!
f47cf66e 2338 * make sure interrupts are enabled because the read will
6af3b9eb
ET		 2339 * have disabled interrupts due to EIAM
2340 * finish the workaround of silicon errata on 82598. Unmask
2341 * the interrupt that we masked before the EICR read.
2342 */
2343 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2344 ixgbe_irq_enable(adapter, true, true);
9a799d71 2345 return IRQ_NONE; /* Not our interrupt */
f47cf66e 2346 }
9a799d71 2347
cf8280ee
JB		 2348 if (eicr & IXGBE_EICR_LSC)
2349 ixgbe_check_lsc(adapter);
021230d4 2350
bd508178
AD		 2351 switch (hw->mac.type) {
2352 case ixgbe_mac_82599EB:
e8e26350 2353 ixgbe_check_sfp_event(adapter, eicr);
0ccb974d
DS		 2354 /* Fall through */
2355 case ixgbe_mac_X540:
2356 if (eicr & IXGBE_EICR_ECC)
2357 e_info(link, "Received unrecoverable ECC err, please "
2358 "reboot\n");
4f51bf70 2359 ixgbe_check_overtemp_event(adapter, eicr);
bd508178
AD		 2360 break;
2361 default:
2362 break;
2363 }
e8e26350 2364
0befdb3e
JB		 2365 ixgbe_check_fan_failure(adapter, eicr);
2366
7a921c93 2367 if (napi_schedule_prep(&(q_vector->napi))) {
021230d4 2368 /* would disable interrupts here but EIAM disabled it */
7a921c93 2369 __napi_schedule(&(q_vector->napi));
9a799d71
AK		 2370 }
2371
6af3b9eb
ET		 2372 /*
2373 * re-enable link(maybe) and non-queue interrupts, no flush.
2374 * ixgbe_poll will re-enable the queue interrupts
2375 */
2376
2377 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2378 ixgbe_irq_enable(adapter, false, false);
2379
9a799d71
AK		 2380 return IRQ_HANDLED;
2381}
2382
021230d4
AV		 2383static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
2384{
efe3d3c8
AD		 2385 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2386 int i;
2387
2388 /* legacy and MSI only use one vector */
2389 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2390 q_vectors = 1;
2391
2392 for (i = 0; i < adapter->num_rx_queues; i++) {
2393 adapter->rx_ring[i]->q_vector = NULL;
2394 adapter->rx_ring[i]->next = NULL;
2395 }
2396 for (i = 0; i < adapter->num_tx_queues; i++) {
2397 adapter->tx_ring[i]->q_vector = NULL;
2398 adapter->tx_ring[i]->next = NULL;
2399 }
021230d4
AV		 2400
2401 for (i = 0; i < q_vectors; i++) {
7a921c93 2402 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
efe3d3c8
AD		 2403 memset(&q_vector->rx, 0, sizeof(struct ixgbe_ring_container));
2404 memset(&q_vector->tx, 0, sizeof(struct ixgbe_ring_container));
021230d4
AV		 2405 }
2406}
2407
9a799d71
AK		 2408/**
2409 * ixgbe_request_irq - initialize interrupts
2410 * @adapter: board private structure
2411 *
2412 * Attempts to configure interrupts using the best available
2413 * capabilities of the hardware and kernel.
2414 **/
021230d4 2415static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
9a799d71
AK		 2416{
2417 struct net_device *netdev = adapter->netdev;
021230d4 2418 int err;
9a799d71 2419
4cc6df29
AD		 2420 /* map all of the rings to the q_vectors */
2421 ixgbe_map_rings_to_vectors(adapter);
2422
2423 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
021230d4 2424 err = ixgbe_request_msix_irqs(adapter);
4cc6df29 2425 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
a0607fd3 2426 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
a65151ba 2427 netdev->name, adapter);
4cc6df29 2428 else
a0607fd3 2429 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
a65151ba 2430 netdev->name, adapter);
9a799d71 2431
4cc6df29 2432 if (err) {
396e799c 2433 e_err(probe, "request_irq failed, Error %d\n", err);
9a799d71 2434
4cc6df29
AD		 2435 /* place q_vectors and rings back into a known good state */
2436 ixgbe_reset_q_vectors(adapter);
2437 }
2438
9a799d71
AK		 2439 return err;
2440}
2441
2442static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2443{
9a799d71 2444 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
021230d4 2445 int i, q_vectors;
9a799d71 2446
021230d4 2447 q_vectors = adapter->num_msix_vectors;
021230d4 2448 i = q_vectors - 1;
a65151ba 2449 free_irq(adapter->msix_entries[i].vector, adapter);
021230d4 2450 i--;
4cc6df29 2451
021230d4 2452 for (; i >= 0; i--) {
894ff7cf 2453 /* free only the irqs that were actually requested */
4ff7fb12
AD		 2454 if (!adapter->q_vector[i]->rx.ring &&
2455 !adapter->q_vector[i]->tx.ring)
894ff7cf
AD		 2456 continue;
2457
207867f5
AD		 2458 /* clear the affinity_mask in the IRQ descriptor */
2459 irq_set_affinity_hint(adapter->msix_entries[i].vector,
2460 NULL);
2461
021230d4 2462 free_irq(adapter->msix_entries[i].vector,
e8e9f696 2463 adapter->q_vector[i]);
021230d4 2464 }
021230d4 2465 } else {
a65151ba 2466 free_irq(adapter->pdev->irq, adapter);
9a799d71 2467 }
207867f5
AD		 2468
2469 /* clear q_vector state information */
2470 ixgbe_reset_q_vectors(adapter);
9a799d71
AK		 2471}
2472
22d5a71b
JB		 2473/**
2474 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2475 * @adapter: board private structure
2476 **/
2477static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2478{
bd508178
AD		 2479 switch (adapter->hw.mac.type) {
2480 case ixgbe_mac_82598EB:
835462fc 2481 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
bd508178
AD		 2482 break;
2483 case ixgbe_mac_82599EB:
b93a2226 2484 case ixgbe_mac_X540:
835462fc
NS		 2485 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2486 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
22d5a71b 2487 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
bd508178
AD		 2488 break;
2489 default:
2490 break;
22d5a71b
JB		 2491 }
2492 IXGBE_WRITE_FLUSH(&adapter->hw);
2493 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2494 int i;
2495 for (i = 0; i < adapter->num_msix_vectors; i++)
2496 synchronize_irq(adapter->msix_entries[i].vector);
2497 } else {
2498 synchronize_irq(adapter->pdev->irq);
2499 }
2500}
2501
9a799d71
AK		 2502/**
2503 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2504 *
2505 **/
2506static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2507{
d5bf4f67 2508 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
9a799d71 2509
d5bf4f67
ET		 2510 /* rx/tx vector */
2511 if (adapter->rx_itr_setting == 1)
2512 q_vector->itr = IXGBE_20K_ITR;
2513 else
2514 q_vector->itr = adapter->rx_itr_setting;
2515
2516 ixgbe_write_eitr(q_vector);
9a799d71 2517
e8e26350
PW		 2518 ixgbe_set_ivar(adapter, 0, 0, 0);
2519 ixgbe_set_ivar(adapter, 1, 0, 0);
021230d4 2520
396e799c 2521 e_info(hw, "Legacy interrupt IVAR setup done\n");
9a799d71
AK		 2522}
2523
43e69bf0
AD		 2524/**
2525 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2526 * @adapter: board private structure
2527 * @ring: structure containing ring specific data
2528 *
2529 * Configure the Tx descriptor ring after a reset.
2530 **/
84418e3b
AD		 2531void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2532 struct ixgbe_ring *ring)
43e69bf0
AD		 2533{
2534 struct ixgbe_hw *hw = &adapter->hw;
2535 u64 tdba = ring->dma;
2f1860b8 2536 int wait_loop = 10;
b88c6de2 2537 u32 txdctl = IXGBE_TXDCTL_ENABLE;
bf29ee6c 2538 u8 reg_idx = ring->reg_idx;
43e69bf0 2539
2f1860b8 2540 /* disable queue to avoid issues while updating state */
b88c6de2 2541 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2f1860b8
AD		 2542 IXGBE_WRITE_FLUSH(hw);
2543
43e69bf0 2544 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
e8e9f696 2545 (tdba & DMA_BIT_MASK(32)));
43e69bf0
AD		 2546 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2547 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2548 ring->count * sizeof(union ixgbe_adv_tx_desc));
2549 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2550 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
84ea2591 2551 ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
43e69bf0 2552
b88c6de2
AD		 2553 /*
2554 * set WTHRESH to encourage burst writeback, it should not be set
2555 * higher than 1 when ITR is 0 as it could cause false TX hangs
2556 *
2557 * In order to avoid issues WTHRESH + PTHRESH should always be equal
2558 * to or less than the number of on chip descriptors, which is
2559 * currently 40.
2560 */
2561 if (!adapter->tx_itr_setting || !adapter->rx_itr_setting)
2562 txdctl |= (1 << 16); /* WTHRESH = 1 */
2563 else
2564 txdctl |= (8 << 16); /* WTHRESH = 8 */
2565
2566 /* PTHRESH=32 is needed to avoid a Tx hang with DFP enabled. */
2567 txdctl |= (1 << 8) | /* HTHRESH = 1 */
2568 32; /* PTHRESH = 32 */
2f1860b8
AD		 2569
2570 /* reinitialize flowdirector state */
ee9e0f0b
AD		 2571 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2572 adapter->atr_sample_rate) {
2573 ring->atr_sample_rate = adapter->atr_sample_rate;
2574 ring->atr_count = 0;
2575 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2576 } else {
2577 ring->atr_sample_rate = 0;
2578 }
2f1860b8 2579
c84d324c
JF		 2580 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2581
2f1860b8 2582 /* enable queue */
2f1860b8
AD		 2583 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2584
2585 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2586 if (hw->mac.type == ixgbe_mac_82598EB &&
2587 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2588 return;
2589
2590 /* poll to verify queue is enabled */
2591 do {
032b4325 2592 usleep_range(1000, 2000);
2f1860b8
AD		 2593 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2594 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2595 if (!wait_loop)
2596 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
43e69bf0
AD		 2597}
2598
120ff942
AD		 2599static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2600{
2601 struct ixgbe_hw *hw = &adapter->hw;
2602 u32 rttdcs;
72a32f1f 2603 u32 reg;
8b1c0b24 2604 u8 tcs = netdev_get_num_tc(adapter->netdev);
120ff942
AD		 2605
2606 if (hw->mac.type == ixgbe_mac_82598EB)
2607 return;
2608
2609 /* disable the arbiter while setting MTQC */
2610 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2611 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2612 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2613
2614 /* set transmit pool layout */
8b1c0b24 2615 switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
120ff942
AD		 2616 case (IXGBE_FLAG_SRIOV_ENABLED):
2617 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2618 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
2619 break;
8b1c0b24
JF		 2620 default:
2621 if (!tcs)
2622 reg = IXGBE_MTQC_64Q_1PB;
2623 else if (tcs <= 4)
2624 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2625 else
2626 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
120ff942 2627
8b1c0b24 2628 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
120ff942 2629
8b1c0b24
JF		 2630 /* Enable Security TX Buffer IFG for multiple pb */
2631 if (tcs) {
2632 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2633 reg |= IXGBE_SECTX_DCB;
2634 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
2635 }
120ff942
AD		 2636 break;
2637 }
2638
2639 /* re-enable the arbiter */
2640 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2641 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2642}
2643
9a799d71 2644/**
3a581073 2645 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
9a799d71
AK		 2646 * @adapter: board private structure
2647 *
2648 * Configure the Tx unit of the MAC after a reset.
2649 **/
2650static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2651{
2f1860b8
AD		 2652 struct ixgbe_hw *hw = &adapter->hw;
2653 u32 dmatxctl;
43e69bf0 2654 u32 i;
9a799d71 2655
2f1860b8
AD		 2656 ixgbe_setup_mtqc(adapter);
2657
2658 if (hw->mac.type != ixgbe_mac_82598EB) {
2659 /* DMATXCTL.EN must be before Tx queues are enabled */
2660 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2661 dmatxctl |= IXGBE_DMATXCTL_TE;
2662 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2663 }
2664
9a799d71 2665 /* Setup the HW Tx Head and Tail descriptor pointers */
43e69bf0
AD		 2666 for (i = 0; i < adapter->num_tx_queues; i++)
2667 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
9a799d71
AK		 2668}
2669
e8e26350 2670#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
cc41ac7c 2671
a6616b42 2672static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
e8e9f696 2673 struct ixgbe_ring *rx_ring)
cc41ac7c 2674{
cc41ac7c 2675 u32 srrctl;
bf29ee6c 2676 u8 reg_idx = rx_ring->reg_idx;
3be1adfb 2677
bd508178
AD		 2678 switch (adapter->hw.mac.type) {
2679 case ixgbe_mac_82598EB: {
2680 struct ixgbe_ring_feature *feature = adapter->ring_feature;
2681 const int mask = feature[RING_F_RSS].mask;
bf29ee6c 2682 reg_idx = reg_idx & mask;
cc41ac7c 2683 }
bd508178
AD		 2684 break;
2685 case ixgbe_mac_82599EB:
b93a2226 2686 case ixgbe_mac_X540:
bd508178
AD		 2687 default:
2688 break;
2689 }
2690
bf29ee6c 2691 srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
cc41ac7c
JB		 2692
2693 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2694 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
9e10e045
AD		 2695 if (adapter->num_vfs)
2696 srrctl |= IXGBE_SRRCTL_DROP_EN;
cc41ac7c 2697
afafd5b0
AD		 2698 srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
2699 IXGBE_SRRCTL_BSIZEHDR_MASK;
2700
7d637bcc 2701 if (ring_is_ps_enabled(rx_ring)) {
afafd5b0
AD		 2702#if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
2703 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2704#else
2705 srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2706#endif
cc41ac7c 2707 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
cc41ac7c 2708 } else {
afafd5b0
AD		 2709 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
2710 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
cc41ac7c 2711 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
cc41ac7c 2712 }
e8e26350 2713
bf29ee6c 2714 IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
cc41ac7c 2715}
9a799d71 2716
05abb126 2717static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
0cefafad 2718{
05abb126
AD		 2719 struct ixgbe_hw *hw = &adapter->hw;
2720 static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
e8e9f696
JP		 2721 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2722 0x6A3E67EA, 0x14364D17, 0x3BED200D};
05abb126
AD		 2723 u32 mrqc = 0, reta = 0;
2724 u32 rxcsum;
2725 int i, j;
8b1c0b24 2726 u8 tcs = netdev_get_num_tc(adapter->netdev);
86b4db3b
JF		 2727 int maxq = adapter->ring_feature[RING_F_RSS].indices;
2728
2729 if (tcs)
2730 maxq = min(maxq, adapter->num_tx_queues / tcs);
0cefafad 2731
05abb126
AD		 2732 /* Fill out hash function seeds */
2733 for (i = 0; i < 10; i++)
2734 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2735
2736 /* Fill out redirection table */
2737 for (i = 0, j = 0; i < 128; i++, j++) {
86b4db3b 2738 if (j == maxq)
05abb126
AD		 2739 j = 0;
2740 /* reta = 4-byte sliding window of
2741 * 0x00..(indices-1)(indices-1)00..etc. */
2742 reta = (reta << 8) | (j * 0x11);
2743 if ((i & 3) == 3)
2744 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2745 }
0cefafad 2746
05abb126
AD		 2747 /* Disable indicating checksum in descriptor, enables RSS hash */
2748 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2749 rxcsum |= IXGBE_RXCSUM_PCSD;
2750 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2751
8b1c0b24
JF		 2752 if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
2753 (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
0cefafad 2754 mrqc = IXGBE_MRQC_RSSEN;
8b1c0b24
JF		 2755 } else {
2756 int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
2757 | IXGBE_FLAG_SRIOV_ENABLED);
2758
2759 switch (mask) {
2760 case (IXGBE_FLAG_RSS_ENABLED):
2761 if (!tcs)
2762 mrqc = IXGBE_MRQC_RSSEN;
2763 else if (tcs <= 4)
2764 mrqc = IXGBE_MRQC_RTRSS4TCEN;
2765 else
2766 mrqc = IXGBE_MRQC_RTRSS8TCEN;
2767 break;
2768 case (IXGBE_FLAG_SRIOV_ENABLED):
2769 mrqc = IXGBE_MRQC_VMDQEN;
2770 break;
2771 default:
2772 break;
2773 }
0cefafad
JB		 2774 }
2775
05abb126
AD		 2776 /* Perform hash on these packet types */
2777 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
2778 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2779 | IXGBE_MRQC_RSS_FIELD_IPV6
2780 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2781
2782 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
0cefafad
JB		 2783}
2784
bb5a9ad2
NS		 2785/**
2786 * ixgbe_configure_rscctl - enable RSC for the indicated ring
2787 * @adapter: address of board private structure
2788 * @index: index of ring to set
bb5a9ad2 2789 **/
082757af 2790static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
7367096a 2791 struct ixgbe_ring *ring)
bb5a9ad2 2792{
bb5a9ad2 2793 struct ixgbe_hw *hw = &adapter->hw;
bb5a9ad2 2794 u32 rscctrl;
edd2ea55 2795 int rx_buf_len;
bf29ee6c 2796 u8 reg_idx = ring->reg_idx;
7367096a 2797
7d637bcc 2798 if (!ring_is_rsc_enabled(ring))
7367096a 2799 return;
bb5a9ad2 2800
7367096a
AD		 2801 rx_buf_len = ring->rx_buf_len;
2802 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
bb5a9ad2
NS		 2803 rscctrl |= IXGBE_RSCCTL_RSCEN;
2804 /*
2805 * we must limit the number of descriptors so that the
2806 * total size of max desc * buf_len is not greater
2807 * than 65535
2808 */
7d637bcc 2809 if (ring_is_ps_enabled(ring)) {
bb5a9ad2
NS		 2810#if (MAX_SKB_FRAGS > 16)
2811 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2812#elif (MAX_SKB_FRAGS > 8)
2813 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2814#elif (MAX_SKB_FRAGS > 4)
2815 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2816#else
2817 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2818#endif
2819 } else {
919e78a6 2820 if (rx_buf_len < IXGBE_RXBUFFER_4K)
bb5a9ad2 2821 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
919e78a6 2822 else if (rx_buf_len < IXGBE_RXBUFFER_8K)
bb5a9ad2
NS		 2823 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2824 else
2825 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2826 }
7367096a 2827 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
bb5a9ad2
NS		 2828}
2829
9e10e045
AD		 2830/**
2831 * ixgbe_set_uta - Set unicast filter table address
2832 * @adapter: board private structure
2833 *
2834 * The unicast table address is a register array of 32-bit registers.
2835 * The table is meant to be used in a way similar to how the MTA is used
2836 * however due to certain limitations in the hardware it is necessary to
2837 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
2838 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
2839 **/
2840static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
2841{
2842 struct ixgbe_hw *hw = &adapter->hw;
2843 int i;
2844
2845 /* The UTA table only exists on 82599 hardware and newer */
2846 if (hw->mac.type < ixgbe_mac_82599EB)
2847 return;
2848
2849 /* we only need to do this if VMDq is enabled */
2850 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2851 return;
2852
2853 for (i = 0; i < 128; i++)
2854 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2855}
2856
2857#define IXGBE_MAX_RX_DESC_POLL 10
2858static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
2859 struct ixgbe_ring *ring)
2860{
2861 struct ixgbe_hw *hw = &adapter->hw;
9e10e045
AD		 2862 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2863 u32 rxdctl;
bf29ee6c 2864 u8 reg_idx = ring->reg_idx;
9e10e045
AD		 2865
2866 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2867 if (hw->mac.type == ixgbe_mac_82598EB &&
2868 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2869 return;
2870
2871 do {
032b4325 2872 usleep_range(1000, 2000);
9e10e045
AD		 2873 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2874 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
2875
2876 if (!wait_loop) {
2877 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
2878 "the polling period\n", reg_idx);
2879 }
2880}
2881
2d39d576
YZ		 2882void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
2883 struct ixgbe_ring *ring)
2884{
2885 struct ixgbe_hw *hw = &adapter->hw;
2886 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2887 u32 rxdctl;
2888 u8 reg_idx = ring->reg_idx;
2889
2890 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2891 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
2892
2893 /* write value back with RXDCTL.ENABLE bit cleared */
2894 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2895
2896 if (hw->mac.type == ixgbe_mac_82598EB &&
2897 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2898 return;
2899
2900 /* the hardware may take up to 100us to really disable the rx queue */
2901 do {
2902 udelay(10);
2903 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2904 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
2905
2906 if (!wait_loop) {
2907 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
2908 "the polling period\n", reg_idx);
2909 }
2910}
2911
84418e3b
AD		 2912void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
2913 struct ixgbe_ring *ring)
acd37177
AD		 2914{
2915 struct ixgbe_hw *hw = &adapter->hw;
2916 u64 rdba = ring->dma;
9e10e045 2917 u32 rxdctl;
bf29ee6c 2918 u8 reg_idx = ring->reg_idx;
acd37177 2919
9e10e045
AD		 2920 /* disable queue to avoid issues while updating state */
2921 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2d39d576 2922 ixgbe_disable_rx_queue(adapter, ring);
9e10e045 2923
acd37177
AD		 2924 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
2925 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
2926 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
2927 ring->count * sizeof(union ixgbe_adv_rx_desc));
2928 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
2929 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
84ea2591 2930 ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
9e10e045
AD		 2931
2932 ixgbe_configure_srrctl(adapter, ring);
2933 ixgbe_configure_rscctl(adapter, ring);
2934
e9f98072
GR		 2935 /* If operating in IOV mode set RLPML for X540 */
2936 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
2937 hw->mac.type == ixgbe_mac_X540) {
2938 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
2939 rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
2940 ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
2941 }
2942
9e10e045
AD		 2943 if (hw->mac.type == ixgbe_mac_82598EB) {
2944 /*
2945 * enable cache line friendly hardware writes:
2946 * PTHRESH=32 descriptors (half the internal cache),
2947 * this also removes ugly rx_no_buffer_count increment
2948 * HTHRESH=4 descriptors (to minimize latency on fetch)
2949 * WTHRESH=8 burst writeback up to two cache lines
2950 */
2951 rxdctl &= ~0x3FFFFF;
2952 rxdctl |= 0x080420;
2953 }
2954
2955 /* enable receive descriptor ring */
2956 rxdctl |= IXGBE_RXDCTL_ENABLE;
2957 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2958
2959 ixgbe_rx_desc_queue_enable(adapter, ring);
7d4987de 2960 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
acd37177
AD		 2961}
2962
48654521
AD		 2963static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
2964{
2965 struct ixgbe_hw *hw = &adapter->hw;
2966 int p;
2967
2968 /* PSRTYPE must be initialized in non 82598 adapters */
2969 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
e8e9f696
JP		 2970 IXGBE_PSRTYPE_UDPHDR |
2971 IXGBE_PSRTYPE_IPV4HDR |
48654521 2972 IXGBE_PSRTYPE_L2HDR |
e8e9f696 2973 IXGBE_PSRTYPE_IPV6HDR;
48654521
AD		 2974
2975 if (hw->mac.type == ixgbe_mac_82598EB)
2976 return;
2977
2978 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
2979 psrtype |= (adapter->num_rx_queues_per_pool << 29);
2980
2981 for (p = 0; p < adapter->num_rx_pools; p++)
2982 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
2983 psrtype);
2984}
2985
f5b4a52e
AD		 2986static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
2987{
2988 struct ixgbe_hw *hw = &adapter->hw;
2989 u32 gcr_ext;
2990 u32 vt_reg_bits;
2991 u32 reg_offset, vf_shift;
2992 u32 vmdctl;
de4c7f65 2993 int i;
f5b4a52e
AD		 2994
2995 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2996 return;
2997
2998 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2999 vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
3000 vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
3001 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
3002
3003 vf_shift = adapter->num_vfs % 32;
3004 reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
3005
3006 /* Enable only the PF's pool for Tx/Rx */
3007 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
3008 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
3009 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
3010 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
3011 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3012
3013 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
3014 hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
3015
3016 /*
3017 * Set up VF register offsets for selected VT Mode,
3018 * i.e. 32 or 64 VFs for SR-IOV
3019 */
3020 gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
3021 gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
3022 gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
3023 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
3024
3025 /* enable Tx loopback for VF/PF communication */
3026 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
a985b6c3 3027 /* Enable MAC Anti-Spoofing */
a1cbb15c 3028 hw->mac.ops.set_mac_anti_spoofing(hw,
de4c7f65 3029 (adapter->num_vfs != 0),
a985b6c3 3030 adapter->num_vfs);
de4c7f65
GR		 3031 /* For VFs that have spoof checking turned off */
3032 for (i = 0; i < adapter->num_vfs; i++) {
3033 if (!adapter->vfinfo[i].spoofchk_enabled)
3034 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
3035 }
f5b4a52e
AD		 3036}
3037
477de6ed 3038static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
9a799d71 3039{
9a799d71
AK		 3040 struct ixgbe_hw *hw = &adapter->hw;
3041 struct net_device *netdev = adapter->netdev;
3042 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
7c6e0a43 3043 int rx_buf_len;
477de6ed
AD		 3044 struct ixgbe_ring *rx_ring;
3045 int i;
3046 u32 mhadd, hlreg0;
48654521 3047
9a799d71 3048 /* Decide whether to use packet split mode or not */
a124339a
DS		 3049 /* On by default */
3050 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
3051
1cdd1ec8 3052 /* Do not use packet split if we're in SR-IOV Mode */
a124339a
DS		 3053 if (adapter->num_vfs)
3054 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3055
3056 /* Disable packet split due to 82599 erratum #45 */
3057 if (hw->mac.type == ixgbe_mac_82599EB)
3058 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
9a799d71 3059
63f39bd1 3060#ifdef IXGBE_FCOE
477de6ed
AD		 3061 /* adjust max frame to be able to do baby jumbo for FCoE */
3062 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
3063 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
3064 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
9a799d71 3065
477de6ed
AD		 3066#endif /* IXGBE_FCOE */
3067 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3068 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
3069 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3070 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
3071
3072 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3073 }
3074
919e78a6
AD		 3075 /* MHADD will allow an extra 4 bytes past for vlan tagged frames */
3076 max_frame += VLAN_HLEN;
3077
3078 /* Set the RX buffer length according to the mode */
3079 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
3080 rx_buf_len = IXGBE_RX_HDR_SIZE;
3081 } else {
3082 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
3083 (netdev->mtu <= ETH_DATA_LEN))
3084 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3085 /*
3086 * Make best use of allocation by using all but 1K of a
3087 * power of 2 allocation that will be used for skb->head.
3088 */
3089 else if (max_frame <= IXGBE_RXBUFFER_3K)
3090 rx_buf_len = IXGBE_RXBUFFER_3K;
3091 else if (max_frame <= IXGBE_RXBUFFER_7K)
3092 rx_buf_len = IXGBE_RXBUFFER_7K;
3093 else if (max_frame <= IXGBE_RXBUFFER_15K)
3094 rx_buf_len = IXGBE_RXBUFFER_15K;
3095 else
3096 rx_buf_len = IXGBE_MAX_RXBUFFER;
3097 }
3098
477de6ed
AD		 3099 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3100 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
3101 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3102 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
9a799d71 3103
0cefafad
JB		 3104 /*
3105 * Setup the HW Rx Head and Tail Descriptor Pointers and
3106 * the Base and Length of the Rx Descriptor Ring
3107 */
9a799d71 3108 for (i = 0; i < adapter->num_rx_queues; i++) {
4a0b9ca0 3109 rx_ring = adapter->rx_ring[i];
a6616b42 3110 rx_ring->rx_buf_len = rx_buf_len;
cc41ac7c 3111
6e455b89 3112 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
7d637bcc
AD		 3113 set_ring_ps_enabled(rx_ring);
3114 else
3115 clear_ring_ps_enabled(rx_ring);
3116
3117 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
3118 set_ring_rsc_enabled(rx_ring);
1b3ff02e 3119 else
7d637bcc 3120 clear_ring_rsc_enabled(rx_ring);
cc41ac7c 3121
63f39bd1 3122#ifdef IXGBE_FCOE
e8e9f696 3123 if (netdev->features & NETIF_F_FCOE_MTU) {
63f39bd1
YZ		 3124 struct ixgbe_ring_feature *f;
3125 f = &adapter->ring_feature[RING_F_FCOE];
6e455b89 3126 if ((i >= f->mask) && (i < f->mask + f->indices)) {
7d637bcc 3127 clear_ring_ps_enabled(rx_ring);
6e455b89
YZ		 3128 if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE)
3129 rx_ring->rx_buf_len =
e8e9f696 3130 IXGBE_FCOE_JUMBO_FRAME_SIZE;
7d637bcc
AD		 3131 } else if (!ring_is_rsc_enabled(rx_ring) &&
3132 !ring_is_ps_enabled(rx_ring)) {
3133 rx_ring->rx_buf_len =
3134 IXGBE_FCOE_JUMBO_FRAME_SIZE;
6e455b89 3135 }
63f39bd1 3136 }
63f39bd1 3137#endif /* IXGBE_FCOE */
477de6ed 3138 }
477de6ed
AD		 3139}
3140
7367096a
AD		 3141static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
3142{
3143 struct ixgbe_hw *hw = &adapter->hw;
3144 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3145
3146 switch (hw->mac.type) {
3147 case ixgbe_mac_82598EB:
3148 /*
3149 * For VMDq support of different descriptor types or
3150 * buffer sizes through the use of multiple SRRCTL
3151 * registers, RDRXCTL.MVMEN must be set to 1
3152 *
3153 * also, the manual doesn't mention it clearly but DCA hints
3154 * will only use queue 0's tags unless this bit is set. Side
3155 * effects of setting this bit are only that SRRCTL must be
3156 * fully programmed [0..15]
3157 */
3158 rdrxctl |= IXGBE_RDRXCTL_MVMEN;
3159 break;
3160 case ixgbe_mac_82599EB:
b93a2226 3161 case ixgbe_mac_X540:
7367096a
AD		 3162 /* Disable RSC for ACK packets */
3163 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3164 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3165 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3166 /* hardware requires some bits to be set by default */
3167 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
3168 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3169 break;
3170 default:
3171 /* We should do nothing since we don't know this hardware */
3172 return;
3173 }
3174
3175 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3176}
3177
477de6ed
AD		 3178/**
3179 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
3180 * @adapter: board private structure
3181 *
3182 * Configure the Rx unit of the MAC after a reset.
3183 **/
3184static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
3185{
3186 struct ixgbe_hw *hw = &adapter->hw;
477de6ed
AD		 3187 int i;
3188 u32 rxctrl;
477de6ed
AD		 3189
3190 /* disable receives while setting up the descriptors */
3191 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3192 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
3193
3194 ixgbe_setup_psrtype(adapter);
7367096a 3195 ixgbe_setup_rdrxctl(adapter);
477de6ed 3196
9e10e045 3197 /* Program registers for the distribution of queues */
f5b4a52e 3198 ixgbe_setup_mrqc(adapter);
f5b4a52e 3199
9e10e045
AD		 3200 ixgbe_set_uta(adapter);
3201
477de6ed
AD		 3202 /* set_rx_buffer_len must be called before ring initialization */
3203 ixgbe_set_rx_buffer_len(adapter);
3204
3205 /*
3206 * Setup the HW Rx Head and Tail Descriptor Pointers and
3207 * the Base and Length of the Rx Descriptor Ring
3208 */
9e10e045
AD		 3209 for (i = 0; i < adapter->num_rx_queues; i++)
3210 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
177db6ff 3211
9e10e045
AD		 3212 /* disable drop enable for 82598 parts */
3213 if (hw->mac.type == ixgbe_mac_82598EB)
3214 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3215
3216 /* enable all receives */
3217 rxctrl |= IXGBE_RXCTRL_RXEN;
3218 hw->mac.ops.enable_rx_dma(hw, rxctrl);
9a799d71
AK		 3219}
3220
8e586137 3221static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
068c89b0
DS		 3222{
3223 struct ixgbe_adapter *adapter = netdev_priv(netdev);
3224 struct ixgbe_hw *hw = &adapter->hw;
1ada1b1b 3225 int pool_ndx = adapter->num_vfs;
068c89b0
DS		 3226
3227 /* add VID to filter table */
1ada1b1b 3228 hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, true);
f62bbb5e 3229 set_bit(vid, adapter->active_vlans);
8e586137
JP		 3230
3231 return 0;
068c89b0
DS		 3232}
3233
8e586137 3234static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
068c89b0
DS		 3235{
3236 struct ixgbe_adapter *adapter = netdev_priv(netdev);
3237 struct ixgbe_hw *hw = &adapter->hw;
1ada1b1b 3238 int pool_ndx = adapter->num_vfs;
068c89b0 3239
068c89b0 3240 /* remove VID from filter table */
1ada1b1b 3241 hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, false);
f62bbb5e 3242 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 3243
3244 return 0;
068c89b0
DS		 3245}
3246
5f6c0181
JB		 3247/**
3248 * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering
3249 * @adapter: driver data
3250 */
3251static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
3252{
3253 struct ixgbe_hw *hw = &adapter->hw;
f62bbb5e
JG		 3254 u32 vlnctrl;
3255
3256 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3257 vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
3258 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3259}
3260
3261/**
3262 * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering
3263 * @adapter: driver data
3264 */
3265static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
3266{
3267 struct ixgbe_hw *hw = &adapter->hw;
3268 u32 vlnctrl;
3269
3270 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3271 vlnctrl |= IXGBE_VLNCTRL_VFE;
3272 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
3273 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3274}
3275
3276/**
3277 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
3278 * @adapter: driver data
3279 */
3280static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
3281{
3282 struct ixgbe_hw *hw = &adapter->hw;
3283 u32 vlnctrl;
5f6c0181
JB		 3284 int i, j;
3285
3286 switch (hw->mac.type) {
3287 case ixgbe_mac_82598EB:
f62bbb5e
JG		 3288 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3289 vlnctrl &= ~IXGBE_VLNCTRL_VME;
5f6c0181
JB		 3290 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3291 break;
3292 case ixgbe_mac_82599EB:
b93a2226 3293 case ixgbe_mac_X540:
5f6c0181
JB		 3294 for (i = 0; i < adapter->num_rx_queues; i++) {
3295 j = adapter->rx_ring[i]->reg_idx;
3296 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3297 vlnctrl &= ~IXGBE_RXDCTL_VME;
3298 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3299 }
3300 break;
3301 default:
3302 break;
3303 }
3304}
3305
3306/**
f62bbb5e 3307 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
5f6c0181
JB		 3308 * @adapter: driver data
3309 */
f62bbb5e 3310static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
5f6c0181
JB		 3311{
3312 struct ixgbe_hw *hw = &adapter->hw;
f62bbb5e 3313 u32 vlnctrl;
5f6c0181
JB		 3314 int i, j;
3315
3316 switch (hw->mac.type) {
3317 case ixgbe_mac_82598EB:
f62bbb5e
JG		 3318 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3319 vlnctrl |= IXGBE_VLNCTRL_VME;
5f6c0181
JB		 3320 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3321 break;
3322 case ixgbe_mac_82599EB:
b93a2226 3323 case ixgbe_mac_X540:
5f6c0181
JB		 3324 for (i = 0; i < adapter->num_rx_queues; i++) {
3325 j = adapter->rx_ring[i]->reg_idx;
3326 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3327 vlnctrl |= IXGBE_RXDCTL_VME;
3328 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3329 }
3330 break;
3331 default:
3332 break;
3333 }
3334}
3335
9a799d71
AK		 3336static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
3337{
f62bbb5e 3338 u16 vid;
9a799d71 3339
f62bbb5e
JG		 3340 ixgbe_vlan_rx_add_vid(adapter->netdev, 0);
3341
3342 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
3343 ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
9a799d71
AK		 3344}
3345
2850062a
AD		 3346/**
3347 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
3348 * @netdev: network interface device structure
3349 *
3350 * Writes unicast address list to the RAR table.
3351 * Returns: -ENOMEM on failure/insufficient address space
3352 * 0 on no addresses written
3353 * X on writing X addresses to the RAR table
3354 **/
3355static int ixgbe_write_uc_addr_list(struct net_device *netdev)
3356{
3357 struct ixgbe_adapter *adapter = netdev_priv(netdev);
3358 struct ixgbe_hw *hw = &adapter->hw;
3359 unsigned int vfn = adapter->num_vfs;
a1cbb15c 3360 unsigned int rar_entries = IXGBE_MAX_PF_MACVLANS;
2850062a
AD		 3361 int count = 0;
3362
3363 /* return ENOMEM indicating insufficient memory for addresses */
3364 if (netdev_uc_count(netdev) > rar_entries)
3365 return -ENOMEM;
3366
3367 if (!netdev_uc_empty(netdev) && rar_entries) {
3368 struct netdev_hw_addr *ha;
3369 /* return error if we do not support writing to RAR table */
3370 if (!hw->mac.ops.set_rar)
3371 return -ENOMEM;
3372
3373 netdev_for_each_uc_addr(ha, netdev) {
3374 if (!rar_entries)
3375 break;
3376 hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
3377 vfn, IXGBE_RAH_AV);
3378 count++;
3379 }
3380 }
3381 /* write the addresses in reverse order to avoid write combining */
3382 for (; rar_entries > 0 ; rar_entries--)
3383 hw->mac.ops.clear_rar(hw, rar_entries);
3384
3385 return count;
3386}
3387
9a799d71 3388/**
2c5645cf 3389 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
9a799d71
AK		 3390 * @netdev: network interface device structure
3391 *
2c5645cf
CL		 3392 * The set_rx_method entry point is called whenever the unicast/multicast
3393 * address list or the network interface flags are updated. This routine is
3394 * responsible for configuring the hardware for proper unicast, multicast and
3395 * promiscuous mode.
9a799d71 3396 **/
7f870475 3397void ixgbe_set_rx_mode(struct net_device *netdev)
9a799d71
AK		 3398{
3399 struct ixgbe_adapter *adapter = netdev_priv(netdev);
3400 struct ixgbe_hw *hw = &adapter->hw;
2850062a
AD		 3401 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
3402 int count;
9a799d71
AK		 3403
3404 /* Check for Promiscuous and All Multicast modes */
3405
3406 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3407
f5dc442b
AD		 3408 /* set all bits that we expect to always be set */
3409 fctrl |= IXGBE_FCTRL_BAM;
3410 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
3411 fctrl |= IXGBE_FCTRL_PMCF;
3412
2850062a
AD		 3413 /* clear the bits we are changing the status of */
3414 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3415
9a799d71 3416 if (netdev->flags & IFF_PROMISC) {
e433ea1f 3417 hw->addr_ctrl.user_set_promisc = true;
9a799d71 3418 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2850062a 3419 vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
5f6c0181
JB		 3420 /* don't hardware filter vlans in promisc mode */
3421 ixgbe_vlan_filter_disable(adapter);
9a799d71 3422 } else {
746b9f02
PM		 3423 if (netdev->flags & IFF_ALLMULTI) {
3424 fctrl |= IXGBE_FCTRL_MPE;
2850062a
AD		 3425 vmolr |= IXGBE_VMOLR_MPE;
3426 } else {
3427 /*
3428 * Write addresses to the MTA, if the attempt fails
25985edc 3429 * then we should just turn on promiscuous mode so
2850062a
AD		 3430 * that we can at least receive multicast traffic
3431 */
3432 hw->mac.ops.update_mc_addr_list(hw, netdev);
3433 vmolr |= IXGBE_VMOLR_ROMPE;
746b9f02 3434 }
5f6c0181 3435 ixgbe_vlan_filter_enable(adapter);
e433ea1f 3436 hw->addr_ctrl.user_set_promisc = false;
2850062a
AD		 3437 /*
3438 * Write addresses to available RAR registers, if there is not
3439 * sufficient space to store all the addresses then enable
25985edc 3440 * unicast promiscuous mode
2850062a
AD		 3441 */
3442 count = ixgbe_write_uc_addr_list(netdev);
3443 if (count < 0) {
3444 fctrl |= IXGBE_FCTRL_UPE;
3445 vmolr |= IXGBE_VMOLR_ROPE;
3446 }
9a799d71
AK		 3447 }
3448
2850062a 3449 if (adapter->num_vfs) {
1cdd1ec8 3450 ixgbe_restore_vf_multicasts(adapter);
2850062a
AD		 3451 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) &
3452 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
3453 IXGBE_VMOLR_ROPE);
3454 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr);
3455 }
3456
3457 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
f62bbb5e
JG		 3458
3459 if (netdev->features & NETIF_F_HW_VLAN_RX)
3460 ixgbe_vlan_strip_enable(adapter);
3461 else
3462 ixgbe_vlan_strip_disable(adapter);
9a799d71
AK		 3463}
3464
021230d4
AV		 3465static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
3466{
3467 int q_idx;
3468 struct ixgbe_q_vector *q_vector;
3469 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
3470
3471 /* legacy and MSI only use one vector */
3472 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
3473 q_vectors = 1;
3474
3475 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
7a921c93 3476 q_vector = adapter->q_vector[q_idx];
4ff7fb12 3477 napi_enable(&q_vector->napi);
021230d4
AV		 3478 }
3479}
3480
3481static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
3482{
3483 int q_idx;
3484 struct ixgbe_q_vector *q_vector;
3485 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
3486
3487 /* legacy and MSI only use one vector */
3488 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
3489 q_vectors = 1;
3490
3491 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
7a921c93 3492 q_vector = adapter->q_vector[q_idx];
021230d4
AV		 3493 napi_disable(&q_vector->napi);
3494 }
3495}
3496
7a6b6f51 3497#ifdef CONFIG_IXGBE_DCB
2f90b865
AD		 3498/*
3499 * ixgbe_configure_dcb - Configure DCB hardware
3500 * @adapter: ixgbe adapter struct
3501 *
3502 * This is called by the driver on open to configure the DCB hardware.
3503 * This is also called by the gennetlink interface when reconfiguring
3504 * the DCB state.
3505 */
3506static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
3507{
3508 struct ixgbe_hw *hw = &adapter->hw;
9806307a 3509 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2f90b865 3510
67ebd791
AD		 3511 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
3512 if (hw->mac.type == ixgbe_mac_82598EB)
3513 netif_set_gso_max_size(adapter->netdev, 65536);
3514 return;
3515 }
3516
3517 if (hw->mac.type == ixgbe_mac_82598EB)
3518 netif_set_gso_max_size(adapter->netdev, 32768);
3519
2f90b865 3520
2f90b865 3521 /* Enable VLAN tag insert/strip */
f62bbb5e 3522 adapter->netdev->features |= NETIF_F_HW_VLAN_RX;
5f6c0181 3523
2f90b865 3524 hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true);
01fa7d90 3525
971060b1 3526#ifdef IXGBE_FCOE
b120818e
JF		 3527 if (adapter->netdev->features & NETIF_F_FCOE_MTU)
3528 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
c27931da 3529#endif
b120818e
JF		 3530
3531 /* reconfigure the hardware */
3532 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
c27931da
JF		 3533 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3534 DCB_TX_CONFIG);
3535 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3536 DCB_RX_CONFIG);
3537 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
b120818e
JF		 3538 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
3539 ixgbe_dcb_hw_ets(&adapter->hw,
3540 adapter->ixgbe_ieee_ets,
3541 max_frame);
3542 ixgbe_dcb_hw_pfc_config(&adapter->hw,
3543 adapter->ixgbe_ieee_pfc->pfc_en,
3544 adapter->ixgbe_ieee_ets->prio_tc);
c27931da 3545 }
8187cd48
JF		 3546
3547 /* Enable RSS Hash per TC */
3548 if (hw->mac.type != ixgbe_mac_82598EB) {
3549 int i;
3550 u32 reg = 0;
3551
3552 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
3553 u8 msb = 0;
3554 u8 cnt = adapter->netdev->tc_to_txq[i].count;
3555
3556 while (cnt >>= 1)
3557 msb++;
3558
3559 reg |= msb << IXGBE_RQTC_SHIFT_TC(i);
3560 }
3561 IXGBE_WRITE_REG(hw, IXGBE_RQTC, reg);
3562 }
2f90b865 3563}
9da712d2
JF		 3564#endif
3565
3566/* Additional bittime to account for IXGBE framing */
3567#define IXGBE_ETH_FRAMING 20
3568
3569/*
3570 * ixgbe_hpbthresh - calculate high water mark for flow control
3571 *
3572 * @adapter: board private structure to calculate for
3573 * @pb - packet buffer to calculate
3574 */
3575static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
3576{
3577 struct ixgbe_hw *hw = &adapter->hw;
3578 struct net_device *dev = adapter->netdev;
3579 int link, tc, kb, marker;
3580 u32 dv_id, rx_pba;
3581
3582 /* Calculate max LAN frame size */
3583 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
3584
3585#ifdef IXGBE_FCOE
3586 /* FCoE traffic class uses FCOE jumbo frames */
3587 if (dev->features & NETIF_F_FCOE_MTU) {
3588 int fcoe_pb = 0;
2f90b865 3589
9da712d2
JF		 3590#ifdef CONFIG_IXGBE_DCB
3591 fcoe_pb = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
3592
3593#endif
3594 if (fcoe_pb == pb && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE)
3595 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3596 }
2f90b865 3597#endif
80605c65 3598
9da712d2
JF		 3599 /* Calculate delay value for device */
3600 switch (hw->mac.type) {
3601 case ixgbe_mac_X540:
3602 dv_id = IXGBE_DV_X540(link, tc);
3603 break;
3604 default:
3605 dv_id = IXGBE_DV(link, tc);
3606 break;
3607 }
3608
3609 /* Loopback switch introduces additional latency */
3610 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3611 dv_id += IXGBE_B2BT(tc);
3612
3613 /* Delay value is calculated in bit times convert to KB */
3614 kb = IXGBE_BT2KB(dv_id);
3615 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
3616
3617 marker = rx_pba - kb;
3618
3619 /* It is possible that the packet buffer is not large enough
3620 * to provide required headroom. In this case throw an error
3621 * to user and a do the best we can.
3622 */
3623 if (marker < 0) {
3624 e_warn(drv, "Packet Buffer(%i) can not provide enough"
3625 "headroom to support flow control."
3626 "Decrease MTU or number of traffic classes\n", pb);
3627 marker = tc + 1;
3628 }
3629
3630 return marker;
3631}
3632
3633/*
3634 * ixgbe_lpbthresh - calculate low water mark for for flow control
3635 *
3636 * @adapter: board private structure to calculate for
3637 * @pb - packet buffer to calculate
3638 */
3639static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter)
3640{
3641 struct ixgbe_hw *hw = &adapter->hw;
3642 struct net_device *dev = adapter->netdev;
3643 int tc;
3644 u32 dv_id;
3645
3646 /* Calculate max LAN frame size */
3647 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
3648
3649 /* Calculate delay value for device */
3650 switch (hw->mac.type) {
3651 case ixgbe_mac_X540:
3652 dv_id = IXGBE_LOW_DV_X540(tc);
3653 break;
3654 default:
3655 dv_id = IXGBE_LOW_DV(tc);
3656 break;
3657 }
3658
3659 /* Delay value is calculated in bit times convert to KB */
3660 return IXGBE_BT2KB(dv_id);
3661}
3662
3663/*
3664 * ixgbe_pbthresh_setup - calculate and setup high low water marks
3665 */
3666static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
3667{
3668 struct ixgbe_hw *hw = &adapter->hw;
3669 int num_tc = netdev_get_num_tc(adapter->netdev);
3670 int i;
3671
3672 if (!num_tc)
3673 num_tc = 1;
3674
3675 hw->fc.low_water = ixgbe_lpbthresh(adapter);
3676
3677 for (i = 0; i < num_tc; i++) {
3678 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);
3679
3680 /* Low water marks must not be larger than high water marks */
3681 if (hw->fc.low_water > hw->fc.high_water[i])
3682 hw->fc.low_water = 0;
3683 }
3684}
3685
80605c65
JF		 3686static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
3687{
80605c65 3688 struct ixgbe_hw *hw = &adapter->hw;
f7e1027f
AD		 3689 int hdrm;
3690 u8 tc = netdev_get_num_tc(adapter->netdev);
80605c65
JF		 3691
3692 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
3693 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
f7e1027f
AD		 3694 hdrm = 32 << adapter->fdir_pballoc;
3695 else
3696 hdrm = 0;
80605c65 3697
f7e1027f 3698 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
9da712d2 3699 ixgbe_pbthresh_setup(adapter);
80605c65
JF		 3700}
3701
e4911d57
AD		 3702static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
3703{
3704 struct ixgbe_hw *hw = &adapter->hw;
3705 struct hlist_node *node, *node2;
3706 struct ixgbe_fdir_filter *filter;
3707
3708 spin_lock(&adapter->fdir_perfect_lock);
3709
3710 if (!hlist_empty(&adapter->fdir_filter_list))
3711 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);
3712
3713 hlist_for_each_entry_safe(filter, node, node2,
3714 &adapter->fdir_filter_list, fdir_node) {
3715 ixgbe_fdir_write_perfect_filter_82599(hw,
1f4d5183
AD		 3716 &filter->filter,
3717 filter->sw_idx,
3718 (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
3719 IXGBE_FDIR_DROP_QUEUE :
3720 adapter->rx_ring[filter->action]->reg_idx);
e4911d57
AD		 3721 }
3722
3723 spin_unlock(&adapter->fdir_perfect_lock);
3724}
3725
9a799d71
AK		 3726static void ixgbe_configure(struct ixgbe_adapter *adapter)
3727{
80605c65 3728 ixgbe_configure_pb(adapter);
7a6b6f51 3729#ifdef CONFIG_IXGBE_DCB
67ebd791 3730 ixgbe_configure_dcb(adapter);
2f90b865 3731#endif
9a799d71 3732
4c1d7b4b 3733 ixgbe_set_rx_mode(adapter->netdev);
f62bbb5e
JG		 3734 ixgbe_restore_vlan(adapter);
3735
eacd73f7
YZ		 3736#ifdef IXGBE_FCOE
3737 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
3738 ixgbe_configure_fcoe(adapter);
3739
3740#endif /* IXGBE_FCOE */
c4cf55e5 3741 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
4c1d7b4b
AD		 3742 ixgbe_init_fdir_signature_82599(&adapter->hw,
3743 adapter->fdir_pballoc);
e4911d57
AD		 3744 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
3745 ixgbe_init_fdir_perfect_82599(&adapter->hw,
3746 adapter->fdir_pballoc);
3747 ixgbe_fdir_filter_restore(adapter);
c4cf55e5 3748 }
4c1d7b4b 3749
933d41f1 3750 ixgbe_configure_virtualization(adapter);
c4cf55e5 3751
9a799d71
AK		 3752 ixgbe_configure_tx(adapter);
3753 ixgbe_configure_rx(adapter);
9a799d71
AK		 3754}
3755
e8e26350
PW		 3756static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
3757{
3758 switch (hw->phy.type) {
3759 case ixgbe_phy_sfp_avago:
3760 case ixgbe_phy_sfp_ftl:
3761 case ixgbe_phy_sfp_intel:
3762 case ixgbe_phy_sfp_unknown:
ea0a04df
DS		 3763 case ixgbe_phy_sfp_passive_tyco:
3764 case ixgbe_phy_sfp_passive_unknown:
3765 case ixgbe_phy_sfp_active_unknown:
3766 case ixgbe_phy_sfp_ftl_active:
e8e26350 3767 return true;
8917b447
AD		 3768 case ixgbe_phy_nl:
3769 if (hw->mac.type == ixgbe_mac_82598EB)
3770 return true;
e8e26350
PW		 3771 default:
3772 return false;
3773 }
3774}
3775
0ecc061d 3776/**
e8e26350
PW		 3777 * ixgbe_sfp_link_config - set up SFP+ link
3778 * @adapter: pointer to private adapter struct
3779 **/
3780static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
3781{
7086400d 3782 /*
52f33af8 3783 * We are assuming the worst case scenario here, and that
7086400d
AD		 3784 * is that an SFP was inserted/removed after the reset
3785 * but before SFP detection was enabled. As such the best
3786 * solution is to just start searching as soon as we start
3787 */
3788 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
3789 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
e8e26350 3790
7086400d 3791 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
e8e26350
PW		 3792}
3793
3794/**
3795 * ixgbe_non_sfp_link_config - set up non-SFP+ link
0ecc061d
PWJ		 3796 * @hw: pointer to private hardware struct
3797 *
3798 * Returns 0 on success, negative on failure
3799 **/
e8e26350 3800static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
0ecc061d
PWJ		 3801{
3802 u32 autoneg;
8620a103 3803 bool negotiation, link_up = false;
0ecc061d
PWJ		 3804 u32 ret = IXGBE_ERR_LINK_SETUP;
3805
3806 if (hw->mac.ops.check_link)
3807 ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
3808
3809 if (ret)
3810 goto link_cfg_out;
3811
0b0c2b31
ET		 3812 autoneg = hw->phy.autoneg_advertised;
3813 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
e8e9f696
JP		 3814 ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
3815 &negotiation);
0ecc061d
PWJ		 3816 if (ret)
3817 goto link_cfg_out;
3818
8620a103
MC		 3819 if (hw->mac.ops.setup_link)
3820 ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
0ecc061d
PWJ		 3821link_cfg_out:
3822 return ret;
3823}
3824
a34bcfff 3825static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
9a799d71 3826{
9a799d71 3827 struct ixgbe_hw *hw = &adapter->hw;
a34bcfff 3828 u32 gpie = 0;
9a799d71 3829
9b471446 3830 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
a34bcfff
AD		 3831 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
3832 IXGBE_GPIE_OCD;
3833 gpie |= IXGBE_GPIE_EIAME;
9b471446
JB		 3834 /*
3835 * use EIAM to auto-mask when MSI-X interrupt is asserted
3836 * this saves a register write for every interrupt
3837 */
3838 switch (hw->mac.type) {
3839 case ixgbe_mac_82598EB:
3840 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3841 break;
9b471446 3842 case ixgbe_mac_82599EB:
b93a2226
DS		 3843 case ixgbe_mac_X540:
3844 default:
9b471446
JB		 3845 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
3846 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
3847 break;
3848 }
3849 } else {
021230d4
AV		 3850 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
3851 * specifically only auto mask tx and rx interrupts */
3852 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3853 }
9a799d71 3854
a34bcfff
AD		 3855 /* XXX: to interrupt immediately for EICS writes, enable this */
3856 /* gpie |= IXGBE_GPIE_EIMEN; */
3857
3858 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3859 gpie &= ~IXGBE_GPIE_VTMODE_MASK;
3860 gpie |= IXGBE_GPIE_VTMODE_64;
119fc60a
MC		 3861 }
3862
5fdd31f9 3863 /* Enable Thermal over heat sensor interrupt */
f3df98ec
DS		 3864 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
3865 switch (adapter->hw.mac.type) {
3866 case ixgbe_mac_82599EB:
3867 gpie |= IXGBE_SDP0_GPIEN;
3868 break;
3869 case ixgbe_mac_X540:
3870 gpie |= IXGBE_EIMS_TS;
3871 break;
3872 default:
3873 break;
3874 }
3875 }
5fdd31f9 3876
a34bcfff
AD		 3877 /* Enable fan failure interrupt */
3878 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
0befdb3e 3879 gpie |= IXGBE_SDP1_GPIEN;
0befdb3e 3880
2698b208 3881 if (hw->mac.type == ixgbe_mac_82599EB) {
e8e26350
PW		 3882 gpie |= IXGBE_SDP1_GPIEN;
3883 gpie |= IXGBE_SDP2_GPIEN;
2698b208 3884 }
a34bcfff
AD		 3885
3886 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3887}
3888
c7ccde0f 3889static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
a34bcfff
AD		 3890{
3891 struct ixgbe_hw *hw = &adapter->hw;
a34bcfff 3892 int err;
a34bcfff
AD		 3893 u32 ctrl_ext;
3894
3895 ixgbe_get_hw_control(adapter);
3896 ixgbe_setup_gpie(adapter);
e8e26350 3897
9a799d71
AK		 3898 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
3899 ixgbe_configure_msix(adapter);
3900 else
3901 ixgbe_configure_msi_and_legacy(adapter);
3902
c6ecf39a
DS		 3903 /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
3904 if (hw->mac.ops.enable_tx_laser &&
3905 ((hw->phy.multispeed_fiber) ||
9f911707 3906 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
c6ecf39a 3907 (hw->mac.type == ixgbe_mac_82599EB))))
61fac744
PW		 3908 hw->mac.ops.enable_tx_laser(hw);
3909
9a799d71 3910 clear_bit(__IXGBE_DOWN, &adapter->state);
021230d4
AV		 3911 ixgbe_napi_enable_all(adapter);
3912
73c4b7cd
AD		 3913 if (ixgbe_is_sfp(hw)) {
3914 ixgbe_sfp_link_config(adapter);
3915 } else {
3916 err = ixgbe_non_sfp_link_config(hw);
3917 if (err)
3918 e_err(probe, "link_config FAILED %d\n", err);
3919 }
3920
021230d4
AV		 3921 /* clear any pending interrupts, may auto mask */
3922 IXGBE_READ_REG(hw, IXGBE_EICR);
6af3b9eb 3923 ixgbe_irq_enable(adapter, true, true);
9a799d71 3924
bf069c97
DS		 3925 /*
3926 * If this adapter has a fan, check to see if we had a failure
3927 * before we enabled the interrupt.
3928 */
3929 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
3930 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
3931 if (esdp & IXGBE_ESDP_SDP1)
396e799c 3932 e_crit(drv, "Fan has stopped, replace the adapter\n");
bf069c97
DS		 3933 }
3934
1da100bb 3935 /* enable transmits */
477de6ed 3936 netif_tx_start_all_queues(adapter->netdev);
1da100bb 3937
9a799d71
AK		 3938 /* bring the link up in the watchdog, this could race with our first
3939 * link up interrupt but shouldn't be a problem */
cf8280ee
JB		 3940 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3941 adapter->link_check_timeout = jiffies;
7086400d 3942 mod_timer(&adapter->service_timer, jiffies);
c9205697
GR		 3943
3944 /* Set PF Reset Done bit so PF/VF Mail Ops can work */
3945 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3946 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
3947 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
9a799d71
AK		 3948}
3949
d4f80882
AV		 3950void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
3951{
3952 WARN_ON(in_interrupt());
7086400d
AD		 3953 /* put off any impending NetWatchDogTimeout */
3954 adapter->netdev->trans_start = jiffies;
3955
d4f80882 3956 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
032b4325 3957 usleep_range(1000, 2000);
d4f80882 3958 ixgbe_down(adapter);
5809a1ae
GR		 3959 /*
3960 * If SR-IOV enabled then wait a bit before bringing the adapter
3961 * back up to give the VFs time to respond to the reset. The
3962 * two second wait is based upon the watchdog timer cycle in
3963 * the VF driver.
3964 */
3965 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3966 msleep(2000);
d4f80882
AV		 3967 ixgbe_up(adapter);
3968 clear_bit(__IXGBE_RESETTING, &adapter->state);
3969}
3970
c7ccde0f 3971void ixgbe_up(struct ixgbe_adapter *adapter)
9a799d71
AK		 3972{
3973 /* hardware has been reset, we need to reload some things */
3974 ixgbe_configure(adapter);
3975
c7ccde0f 3976 ixgbe_up_complete(adapter);
9a799d71
AK		 3977}
3978
3979void ixgbe_reset(struct ixgbe_adapter *adapter)
3980{
c44ade9e 3981 struct ixgbe_hw *hw = &adapter->hw;
8ca783ab
DS		 3982 int err;
3983
7086400d
AD		 3984 /* lock SFP init bit to prevent race conditions with the watchdog */
3985 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
3986 usleep_range(1000, 2000);
3987
3988 /* clear all SFP and link config related flags while holding SFP_INIT */
3989 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
3990 IXGBE_FLAG2_SFP_NEEDS_RESET);
3991 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
3992
8ca783ab 3993 err = hw->mac.ops.init_hw(hw);
da4dd0f7
PWJ		 3994 switch (err) {
3995 case 0:
3996 case IXGBE_ERR_SFP_NOT_PRESENT:
7086400d 3997 case IXGBE_ERR_SFP_NOT_SUPPORTED:
da4dd0f7
PWJ		 3998 break;
3999 case IXGBE_ERR_MASTER_REQUESTS_PENDING:
849c4542 4000 e_dev_err("master disable timed out\n");
da4dd0f7 4001 break;
794caeb2
PWJ		 4002 case IXGBE_ERR_EEPROM_VERSION:
4003 /* We are running on a pre-production device, log a warning */
849c4542 4004 e_dev_warn("This device is a pre-production adapter/LOM. "
52f33af8 4005 "Please be aware there may be issues associated with "
849c4542
ET		 4006 "your hardware. If you are experiencing problems "
4007 "please contact your Intel or hardware "
4008 "representative who provided you with this "
4009 "hardware.\n");
794caeb2 4010 break;
da4dd0f7 4011 default:
849c4542 4012 e_dev_err("Hardware Error: %d\n", err);
da4dd0f7 4013 }
9a799d71 4014
7086400d
AD		 4015 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
4016
9a799d71 4017 /* reprogram the RAR[0] in case user changed it. */
1cdd1ec8
GR		 4018 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
4019 IXGBE_RAH_AV);
9a799d71
AK		 4020}
4021
9a799d71
AK		 4022/**
4023 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
9a799d71
AK		 4024 * @rx_ring: ring to free buffers from
4025 **/
b6ec895e 4026static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
9a799d71 4027{
b6ec895e 4028 struct device *dev = rx_ring->dev;
9a799d71 4029 unsigned long size;
b6ec895e 4030 u16 i;
9a799d71 4031
84418e3b
AD		 4032 /* ring already cleared, nothing to do */
4033 if (!rx_ring->rx_buffer_info)
4034 return;
9a799d71 4035
84418e3b 4036 /* Free all the Rx ring sk_buffs */
9a799d71
AK		 4037 for (i = 0; i < rx_ring->count; i++) {
4038 struct ixgbe_rx_buffer *rx_buffer_info;
4039
4040 rx_buffer_info = &rx_ring->rx_buffer_info[i];
4041 if (rx_buffer_info->dma) {
b6ec895e 4042 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
e8e9f696 4043 rx_ring->rx_buf_len,
1b507730 4044 DMA_FROM_DEVICE);
9a799d71
AK		 4045 rx_buffer_info->dma = 0;
4046 }
4047 if (rx_buffer_info->skb) {
f8212f97 4048 struct sk_buff *skb = rx_buffer_info->skb;
9a799d71 4049 rx_buffer_info->skb = NULL;
4c1975d7
AD		 4050 /* We need to clean up RSC frag lists */
4051 skb = ixgbe_merge_active_tail(skb);
4052 ixgbe_close_active_frag_list(skb);
4053 if (IXGBE_CB(skb)->delay_unmap) {
4054 dma_unmap_single(dev,
4055 IXGBE_CB(skb)->dma,
4056 rx_ring->rx_buf_len,
4057 DMA_FROM_DEVICE);
4058 IXGBE_CB(skb)->dma = 0;
4059 IXGBE_CB(skb)->delay_unmap = false;
4060 }
4061 dev_kfree_skb(skb);
9a799d71
AK		 4062 }
4063 if (!rx_buffer_info->page)
4064 continue;
4f57ca6e 4065 if (rx_buffer_info->page_dma) {
b6ec895e 4066 dma_unmap_page(dev, rx_buffer_info->page_dma,
1b507730 4067 PAGE_SIZE / 2, DMA_FROM_DEVICE);
4f57ca6e
JB		 4068 rx_buffer_info->page_dma = 0;
4069 }
9a799d71
AK		 4070 put_page(rx_buffer_info->page);
4071 rx_buffer_info->page = NULL;
762f4c57 4072 rx_buffer_info->page_offset = 0;
9a799d71
AK		 4073 }
4074
4075 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4076 memset(rx_ring->rx_buffer_info, 0, size);
4077
4078 /* Zero out the descriptor ring */
4079 memset(rx_ring->desc, 0, rx_ring->size);
4080
4081 rx_ring->next_to_clean = 0;
4082 rx_ring->next_to_use = 0;
9a799d71
AK		 4083}
4084
4085/**
4086 * ixgbe_clean_tx_ring - Free Tx Buffers
9a799d71
AK		 4087 * @tx_ring: ring to be cleaned
4088 **/
b6ec895e 4089static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
9a799d71
AK		 4090{
4091 struct ixgbe_tx_buffer *tx_buffer_info;
4092 unsigned long size;
b6ec895e 4093 u16 i;
9a799d71 4094
84418e3b
AD		 4095 /* ring already cleared, nothing to do */
4096 if (!tx_ring->tx_buffer_info)
4097 return;
9a799d71 4098
84418e3b 4099 /* Free all the Tx ring sk_buffs */
9a799d71
AK		 4100 for (i = 0; i < tx_ring->count; i++) {
4101 tx_buffer_info = &tx_ring->tx_buffer_info[i];
b6ec895e 4102 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
9a799d71
AK		 4103 }
4104
4105 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
4106 memset(tx_ring->tx_buffer_info, 0, size);
4107
4108 /* Zero out the descriptor ring */
4109 memset(tx_ring->desc, 0, tx_ring->size);
4110
4111 tx_ring->next_to_use = 0;
4112 tx_ring->next_to_clean = 0;
9a799d71
AK		 4113}
4114
4115/**
021230d4 4116 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
9a799d71
AK		 4117 * @adapter: board private structure
4118 **/
021230d4 4119static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
9a799d71
AK		 4120{
4121 int i;
4122
021230d4 4123 for (i = 0; i < adapter->num_rx_queues; i++)
b6ec895e 4124 ixgbe_clean_rx_ring(adapter->rx_ring[i]);
9a799d71
AK		 4125}
4126
4127/**
021230d4 4128 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
9a799d71
AK		 4129 * @adapter: board private structure
4130 **/
021230d4 4131static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
9a799d71
AK		 4132{
4133 int i;
4134
021230d4 4135 for (i = 0; i < adapter->num_tx_queues; i++)
b6ec895e 4136 ixgbe_clean_tx_ring(adapter->tx_ring[i]);
9a799d71
AK		 4137}
4138
e4911d57
AD		 4139static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
4140{
4141 struct hlist_node *node, *node2;
4142 struct ixgbe_fdir_filter *filter;
4143
4144 spin_lock(&adapter->fdir_perfect_lock);
4145
4146 hlist_for_each_entry_safe(filter, node, node2,
4147 &adapter->fdir_filter_list, fdir_node) {
4148 hlist_del(&filter->fdir_node);
4149 kfree(filter);
4150 }
4151 adapter->fdir_filter_count = 0;
4152
4153 spin_unlock(&adapter->fdir_perfect_lock);
4154}
4155
9a799d71
AK		 4156void ixgbe_down(struct ixgbe_adapter *adapter)
4157{
4158 struct net_device *netdev = adapter->netdev;
7f821875 4159 struct ixgbe_hw *hw = &adapter->hw;
9a799d71 4160 u32 rxctrl;
bf29ee6c 4161 int i;
9a799d71
AK		 4162
4163 /* signal that we are down to the interrupt handler */
4164 set_bit(__IXGBE_DOWN, &adapter->state);
4165
4166 /* disable receives */
7f821875
JB		 4167 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4168 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
9a799d71 4169
2d39d576
YZ		 4170 /* disable all enabled rx queues */
4171 for (i = 0; i < adapter->num_rx_queues; i++)
4172 /* this call also flushes the previous write */
4173 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
4174
032b4325 4175 usleep_range(10000, 20000);
9a799d71 4176
7f821875
JB		 4177 netif_tx_stop_all_queues(netdev);
4178
7086400d 4179 /* call carrier off first to avoid false dev_watchdog timeouts */
c0dfb90e
JF		 4180 netif_carrier_off(netdev);
4181 netif_tx_disable(netdev);
4182
4183 ixgbe_irq_disable(adapter);
4184
4185 ixgbe_napi_disable_all(adapter);
4186
d034acf1
AD		 4187 adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
4188 IXGBE_FLAG2_RESET_REQUESTED);
7086400d
AD		 4189 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
4190
4191 del_timer_sync(&adapter->service_timer);
4192
34cecbbf 4193 if (adapter->num_vfs) {
8e34d1aa
AD		 4194 /* Clear EITR Select mapping */
4195 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
34cecbbf
AD		 4196
4197 /* Mark all the VFs as inactive */
4198 for (i = 0 ; i < adapter->num_vfs; i++)
3db1cd5c 4199 adapter->vfinfo[i].clear_to_send = false;
34cecbbf 4200
34cecbbf
AD		 4201 /* ping all the active vfs to let them know we are going down */
4202 ixgbe_ping_all_vfs(adapter);
4203
4204 /* Disable all VFTE/VFRE TX/RX */
4205 ixgbe_disable_tx_rx(adapter);
b25ebfd2
PW		 4206 }
4207
7f821875
JB		 4208 /* disable transmits in the hardware now that interrupts are off */
4209 for (i = 0; i < adapter->num_tx_queues; i++) {
bf29ee6c 4210 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
34cecbbf 4211 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
7f821875 4212 }
34cecbbf
AD		 4213
4214 /* Disable the Tx DMA engine on 82599 and X540 */
bd508178
AD		 4215 switch (hw->mac.type) {
4216 case ixgbe_mac_82599EB:
b93a2226 4217 case ixgbe_mac_X540:
88512539 4218 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
e8e9f696
JP		 4219 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
4220 ~IXGBE_DMATXCTL_TE));
bd508178
AD		 4221 break;
4222 default:
4223 break;
4224 }
7f821875 4225
6f4a0e45
PL		 4226 if (!pci_channel_offline(adapter->pdev))
4227 ixgbe_reset(adapter);
c6ecf39a
DS		 4228
4229 /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
4230 if (hw->mac.ops.disable_tx_laser &&
4231 ((hw->phy.multispeed_fiber) ||
9f911707 4232 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
c6ecf39a
DS		 4233 (hw->mac.type == ixgbe_mac_82599EB))))
4234 hw->mac.ops.disable_tx_laser(hw);
4235
9a799d71
AK		 4236 ixgbe_clean_all_tx_rings(adapter);
4237 ixgbe_clean_all_rx_rings(adapter);
4238
5dd2d332 4239#ifdef CONFIG_IXGBE_DCA
96b0e0f6 4240 /* since we reset the hardware DCA settings were cleared */
e35ec126 4241 ixgbe_setup_dca(adapter);
96b0e0f6 4242#endif
9a799d71
AK		 4243}
4244
9a799d71 4245/**
021230d4
AV		 4246 * ixgbe_poll - NAPI Rx polling callback
4247 * @napi: structure for representing this polling device
4248 * @budget: how many packets driver is allowed to clean
4249 *
4250 * This function is used for legacy and MSI, NAPI mode
9a799d71 4251 **/
021230d4 4252static int ixgbe_poll(struct napi_struct *napi, int budget)
9a799d71 4253{
9a1a69ad 4254 struct ixgbe_q_vector *q_vector =
e8e9f696 4255 container_of(napi, struct ixgbe_q_vector, napi);
021230d4 4256 struct ixgbe_adapter *adapter = q_vector->adapter;
4ff7fb12
AD		 4257 struct ixgbe_ring *ring;
4258 int per_ring_budget;
4259 bool clean_complete = true;
9a799d71 4260
5dd2d332 4261#ifdef CONFIG_IXGBE_DCA
33cf09c9
AD		 4262 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
4263 ixgbe_update_dca(q_vector);
bd0362dd
JC		 4264#endif
4265
4ff7fb12
AD		 4266 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
4267 clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring);
9a799d71 4268
4ff7fb12
AD		 4269 /* attempt to distribute budget to each queue fairly, but don't allow
4270 * the budget to go below 1 because we'll exit polling */
4271 if (q_vector->rx.count > 1)
4272 per_ring_budget = max(budget/q_vector->rx.count, 1);
4273 else
4274 per_ring_budget = budget;
d2c7ddd6 4275
4ff7fb12
AD		 4276 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
4277 clean_complete &= ixgbe_clean_rx_irq(q_vector, ring,
4278 per_ring_budget);
4279
4280 /* If all work not completed, return budget and keep polling */
4281 if (!clean_complete)
4282 return budget;
4283
4284 /* all work done, exit the polling mode */
4285 napi_complete(napi);
4286 if (adapter->rx_itr_setting & 1)
4287 ixgbe_set_itr(q_vector);
4288 if (!test_bit(__IXGBE_DOWN, &adapter->state))
4289 ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));
4290
4291 return 0;
9a799d71
AK		 4292}
4293
4294/**
4295 * ixgbe_tx_timeout - Respond to a Tx Hang
4296 * @netdev: network interface device structure
4297 **/
4298static void ixgbe_tx_timeout(struct net_device *netdev)
4299{
4300 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4301
4302 /* Do the reset outside of interrupt context */
c83c6cbd 4303 ixgbe_tx_timeout_reset(adapter);
9a799d71
AK		 4304}
4305
4df10466
JB		 4306/**
4307 * ixgbe_set_rss_queues: Allocate queues for RSS
4308 * @adapter: board private structure to initialize
4309 *
4310 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try
4311 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
4312 *
4313 **/
bc97114d
PWJ		 4314static inline bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter)
4315{
4316 bool ret = false;
0cefafad 4317 struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_RSS];
bc97114d
PWJ		 4318
4319 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
0cefafad
JB		 4320 f->mask = 0xF;
4321 adapter->num_rx_queues = f->indices;
4322 adapter->num_tx_queues = f->indices;
bc97114d
PWJ		 4323 ret = true;
4324 } else {
bc97114d 4325 ret = false;
b9804972
JB		 4326 }
4327
bc97114d
PWJ		 4328 return ret;
4329}
4330
c4cf55e5
PWJ		 4331/**
4332 * ixgbe_set_fdir_queues: Allocate queues for Flow Director
4333 * @adapter: board private structure to initialize
4334 *
4335 * Flow Director is an advanced Rx filter, attempting to get Rx flows back
4336 * to the original CPU that initiated the Tx session. This runs in addition
4337 * to RSS, so if a packet doesn't match an FDIR filter, we can still spread the
4338 * Rx load across CPUs using RSS.
4339 *
4340 **/
e8e9f696 4341static inline bool ixgbe_set_fdir_queues(struct ixgbe_adapter *adapter)
c4cf55e5
PWJ		 4342{
4343 bool ret = false;
4344 struct ixgbe_ring_feature *f_fdir = &adapter->ring_feature[RING_F_FDIR];
4345
4346 f_fdir->indices = min((int)num_online_cpus(), f_fdir->indices);
4347 f_fdir->mask = 0;
4348
4349 /* Flow Director must have RSS enabled */
03ecf91a
AD		 4350 if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
4351 (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
c4cf55e5
PWJ		 4352 adapter->num_tx_queues = f_fdir->indices;
4353 adapter->num_rx_queues = f_fdir->indices;
4354 ret = true;
4355 } else {
4356 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
c4cf55e5
PWJ		 4357 }
4358 return ret;
4359}
4360
0331a832
YZ		 4361#ifdef IXGBE_FCOE
4362/**
4363 * ixgbe_set_fcoe_queues: Allocate queues for Fiber Channel over Ethernet (FCoE)
4364 * @adapter: board private structure to initialize
4365 *
4366 * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges.
4367 * The ring feature mask is not used as a mask for FCoE, as it can take any 8
4368 * rx queues out of the max number of rx queues, instead, it is used as the
4369 * index of the first rx queue used by FCoE.
4370 *
4371 **/
4372static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter)
4373{
0331a832
YZ		 4374 struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
4375
e5b64635
JF		 4376 if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
4377 return false;
4378
e901acd6 4379 f->indices = min((int)num_online_cpus(), f->indices);
e5b64635 4380
e901acd6
JF		 4381 adapter->num_rx_queues = 1;
4382 adapter->num_tx_queues = 1;
e5b64635 4383
e901acd6
JF		 4384 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4385 e_info(probe, "FCoE enabled with RSS\n");
03ecf91a 4386 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
e901acd6
JF		 4387 ixgbe_set_fdir_queues(adapter);
4388 else
4389 ixgbe_set_rss_queues(adapter);
e5b64635 4390 }
03ecf91a 4391
e901acd6
JF		 4392 /* adding FCoE rx rings to the end */
4393 f->mask = adapter->num_rx_queues;
4394 adapter->num_rx_queues += f->indices;
4395 adapter->num_tx_queues += f->indices;
0331a832 4396
e5b64635
JF		 4397 return true;
4398}
4399#endif /* IXGBE_FCOE */
4400
e901acd6
JF		 4401/* Artificial max queue cap per traffic class in DCB mode */
4402#define DCB_QUEUE_CAP 8
4403
e5b64635
JF		 4404#ifdef CONFIG_IXGBE_DCB
4405static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter)
4406{
e901acd6
JF		 4407 int per_tc_q, q, i, offset = 0;
4408 struct net_device *dev = adapter->netdev;
4409 int tcs = netdev_get_num_tc(dev);
e5b64635 4410
e901acd6
JF		 4411 if (!tcs)
4412 return false;
e5b64635 4413
e901acd6
JF		 4414 /* Map queue offset and counts onto allocated tx queues */
4415 per_tc_q = min(dev->num_tx_queues / tcs, (unsigned int)DCB_QUEUE_CAP);
4416 q = min((int)num_online_cpus(), per_tc_q);
8b1c0b24 4417
8b1c0b24 4418 for (i = 0; i < tcs; i++) {
e901acd6
JF		 4419 netdev_set_tc_queue(dev, i, q, offset);
4420 offset += q;
0331a832
YZ		 4421 }
4422
e901acd6
JF		 4423 adapter->num_tx_queues = q * tcs;
4424 adapter->num_rx_queues = q * tcs;
e5b64635
JF		 4425
4426#ifdef IXGBE_FCOE
e901acd6
JF		 4427 /* FCoE enabled queues require special configuration indexed
4428 * by feature specific indices and mask. Here we map FCoE
4429 * indices onto the DCB queue pairs allowing FCoE to own
4430 * configuration later.
e5b64635 4431 */
e901acd6
JF		 4432 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) {
4433 int tc;
4434 struct ixgbe_ring_feature *f =
4435 &adapter->ring_feature[RING_F_FCOE];
4436
4437 tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
4438 f->indices = dev->tc_to_txq[tc].count;
4439 f->mask = dev->tc_to_txq[tc].offset;
4440 }
e5b64635
JF		 4441#endif
4442
e901acd6 4443 return true;
0331a832 4444}
e5b64635 4445#endif
0331a832 4446
1cdd1ec8
GR		 4447/**
4448 * ixgbe_set_sriov_queues: Allocate queues for IOV use
4449 * @adapter: board private structure to initialize
4450 *
4451 * IOV doesn't actually use anything, so just NAK the
4452 * request for now and let the other queue routines
4453 * figure out what to do.
4454 */
4455static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
4456{
4457 return false;
4458}
4459
4df10466 4460/*
25985edc 4461 * ixgbe_set_num_queues: Allocate queues for device, feature dependent
4df10466
JB		 4462 * @adapter: board private structure to initialize
4463 *
4464 * This is the top level queue allocation routine. The order here is very
4465 * important, starting with the "most" number of features turned on at once,
4466 * and ending with the smallest set of features. This way large combinations
4467 * can be allocated if they're turned on, and smaller combinations are the
4468 * fallthrough conditions.
4469 *
4470 **/
847f53ff 4471static int ixgbe_set_num_queues(struct ixgbe_adapter *adapter)
bc97114d 4472{
1cdd1ec8
GR		 4473 /* Start with base case */
4474 adapter->num_rx_queues = 1;
4475 adapter->num_tx_queues = 1;
4476 adapter->num_rx_pools = adapter->num_rx_queues;
4477 adapter->num_rx_queues_per_pool = 1;
4478
4479 if (ixgbe_set_sriov_queues(adapter))
847f53ff 4480 goto done;
1cdd1ec8 4481
bc97114d
PWJ		 4482#ifdef CONFIG_IXGBE_DCB
4483 if (ixgbe_set_dcb_queues(adapter))
af22ab1b 4484 goto done;
bc97114d
PWJ		 4485
4486#endif
e5b64635
JF		 4487#ifdef IXGBE_FCOE
4488 if (ixgbe_set_fcoe_queues(adapter))
4489 goto done;
4490
4491#endif /* IXGBE_FCOE */
c4cf55e5
PWJ		 4492 if (ixgbe_set_fdir_queues(adapter))
4493 goto done;
4494
bc97114d 4495 if (ixgbe_set_rss_queues(adapter))
af22ab1b
WF		 4496 goto done;
4497
4498 /* fallback to base case */
4499 adapter->num_rx_queues = 1;
4500 adapter->num_tx_queues = 1;
4501
4502done:
847f53ff 4503 /* Notify the stack of the (possibly) reduced queue counts. */
f0796d5c 4504 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
847f53ff
BH		 4505 return netif_set_real_num_rx_queues(adapter->netdev,
4506 adapter->num_rx_queues);
b9804972
JB		 4507}
4508
021230d4 4509static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter,
e8e9f696 4510 int vectors)
021230d4
AV		 4511{
4512 int err, vector_threshold;
4513
4514 /* We'll want at least 3 (vector_threshold):
4515 * 1) TxQ[0] Cleanup
4516 * 2) RxQ[0] Cleanup
4517 * 3) Other (Link Status Change, etc.)
4518 * 4) TCP Timer (optional)
4519 */
4520 vector_threshold = MIN_MSIX_COUNT;
4521
4522 /* The more we get, the more we will assign to Tx/Rx Cleanup
4523 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
4524 * Right now, we simply care about how many we'll get; we'll
4525 * set them up later while requesting irq's.
4526 */
4527 while (vectors >= vector_threshold) {
4528 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
e8e9f696 4529 vectors);
021230d4
AV		 4530 if (!err) /* Success in acquiring all requested vectors. */
4531 break;
4532 else if (err < 0)
4533 vectors = 0; /* Nasty failure, quit now */
4534 else /* err == number of vectors we should try again with */
4535 vectors = err;
4536 }
4537
4538 if (vectors < vector_threshold) {
4539 /* Can't allocate enough MSI-X interrupts? Oh well.
4540 * This just means we'll go with either a single MSI
4541 * vector or fall back to legacy interrupts.
4542 */
849c4542
ET		 4543 netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
4544 "Unable to allocate MSI-X interrupts\n");
021230d4
AV		 4545 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
4546 kfree(adapter->msix_entries);
4547 adapter->msix_entries = NULL;
021230d4
AV		 4548 } else {
4549 adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */
eb7f139c
PWJ		 4550 /*
4551 * Adjust for only the vectors we'll use, which is minimum
4552 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
4553 * vectors we were allocated.
4554 */
4555 adapter->num_msix_vectors = min(vectors,
e8e9f696 4556 adapter->max_msix_q_vectors + NON_Q_VECTORS);
021230d4
AV		 4557 }
4558}
4559
021230d4 4560/**
bc97114d 4561 * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS
021230d4
AV		 4562 * @adapter: board private structure to initialize
4563 *
bc97114d
PWJ		 4564 * Cache the descriptor ring offsets for RSS to the assigned rings.
4565 *
021230d4 4566 **/
bc97114d 4567static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter)
021230d4 4568{
bc97114d 4569 int i;
bc97114d 4570
9d6b758f
AD		 4571 if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
4572 return false;
bc97114d 4573
9d6b758f
AD		 4574 for (i = 0; i < adapter->num_rx_queues; i++)
4575 adapter->rx_ring[i]->reg_idx = i;
4576 for (i = 0; i < adapter->num_tx_queues; i++)
4577 adapter->tx_ring[i]->reg_idx = i;
4578
4579 return true;
bc97114d
PWJ		 4580}
4581
4582#ifdef CONFIG_IXGBE_DCB
e5b64635
JF		 4583
4584/* ixgbe_get_first_reg_idx - Return first register index associated with ring */
b32c8dcc
JF		 4585static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
4586 unsigned int *tx, unsigned int *rx)
e5b64635
JF		 4587{
4588 struct net_device *dev = adapter->netdev;
4589 struct ixgbe_hw *hw = &adapter->hw;
4590 u8 num_tcs = netdev_get_num_tc(dev);
4591
4592 *tx = 0;
4593 *rx = 0;
4594
4595 switch (hw->mac.type) {
4596 case ixgbe_mac_82598EB:
aba70d5e
JF		 4597 *tx = tc << 2;
4598 *rx = tc << 3;
e5b64635
JF		 4599 break;
4600 case ixgbe_mac_82599EB:
4601 case ixgbe_mac_X540:
4fa2e0e1 4602 if (num_tcs > 4) {
e5b64635
JF		 4603 if (tc < 3) {
4604 *tx = tc << 5;
4605 *rx = tc << 4;
4606 } else if (tc < 5) {
4607 *tx = ((tc + 2) << 4);
4608 *rx = tc << 4;
4609 } else if (tc < num_tcs) {
4610 *tx = ((tc + 8) << 3);
4611 *rx = tc << 4;
4612 }
4fa2e0e1 4613 } else {
e5b64635
JF		 4614 *rx = tc << 5;
4615 switch (tc) {
4616 case 0:
4617 *tx = 0;
4618 break;
4619 case 1:
4620 *tx = 64;
4621 break;
4622 case 2:
4623 *tx = 96;
4624 break;
4625 case 3:
4626 *tx = 112;
4627 break;
4628 default:
4629 break;
4630 }
4631 }
4632 break;
4633 default:
4634 break;
4635 }
4636}
4637
bc97114d
PWJ		 4638/**
4639 * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB
4640 * @adapter: board private structure to initialize
4641 *
4642 * Cache the descriptor ring offsets for DCB to the assigned rings.
4643 *
4644 **/
4645static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
4646{
e5b64635
JF		 4647 struct net_device *dev = adapter->netdev;
4648 int i, j, k;
4649 u8 num_tcs = netdev_get_num_tc(dev);
bc97114d 4650
8b1c0b24 4651 if (!num_tcs)
bd508178 4652 return false;
f92ef202 4653
e5b64635
JF		 4654 for (i = 0, k = 0; i < num_tcs; i++) {
4655 unsigned int tx_s, rx_s;
4656 u16 count = dev->tc_to_txq[i].count;
4657
4658 ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s);
4659 for (j = 0; j < count; j++, k++) {
4660 adapter->tx_ring[k]->reg_idx = tx_s + j;
4661 adapter->rx_ring[k]->reg_idx = rx_s + j;
4662 adapter->tx_ring[k]->dcb_tc = i;
4663 adapter->rx_ring[k]->dcb_tc = i;
021230d4 4664 }
021230d4 4665 }
e5b64635
JF		 4666
4667 return true;
bc97114d
PWJ		 4668}
4669#endif
4670
c4cf55e5
PWJ		 4671/**
4672 * ixgbe_cache_ring_fdir - Descriptor ring to register mapping for Flow Director
4673 * @adapter: board private structure to initialize
4674 *
4675 * Cache the descriptor ring offsets for Flow Director to the assigned rings.
4676 *
4677 **/
e8e9f696 4678static inline bool ixgbe_cache_ring_fdir(struct ixgbe_adapter *adapter)
c4cf55e5
PWJ		 4679{
4680 int i;
4681 bool ret = false;
4682
03ecf91a
AD		 4683 if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
4684 (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
c4cf55e5 4685 for (i = 0; i < adapter->num_rx_queues; i++)
4a0b9ca0 4686 adapter->rx_ring[i]->reg_idx = i;
c4cf55e5 4687 for (i = 0; i < adapter->num_tx_queues; i++)
4a0b9ca0 4688 adapter->tx_ring[i]->reg_idx = i;
c4cf55e5
PWJ		 4689 ret = true;
4690 }
4691
4692 return ret;
4693}
4694
0331a832
YZ		 4695#ifdef IXGBE_FCOE
4696/**
4697 * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE
4698 * @adapter: board private structure to initialize
4699 *
4700 * Cache the descriptor ring offsets for FCoE mode to the assigned rings.
4701 *
4702 */
4703static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter)
4704{
0331a832 4705 struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
bf29ee6c
AD		 4706 int i;
4707 u8 fcoe_rx_i = 0, fcoe_tx_i = 0;
4708
4709 if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
4710 return false;
0331a832 4711
bf29ee6c 4712 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
03ecf91a 4713 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
bf29ee6c
AD		 4714 ixgbe_cache_ring_fdir(adapter);
4715 else
4716 ixgbe_cache_ring_rss(adapter);
8faa2a78 4717
bf29ee6c
AD		 4718 fcoe_rx_i = f->mask;
4719 fcoe_tx_i = f->mask;
0331a832 4720 }
bf29ee6c
AD		 4721 for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) {
4722 adapter->rx_ring[f->mask + i]->reg_idx = fcoe_rx_i;
4723 adapter->tx_ring[f->mask + i]->reg_idx = fcoe_tx_i;
4724 }
4725 return true;
0331a832
YZ		 4726}
4727
4728#endif /* IXGBE_FCOE */
1cdd1ec8
GR		 4729/**
4730 * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov
4731 * @adapter: board private structure to initialize
4732 *
4733 * SR-IOV doesn't use any descriptor rings but changes the default if
4734 * no other mapping is used.
4735 *
4736 */
4737static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
4738{
4a0b9ca0
PW		 4739 adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2;
4740 adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2;
1cdd1ec8
GR		 4741 if (adapter->num_vfs)
4742 return true;
4743 else
4744 return false;
4745}
4746
bc97114d
PWJ		 4747/**
4748 * ixgbe_cache_ring_register - Descriptor ring to register mapping
4749 * @adapter: board private structure to initialize
4750 *
4751 * Once we know the feature-set enabled for the device, we'll cache
4752 * the register offset the descriptor ring is assigned to.
4753 *
4754 * Note, the order the various feature calls is important. It must start with
4755 * the "most" features enabled at the same time, then trickle down to the
4756 * least amount of features turned on at once.
4757 **/
4758static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter)
4759{
4760 /* start with default case */
4a0b9ca0
PW		 4761 adapter->rx_ring[0]->reg_idx = 0;
4762 adapter->tx_ring[0]->reg_idx = 0;
bc97114d 4763
1cdd1ec8
GR		 4764 if (ixgbe_cache_ring_sriov(adapter))
4765 return;
4766
e5b64635
JF		 4767#ifdef CONFIG_IXGBE_DCB
4768 if (ixgbe_cache_ring_dcb(adapter))
4769 return;
4770#endif
4771
0331a832
YZ		 4772#ifdef IXGBE_FCOE
4773 if (ixgbe_cache_ring_fcoe(adapter))
4774 return;
0331a832 4775#endif /* IXGBE_FCOE */
bc97114d 4776
c4cf55e5
PWJ		 4777 if (ixgbe_cache_ring_fdir(adapter))
4778 return;
4779
bc97114d
PWJ		 4780 if (ixgbe_cache_ring_rss(adapter))
4781 return;
021230d4
AV		 4782}
4783
9a799d71
AK		 4784/**
4785 * ixgbe_alloc_queues - Allocate memory for all rings
4786 * @adapter: board private structure to initialize
4787 *
4788 * We allocate one ring per queue at run-time since we don't know the
4df10466
JB		 4789 * number of queues at compile-time. The polling_netdev array is
4790 * intended for Multiqueue, but should work fine with a single queue.
9a799d71 4791 **/
2f90b865 4792static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter)
9a799d71 4793{
e2ddeba9 4794 int rx = 0, tx = 0, nid = adapter->node;
9a799d71 4795
e2ddeba9
ED		 4796 if (nid < 0 || !node_online(nid))
4797 nid = first_online_node;
4798
4799 for (; tx < adapter->num_tx_queues; tx++) {
4800 struct ixgbe_ring *ring;
4801
4802 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4a0b9ca0 4803 if (!ring)
e2ddeba9 4804 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4a0b9ca0 4805 if (!ring)
e2ddeba9 4806 goto err_allocation;
4a0b9ca0 4807 ring->count = adapter->tx_ring_count;
e2ddeba9
ED		 4808 ring->queue_index = tx;
4809 ring->numa_node = nid;
b6ec895e 4810 ring->dev = &adapter->pdev->dev;
fc77dc3c 4811 ring->netdev = adapter->netdev;
4a0b9ca0 4812
e2ddeba9 4813 adapter->tx_ring[tx] = ring;
021230d4 4814 }
b9804972 4815
e2ddeba9
ED		 4816 for (; rx < adapter->num_rx_queues; rx++) {
4817 struct ixgbe_ring *ring;
4a0b9ca0 4818
e2ddeba9 4819 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4a0b9ca0 4820 if (!ring)
e2ddeba9 4821 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4a0b9ca0 4822 if (!ring)
e2ddeba9
ED		 4823 goto err_allocation;
4824 ring->count = adapter->rx_ring_count;
4825 ring->queue_index = rx;
4826 ring->numa_node = nid;
b6ec895e 4827 ring->dev = &adapter->pdev->dev;
fc77dc3c 4828 ring->netdev = adapter->netdev;
4a0b9ca0 4829
e2ddeba9 4830 adapter->rx_ring[rx] = ring;
021230d4
AV		 4831 }
4832
4833 ixgbe_cache_ring_register(adapter);
4834
4835 return 0;
4836
e2ddeba9
ED		 4837err_allocation:
4838 while (tx)
4839 kfree(adapter->tx_ring[--tx]);
4840
4841 while (rx)
4842 kfree(adapter->rx_ring[--rx]);
021230d4
AV		 4843 return -ENOMEM;
4844}
4845
4846/**
4847 * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported
4848 * @adapter: board private structure to initialize
4849 *
4850 * Attempt to configure the interrupts using the best available
4851 * capabilities of the hardware and the kernel.
4852 **/
feea6a57 4853static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
021230d4 4854{
8be0e467 4855 struct ixgbe_hw *hw = &adapter->hw;
021230d4
AV		 4856 int err = 0;
4857 int vector, v_budget;
4858
4859 /*
4860 * It's easy to be greedy for MSI-X vectors, but it really
4861 * doesn't do us much good if we have a lot more vectors
4862 * than CPU's. So let's be conservative and only ask for
342bde1b 4863 * (roughly) the same number of vectors as there are CPU's.
021230d4
AV		 4864 */
4865 v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
e8e9f696 4866 (int)num_online_cpus()) + NON_Q_VECTORS;
021230d4
AV		 4867
4868 /*
4869 * At the same time, hardware can only support a maximum of
8be0e467
PW		 4870 * hw.mac->max_msix_vectors vectors. With features
4871 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
4872 * descriptor queues supported by our device. Thus, we cap it off in
4873 * those rare cases where the cpu count also exceeds our vector limit.
021230d4 4874 */
8be0e467 4875 v_budget = min(v_budget, (int)hw->mac.max_msix_vectors);
021230d4
AV		 4876
4877 /* A failure in MSI-X entry allocation isn't fatal, but it does
4878 * mean we disable MSI-X capabilities of the adapter. */
4879 adapter->msix_entries = kcalloc(v_budget,
e8e9f696 4880 sizeof(struct msix_entry), GFP_KERNEL);
7a921c93
AD		 4881 if (adapter->msix_entries) {
4882 for (vector = 0; vector < v_budget; vector++)
4883 adapter->msix_entries[vector].entry = vector;
021230d4 4884
7a921c93 4885 ixgbe_acquire_msix_vectors(adapter, v_budget);
021230d4 4886
7a921c93
AD		 4887 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
4888 goto out;
4889 }
26d27844 4890
7a921c93
AD		 4891 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
4892 adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED;
03ecf91a 4893 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
45b9f509 4894 e_err(probe,
03ecf91a 4895 "ATR is not supported while multiple "
45b9f509
AD		 4896 "queues are disabled. Disabling Flow Director\n");
4897 }
c4cf55e5 4898 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
c4cf55e5 4899 adapter->atr_sample_rate = 0;
1cdd1ec8
GR		 4900 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4901 ixgbe_disable_sriov(adapter);
4902
847f53ff
BH		 4903 err = ixgbe_set_num_queues(adapter);
4904 if (err)
4905 return err;
021230d4 4906
021230d4
AV		 4907 err = pci_enable_msi(adapter->pdev);
4908 if (!err) {
4909 adapter->flags |= IXGBE_FLAG_MSI_ENABLED;
4910 } else {
849c4542
ET		 4911 netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
4912 "Unable to allocate MSI interrupt, "
4913 "falling back to legacy. Error: %d\n", err);
021230d4
AV		 4914 /* reset err */
4915 err = 0;
4916 }
4917
4918out:
021230d4
AV		 4919 return err;
4920}
4921
7a921c93
AD		 4922/**
4923 * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors
4924 * @adapter: board private structure to initialize
4925 *
4926 * We allocate one q_vector per queue interrupt. If allocation fails we
4927 * return -ENOMEM.
4928 **/
4929static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter)
4930{
4ff7fb12 4931 int v_idx, num_q_vectors;
7a921c93 4932 struct ixgbe_q_vector *q_vector;
7a921c93 4933
4ff7fb12 4934 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
7a921c93 4935 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4ff7fb12 4936 else
7a921c93 4937 num_q_vectors = 1;
7a921c93 4938
4ff7fb12 4939 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
1a6c14a2 4940 q_vector = kzalloc_node(sizeof(struct ixgbe_q_vector),
e8e9f696 4941 GFP_KERNEL, adapter->node);
1a6c14a2
JB		 4942 if (!q_vector)
4943 q_vector = kzalloc(sizeof(struct ixgbe_q_vector),
e8e9f696 4944 GFP_KERNEL);
7a921c93
AD		 4945 if (!q_vector)
4946 goto err_out;
4ff7fb12 4947
7a921c93 4948 q_vector->adapter = adapter;
4ff7fb12
AD		 4949 q_vector->v_idx = v_idx;
4950
207867f5
AD		 4951 /* Allocate the affinity_hint cpumask, configure the mask */
4952 if (!alloc_cpumask_var(&q_vector->affinity_mask, GFP_KERNEL))
4953 goto err_out;
4954 cpumask_set_cpu(v_idx, q_vector->affinity_mask);
4ff7fb12
AD		 4955 netif_napi_add(adapter->netdev, &q_vector->napi,
4956 ixgbe_poll, 64);
4957 adapter->q_vector[v_idx] = q_vector;
7a921c93
AD		 4958 }
4959
4960 return 0;
4961
4962err_out:
4ff7fb12
AD		 4963 while (v_idx) {
4964 v_idx--;
4965 q_vector = adapter->q_vector[v_idx];
7a921c93 4966 netif_napi_del(&q_vector->napi);
207867f5 4967 free_cpumask_var(q_vector->affinity_mask);
7a921c93 4968 kfree(q_vector);
4ff7fb12 4969 adapter->q_vector[v_idx] = NULL;
7a921c93
AD		 4970 }
4971 return -ENOMEM;
4972}
4973
4974/**
4975 * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors
4976 * @adapter: board private structure to initialize
4977 *
4978 * This function frees the memory allocated to the q_vectors. In addition if
4979 * NAPI is enabled it will delete any references to the NAPI struct prior
4980 * to freeing the q_vector.
4981 **/
4982static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter)
4983{
207867f5 4984 int v_idx, num_q_vectors;
7a921c93 4985
91281fd3 4986 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
7a921c93 4987 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
91281fd3 4988 else
7a921c93 4989 num_q_vectors = 1;
7a921c93 4990
207867f5
AD		 4991 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
4992 struct ixgbe_q_vector *q_vector = adapter->q_vector[v_idx];
4993 adapter->q_vector[v_idx] = NULL;
91281fd3 4994 netif_napi_del(&q_vector->napi);
207867f5 4995 free_cpumask_var(q_vector->affinity_mask);
7a921c93
AD		 4996 kfree(q_vector);
4997 }
4998}
4999
7b25cdba 5000static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter)
021230d4
AV		 5001{
5002 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
5003 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
5004 pci_disable_msix(adapter->pdev);
5005 kfree(adapter->msix_entries);
5006 adapter->msix_entries = NULL;
5007 } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
5008 adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED;
5009 pci_disable_msi(adapter->pdev);
5010 }
021230d4
AV		 5011}
5012
5013/**
5014 * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme
5015 * @adapter: board private structure to initialize
5016 *
5017 * We determine which interrupt scheme to use based on...
5018 * - Kernel support (MSI, MSI-X)
5019 * - which can be user-defined (via MODULE_PARAM)
5020 * - Hardware queue count (num_*_queues)
5021 * - defined by miscellaneous hardware support/features (RSS, etc.)
5022 **/
2f90b865 5023int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter)
021230d4
AV		 5024{
5025 int err;
5026
5027 /* Number of supported queues */
847f53ff
BH		 5028 err = ixgbe_set_num_queues(adapter);
5029 if (err)
5030 return err;
021230d4 5031
021230d4
AV		 5032 err = ixgbe_set_interrupt_capability(adapter);
5033 if (err) {
849c4542 5034 e_dev_err("Unable to setup interrupt capabilities\n");
021230d4 5035 goto err_set_interrupt;
9a799d71
AK		 5036 }
5037
7a921c93
AD		 5038 err = ixgbe_alloc_q_vectors(adapter);
5039 if (err) {
849c4542 5040 e_dev_err("Unable to allocate memory for queue vectors\n");
7a921c93
AD		 5041 goto err_alloc_q_vectors;
5042 }
5043
5044 err = ixgbe_alloc_queues(adapter);
5045 if (err) {
849c4542 5046 e_dev_err("Unable to allocate memory for queues\n");
7a921c93
AD		 5047 goto err_alloc_queues;
5048 }
5049
849c4542 5050 e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n",
396e799c
ET		 5051 (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
5052 adapter->num_rx_queues, adapter->num_tx_queues);
021230d4
AV		 5053
5054 set_bit(__IXGBE_DOWN, &adapter->state);
5055
9a799d71 5056 return 0;
021230d4 5057
7a921c93
AD		 5058err_alloc_queues:
5059 ixgbe_free_q_vectors(adapter);
5060err_alloc_q_vectors:
5061 ixgbe_reset_interrupt_capability(adapter);
021230d4 5062err_set_interrupt:
7a921c93
AD		 5063 return err;
5064}
5065
5066/**
5067 * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings
5068 * @adapter: board private structure to clear interrupt scheme on
5069 *
5070 * We go through and clear interrupt specific resources and reset the structure
5071 * to pre-load conditions
5072 **/
5073void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter)
5074{
4a0b9ca0
PW		 5075 int i;
5076
5077 for (i = 0; i < adapter->num_tx_queues; i++) {
5078 kfree(adapter->tx_ring[i]);
5079 adapter->tx_ring[i] = NULL;
5080 }
5081 for (i = 0; i < adapter->num_rx_queues; i++) {
1a51502b
ED		 5082 struct ixgbe_ring *ring = adapter->rx_ring[i];
5083
5084 /* ixgbe_get_stats64() might access this ring, we must wait
5085 * a grace period before freeing it.
5086 */
bcec8b65 5087 kfree_rcu(ring, rcu);
4a0b9ca0
PW		 5088 adapter->rx_ring[i] = NULL;
5089 }
7a921c93 5090
b8eb3a10
DS		 5091 adapter->num_tx_queues = 0;
5092 adapter->num_rx_queues = 0;
5093
7a921c93
AD		 5094 ixgbe_free_q_vectors(adapter);
5095 ixgbe_reset_interrupt_capability(adapter);
9a799d71
AK		 5096}
5097
5098/**
5099 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
5100 * @adapter: board private structure to initialize
5101 *
5102 * ixgbe_sw_init initializes the Adapter private data structure.
5103 * Fields are initialized based on PCI device information and
5104 * OS network device settings (MTU size).
5105 **/
5106static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
5107{
5108 struct ixgbe_hw *hw = &adapter->hw;
5109 struct pci_dev *pdev = adapter->pdev;
021230d4 5110 unsigned int rss;
7a6b6f51 5111#ifdef CONFIG_IXGBE_DCB
2f90b865
AD		 5112 int j;
5113 struct tc_configuration *tc;
5114#endif
021230d4 5115
c44ade9e
JB		 5116 /* PCI config space info */
5117
5118 hw->vendor_id = pdev->vendor;
5119 hw->device_id = pdev->device;
5120 hw->revision_id = pdev->revision;
5121 hw->subsystem_vendor_id = pdev->subsystem_vendor;
5122 hw->subsystem_device_id = pdev->subsystem_device;
5123
021230d4
AV		 5124 /* Set capability flags */
5125 rss = min(IXGBE_MAX_RSS_INDICES, (int)num_online_cpus());
5126 adapter->ring_feature[RING_F_RSS].indices = rss;
5127 adapter->flags |= IXGBE_FLAG_RSS_ENABLED;
bd508178
AD		 5128 switch (hw->mac.type) {
5129 case ixgbe_mac_82598EB:
bf069c97
DS		 5130 if (hw->device_id == IXGBE_DEV_ID_82598AT)
5131 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
e8e26350 5132 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598;
bd508178 5133 break;
b93a2226 5134 case ixgbe_mac_X540:
4f51bf70
JK		 5135 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5136 case ixgbe_mac_82599EB:
e8e26350 5137 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599;
0c19d6af
PWJ		 5138 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
5139 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
119fc60a
MC		 5140 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
5141 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
45b9f509
AD		 5142 /* Flow Director hash filters enabled */
5143 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
5144 adapter->atr_sample_rate = 20;
c4cf55e5 5145 adapter->ring_feature[RING_F_FDIR].indices =
e8e9f696 5146 IXGBE_MAX_FDIR_INDICES;
c04f6ca8 5147 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
eacd73f7 5148#ifdef IXGBE_FCOE
0d551589
YZ		 5149 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
5150 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5151 adapter->ring_feature[RING_F_FCOE].indices = 0;
61a0f421 5152#ifdef CONFIG_IXGBE_DCB
6ee16520 5153 /* Default traffic class to use for FCoE */
56075a98 5154 adapter->fcoe.up = IXGBE_FCOE_DEFTC;
61a0f421 5155#endif
eacd73f7 5156#endif /* IXGBE_FCOE */
bd508178
AD		 5157 break;
5158 default:
5159 break;
f8212f97 5160 }
2f90b865 5161
1fc5f038
AD		 5162 /* n-tuple support exists, always init our spinlock */
5163 spin_lock_init(&adapter->fdir_perfect_lock);
5164
7a6b6f51 5165#ifdef CONFIG_IXGBE_DCB
4de2a022
JF		 5166 switch (hw->mac.type) {
5167 case ixgbe_mac_X540:
5168 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
5169 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
5170 break;
5171 default:
5172 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
5173 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
5174 break;
5175 }
5176
2f90b865
AD		 5177 /* Configure DCB traffic classes */
5178 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
5179 tc = &adapter->dcb_cfg.tc_config[j];
5180 tc->path[DCB_TX_CONFIG].bwg_id = 0;
5181 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
5182 tc->path[DCB_RX_CONFIG].bwg_id = 0;
5183 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
5184 tc->dcb_pfc = pfc_disabled;
5185 }
4de2a022
JF		 5186
5187 /* Initialize default user to priority mapping, UPx->TC0 */
5188 tc = &adapter->dcb_cfg.tc_config[0];
5189 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
5190 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
5191
2f90b865
AD		 5192 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
5193 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
264857b8 5194 adapter->dcb_cfg.pfc_mode_enable = false;
2f90b865 5195 adapter->dcb_set_bitmap = 0x00;
3032309b 5196 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
2f90b865 5197 ixgbe_copy_dcb_cfg(&adapter->dcb_cfg, &adapter->temp_dcb_cfg,
e5b64635 5198 MAX_TRAFFIC_CLASS);
2f90b865
AD		 5199
5200#endif
9a799d71
AK		 5201
5202 /* default flow control settings */
cd7664f6 5203 hw->fc.requested_mode = ixgbe_fc_full;
71fd570b 5204 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
264857b8
PWJ		 5205#ifdef CONFIG_DCB
5206 adapter->last_lfc_mode = hw->fc.current_mode;
5207#endif
9da712d2 5208 ixgbe_pbthresh_setup(adapter);
2b9ade93
JB		 5209 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
5210 hw->fc.send_xon = true;
71fd570b 5211 hw->fc.disable_fc_autoneg = false;
9a799d71 5212
30efa5a3 5213 /* enable itr by default in dynamic mode */
f7554a2b 5214 adapter->rx_itr_setting = 1;
f7554a2b 5215 adapter->tx_itr_setting = 1;
30efa5a3
JB		 5216
5217 /* set defaults for eitr in MegaBytes */
5218 adapter->eitr_low = 10;
5219 adapter->eitr_high = 20;
5220
5221 /* set default ring sizes */
5222 adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
5223 adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
5224
bd198058 5225 /* set default work limits */
59224555 5226 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
bd198058 5227
9a799d71 5228 /* initialize eeprom parameters */
c44ade9e 5229 if (ixgbe_init_eeprom_params_generic(hw)) {
849c4542 5230 e_dev_err("EEPROM initialization failed\n");
9a799d71
AK		 5231 return -EIO;
5232 }
5233
021230d4 5234 /* enable rx csum by default */
9a799d71
AK		 5235 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
5236
1a6c14a2
JB		 5237 /* get assigned NUMA node */
5238 adapter->node = dev_to_node(&pdev->dev);
5239
9a799d71
AK		 5240 set_bit(__IXGBE_DOWN, &adapter->state);
5241
5242 return 0;
5243}
5244
5245/**
5246 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
3a581073 5247 * @tx_ring: tx descriptor ring (for a specific queue) to setup
9a799d71
AK		 5248 *
5249 * Return 0 on success, negative on failure
5250 **/
b6ec895e 5251int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
9a799d71 5252{
b6ec895e 5253 struct device *dev = tx_ring->dev;
9a799d71
AK		 5254 int size;
5255
3a581073 5256 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
89bf67f1 5257 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
1a6c14a2 5258 if (!tx_ring->tx_buffer_info)
89bf67f1 5259 tx_ring->tx_buffer_info = vzalloc(size);
e01c31a5
JB		 5260 if (!tx_ring->tx_buffer_info)
5261 goto err;
9a799d71
AK		 5262
5263 /* round up to nearest 4K */
12207e49 5264 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
3a581073 5265 tx_ring->size = ALIGN(tx_ring->size, 4096);
9a799d71 5266
b6ec895e 5267 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
1b507730 5268 &tx_ring->dma, GFP_KERNEL);
e01c31a5
JB		 5269 if (!tx_ring->desc)
5270 goto err;
9a799d71 5271
3a581073
JB		 5272 tx_ring->next_to_use = 0;
5273 tx_ring->next_to_clean = 0;
9a799d71 5274 return 0;
e01c31a5
JB		 5275
5276err:
5277 vfree(tx_ring->tx_buffer_info);
5278 tx_ring->tx_buffer_info = NULL;
b6ec895e 5279 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
e01c31a5 5280 return -ENOMEM;
9a799d71
AK		 5281}
5282
69888674
AD		 5283/**
5284 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
5285 * @adapter: board private structure
5286 *
5287 * If this function returns with an error, then it's possible one or
5288 * more of the rings is populated (while the rest are not). It is the
5289 * callers duty to clean those orphaned rings.
5290 *
5291 * Return 0 on success, negative on failure
5292 **/
5293static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
5294{
5295 int i, err = 0;
5296
5297 for (i = 0; i < adapter->num_tx_queues; i++) {
b6ec895e 5298 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
69888674
AD		 5299 if (!err)
5300 continue;
396e799c 5301 e_err(probe, "Allocation for Tx Queue %u failed\n", i);
69888674
AD		 5302 break;
5303 }
5304
5305 return err;
5306}
5307
9a799d71
AK		 5308/**
5309 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
3a581073 5310 * @rx_ring: rx descriptor ring (for a specific queue) to setup
9a799d71
AK		 5311 *
5312 * Returns 0 on success, negative on failure
5313 **/
b6ec895e 5314int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
9a799d71 5315{
b6ec895e 5316 struct device *dev = rx_ring->dev;
021230d4 5317 int size;
9a799d71 5318
3a581073 5319 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
89bf67f1 5320 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
1a6c14a2 5321 if (!rx_ring->rx_buffer_info)
89bf67f1 5322 rx_ring->rx_buffer_info = vzalloc(size);
b6ec895e
AD		 5323 if (!rx_ring->rx_buffer_info)
5324 goto err;
9a799d71 5325
9a799d71 5326 /* Round up to nearest 4K */
3a581073
JB		 5327 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
5328 rx_ring->size = ALIGN(rx_ring->size, 4096);
9a799d71 5329
b6ec895e 5330 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1b507730 5331 &rx_ring->dma, GFP_KERNEL);
9a799d71 5332
b6ec895e
AD		 5333 if (!rx_ring->desc)
5334 goto err;
9a799d71 5335
3a581073
JB		 5336 rx_ring->next_to_clean = 0;
5337 rx_ring->next_to_use = 0;
9a799d71
AK		 5338
5339 return 0;
b6ec895e
AD		 5340err:
5341 vfree(rx_ring->rx_buffer_info);
5342 rx_ring->rx_buffer_info = NULL;
5343 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
177db6ff 5344 return -ENOMEM;
9a799d71
AK		 5345}
5346
69888674
AD		 5347/**
5348 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
5349 * @adapter: board private structure
5350 *
5351 * If this function returns with an error, then it's possible one or
5352 * more of the rings is populated (while the rest are not). It is the
5353 * callers duty to clean those orphaned rings.
5354 *
5355 * Return 0 on success, negative on failure
5356 **/
69888674
AD		 5357static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
5358{
5359 int i, err = 0;
5360
5361 for (i = 0; i < adapter->num_rx_queues; i++) {
b6ec895e 5362 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
69888674
AD		 5363 if (!err)
5364 continue;
396e799c 5365 e_err(probe, "Allocation for Rx Queue %u failed\n", i);
69888674
AD		 5366 break;
5367 }
5368
5369 return err;
5370}
5371
9a799d71
AK		 5372/**
5373 * ixgbe_free_tx_resources - Free Tx Resources per Queue
9a799d71
AK		 5374 * @tx_ring: Tx descriptor ring for a specific queue
5375 *
5376 * Free all transmit software resources
5377 **/
b6ec895e 5378void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
9a799d71 5379{
b6ec895e 5380 ixgbe_clean_tx_ring(tx_ring);
9a799d71
AK		 5381
5382 vfree(tx_ring->tx_buffer_info);
5383 tx_ring->tx_buffer_info = NULL;
5384
b6ec895e
AD		 5385 /* if not set, then don't free */
5386 if (!tx_ring->desc)
5387 return;
5388
5389 dma_free_coherent(tx_ring->dev, tx_ring->size,
5390 tx_ring->desc, tx_ring->dma);
9a799d71
AK		 5391
5392 tx_ring->desc = NULL;
5393}
5394
5395/**
5396 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
5397 * @adapter: board private structure
5398 *
5399 * Free all transmit software resources
5400 **/
5401static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
5402{
5403 int i;
5404
5405 for (i = 0; i < adapter->num_tx_queues; i++)
4a0b9ca0 5406 if (adapter->tx_ring[i]->desc)
b6ec895e 5407 ixgbe_free_tx_resources(adapter->tx_ring[i]);
9a799d71
AK		 5408}
5409
5410/**
b4617240 5411 * ixgbe_free_rx_resources - Free Rx Resources
9a799d71
AK		 5412 * @rx_ring: ring to clean the resources from
5413 *
5414 * Free all receive software resources
5415 **/
b6ec895e 5416void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
9a799d71 5417{
b6ec895e 5418 ixgbe_clean_rx_ring(rx_ring);
9a799d71
AK		 5419
5420 vfree(rx_ring->rx_buffer_info);
5421 rx_ring->rx_buffer_info = NULL;
5422
b6ec895e
AD		 5423 /* if not set, then don't free */
5424 if (!rx_ring->desc)
5425 return;
5426
5427 dma_free_coherent(rx_ring->dev, rx_ring->size,
5428 rx_ring->desc, rx_ring->dma);
9a799d71
AK		 5429
5430 rx_ring->desc = NULL;
5431}
5432
5433/**
5434 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
5435 * @adapter: board private structure
5436 *
5437 * Free all receive software resources
5438 **/
5439static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
5440{
5441 int i;
5442
5443 for (i = 0; i < adapter->num_rx_queues; i++)
4a0b9ca0 5444 if (adapter->rx_ring[i]->desc)
b6ec895e 5445 ixgbe_free_rx_resources(adapter->rx_ring[i]);
9a799d71
AK		 5446}
5447
9a799d71
AK		 5448/**
5449 * ixgbe_change_mtu - Change the Maximum Transfer Unit
5450 * @netdev: network interface device structure
5451 * @new_mtu: new value for maximum frame size
5452 *
5453 * Returns 0 on success, negative on failure
5454 **/
5455static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
5456{
5457 struct ixgbe_adapter *adapter = netdev_priv(netdev);
16b61beb 5458 struct ixgbe_hw *hw = &adapter->hw;
9a799d71
AK		 5459 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5460
42c783c5 5461 /* MTU < 68 is an error and causes problems on some kernels */
e9f98072
GR		 5462 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED &&
5463 hw->mac.type != ixgbe_mac_X540) {
5464 if ((new_mtu < 68) || (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
5465 return -EINVAL;
5466 } else {
5467 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
5468 return -EINVAL;
5469 }
9a799d71 5470
396e799c 5471 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
021230d4 5472 /* must set new MTU before calling down or up */
9a799d71
AK		 5473 netdev->mtu = new_mtu;
5474
d4f80882
AV		 5475 if (netif_running(netdev))
5476 ixgbe_reinit_locked(adapter);
9a799d71
AK		 5477
5478 return 0;
5479}
5480
5481/**
5482 * ixgbe_open - Called when a network interface is made active
5483 * @netdev: network interface device structure
5484 *
5485 * Returns 0 on success, negative value on failure
5486 *
5487 * The open entry point is called when a network interface is made
5488 * active by the system (IFF_UP). At this point all resources needed
5489 * for transmit and receive operations are allocated, the interrupt
5490 * handler is registered with the OS, the watchdog timer is started,
5491 * and the stack is notified that the interface is ready.
5492 **/
5493static int ixgbe_open(struct net_device *netdev)
5494{
5495 struct ixgbe_adapter *adapter = netdev_priv(netdev);
5496 int err;
4bebfaa5
AK		 5497
5498 /* disallow open during test */
5499 if (test_bit(__IXGBE_TESTING, &adapter->state))
5500 return -EBUSY;
9a799d71 5501
54386467
JB		 5502 netif_carrier_off(netdev);
5503
9a799d71
AK		 5504 /* allocate transmit descriptors */
5505 err = ixgbe_setup_all_tx_resources(adapter);
5506 if (err)
5507 goto err_setup_tx;
5508
9a799d71
AK		 5509 /* allocate receive descriptors */
5510 err = ixgbe_setup_all_rx_resources(adapter);
5511 if (err)
5512 goto err_setup_rx;
5513
5514 ixgbe_configure(adapter);
5515
021230d4 5516 err = ixgbe_request_irq(adapter);
9a799d71
AK		 5517 if (err)
5518 goto err_req_irq;
5519
c7ccde0f 5520 ixgbe_up_complete(adapter);
9a799d71
AK		 5521
5522 return 0;
5523
9a799d71 5524err_req_irq:
9a799d71 5525err_setup_rx:
a20a1199 5526 ixgbe_free_all_rx_resources(adapter);
9a799d71 5527err_setup_tx:
a20a1199 5528 ixgbe_free_all_tx_resources(adapter);
9a799d71
AK		 5529 ixgbe_reset(adapter);
5530
5531 return err;
5532}
5533
5534/**
5535 * ixgbe_close - Disables a network interface
5536 * @netdev: network interface device structure
5537 *
5538 * Returns 0, this is not allowed to fail
5539 *
5540 * The close entry point is called when an interface is de-activated
5541 * by the OS. The hardware is still under the drivers control, but
5542 * needs to be disabled. A global MAC reset is issued to stop the
5543 * hardware, and all transmit and receive resources are freed.
5544 **/
5545static int ixgbe_close(struct net_device *netdev)
5546{
5547 struct ixgbe_adapter *adapter = netdev_priv(netdev);
9a799d71
AK		 5548
5549 ixgbe_down(adapter);
5550 ixgbe_free_irq(adapter);
5551
e4911d57
AD		 5552 ixgbe_fdir_filter_exit(adapter);
5553
9a799d71
AK		 5554 ixgbe_free_all_tx_resources(adapter);
5555 ixgbe_free_all_rx_resources(adapter);
5556
5eba3699 5557 ixgbe_release_hw_control(adapter);
9a799d71
AK		 5558
5559 return 0;
5560}
5561
b3c8b4ba
AD		 5562#ifdef CONFIG_PM
5563static int ixgbe_resume(struct pci_dev *pdev)
5564{
c60fbb00
AD		 5565 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
5566 struct net_device *netdev = adapter->netdev;
b3c8b4ba
AD		 5567 u32 err;
5568
5569 pci_set_power_state(pdev, PCI_D0);
5570 pci_restore_state(pdev);
656ab817
DS		 5571 /*
5572 * pci_restore_state clears dev->state_saved so call
5573 * pci_save_state to restore it.
5574 */
5575 pci_save_state(pdev);
9ce77666 5576
5577 err = pci_enable_device_mem(pdev);
b3c8b4ba 5578 if (err) {
849c4542 5579 e_dev_err("Cannot enable PCI device from suspend\n");
b3c8b4ba
AD		 5580 return err;
5581 }
5582 pci_set_master(pdev);
5583
dd4d8ca6 5584 pci_wake_from_d3(pdev, false);
b3c8b4ba
AD		 5585
5586 err = ixgbe_init_interrupt_scheme(adapter);
5587 if (err) {
849c4542 5588 e_dev_err("Cannot initialize interrupts for device\n");
b3c8b4ba
AD		 5589 return err;
5590 }
5591
b3c8b4ba
AD		 5592 ixgbe_reset(adapter);
5593
495dce12
WJP		 5594 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
5595
b3c8b4ba 5596 if (netif_running(netdev)) {
c60fbb00 5597 err = ixgbe_open(netdev);
b3c8b4ba
AD		 5598 if (err)
5599 return err;
5600 }
5601
5602 netif_device_attach(netdev);
5603
5604 return 0;
5605}
b3c8b4ba 5606#endif /* CONFIG_PM */
9d8d05ae
RW		 5607
5608static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
b3c8b4ba 5609{
c60fbb00
AD		 5610 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
5611 struct net_device *netdev = adapter->netdev;
e8e26350
PW		 5612 struct ixgbe_hw *hw = &adapter->hw;
5613 u32 ctrl, fctrl;
5614 u32 wufc = adapter->wol;
b3c8b4ba
AD		 5615#ifdef CONFIG_PM
5616 int retval = 0;
5617#endif
5618
5619 netif_device_detach(netdev);
5620
5621 if (netif_running(netdev)) {
5622 ixgbe_down(adapter);
5623 ixgbe_free_irq(adapter);
5624 ixgbe_free_all_tx_resources(adapter);
5625 ixgbe_free_all_rx_resources(adapter);
5626 }
b3c8b4ba 5627
5f5ae6fc 5628 ixgbe_clear_interrupt_scheme(adapter);
d033d526
JF		 5629#ifdef CONFIG_DCB
5630 kfree(adapter->ixgbe_ieee_pfc);
5631 kfree(adapter->ixgbe_ieee_ets);
5632#endif
5f5ae6fc 5633
b3c8b4ba
AD		 5634#ifdef CONFIG_PM
5635 retval = pci_save_state(pdev);
5636 if (retval)
5637 return retval;
4df10466 5638
b3c8b4ba 5639#endif
e8e26350
PW		 5640 if (wufc) {
5641 ixgbe_set_rx_mode(netdev);
b3c8b4ba 5642
e8e26350
PW		 5643 /* turn on all-multi mode if wake on multicast is enabled */
5644 if (wufc & IXGBE_WUFC_MC) {
5645 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5646 fctrl |= IXGBE_FCTRL_MPE;
5647 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5648 }
5649
5650 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
5651 ctrl |= IXGBE_CTRL_GIO_DIS;
5652 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
5653
5654 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
5655 } else {
5656 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
5657 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
5658 }
5659
bd508178
AD		 5660 switch (hw->mac.type) {
5661 case ixgbe_mac_82598EB:
dd4d8ca6 5662 pci_wake_from_d3(pdev, false);
bd508178
AD		 5663 break;
5664 case ixgbe_mac_82599EB:
b93a2226 5665 case ixgbe_mac_X540:
bd508178
AD		 5666 pci_wake_from_d3(pdev, !!wufc);
5667 break;
5668 default:
5669 break;
5670 }
b3c8b4ba 5671
9d8d05ae
RW		 5672 *enable_wake = !!wufc;
5673
b3c8b4ba
AD		 5674 ixgbe_release_hw_control(adapter);
5675
5676 pci_disable_device(pdev);
5677
9d8d05ae
RW		 5678 return 0;
5679}
5680
5681#ifdef CONFIG_PM
5682static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
5683{
5684 int retval;
5685 bool wake;
5686
5687 retval = __ixgbe_shutdown(pdev, &wake);
5688 if (retval)
5689 return retval;
5690
5691 if (wake) {
5692 pci_prepare_to_sleep(pdev);
5693 } else {
5694 pci_wake_from_d3(pdev, false);
5695 pci_set_power_state(pdev, PCI_D3hot);
5696 }
b3c8b4ba
AD		 5697
5698 return 0;
5699}
9d8d05ae 5700#endif /* CONFIG_PM */
b3c8b4ba
AD		 5701
5702static void ixgbe_shutdown(struct pci_dev *pdev)
5703{
9d8d05ae
RW		 5704 bool wake;
5705
5706 __ixgbe_shutdown(pdev, &wake);
5707
5708 if (system_state == SYSTEM_POWER_OFF) {
5709 pci_wake_from_d3(pdev, wake);
5710 pci_set_power_state(pdev, PCI_D3hot);
5711 }
b3c8b4ba
AD		 5712}
5713
9a799d71
AK		 5714/**
5715 * ixgbe_update_stats - Update the board statistics counters.
5716 * @adapter: board private structure
5717 **/
5718void ixgbe_update_stats(struct ixgbe_adapter *adapter)
5719{
2d86f139 5720 struct net_device *netdev = adapter->netdev;
9a799d71 5721 struct ixgbe_hw *hw = &adapter->hw;
5b7da515 5722 struct ixgbe_hw_stats *hwstats = &adapter->stats;
6f11eef7
AV		 5723 u64 total_mpc = 0;
5724 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
5b7da515
AD		 5725 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
5726 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
5727 u64 bytes = 0, packets = 0;
7b859ebc
AH		 5728#ifdef IXGBE_FCOE
5729 struct ixgbe_fcoe *fcoe = &adapter->fcoe;
5730 unsigned int cpu;
5731 u64 fcoe_noddp_counts_sum = 0, fcoe_noddp_ext_buff_counts_sum = 0;
5732#endif /* IXGBE_FCOE */
9a799d71 5733
d08935c2
DS		 5734 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5735 test_bit(__IXGBE_RESETTING, &adapter->state))
5736 return;
5737
94b982b2 5738 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
f8212f97 5739 u64 rsc_count = 0;
94b982b2 5740 u64 rsc_flush = 0;
d51019a4
PW		 5741 for (i = 0; i < 16; i++)
5742 adapter->hw_rx_no_dma_resources +=
7ca647bd 5743 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
94b982b2 5744 for (i = 0; i < adapter->num_rx_queues; i++) {
5b7da515
AD		 5745 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
5746 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
94b982b2
MC		 5747 }
5748 adapter->rsc_total_count = rsc_count;
5749 adapter->rsc_total_flush = rsc_flush;
d51019a4
PW		 5750 }
5751
5b7da515
AD		 5752 for (i = 0; i < adapter->num_rx_queues; i++) {
5753 struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
5754 non_eop_descs += rx_ring->rx_stats.non_eop_descs;
5755 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
5756 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
5757 bytes += rx_ring->stats.bytes;
5758 packets += rx_ring->stats.packets;
5759 }
5760 adapter->non_eop_descs = non_eop_descs;
5761 adapter->alloc_rx_page_failed = alloc_rx_page_failed;
5762 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
5763 netdev->stats.rx_bytes = bytes;
5764 netdev->stats.rx_packets = packets;
5765
5766 bytes = 0;
5767 packets = 0;
7ca3bc58 5768 /* gather some stats to the adapter struct that are per queue */
5b7da515
AD		 5769 for (i = 0; i < adapter->num_tx_queues; i++) {
5770 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
5771 restart_queue += tx_ring->tx_stats.restart_queue;
5772 tx_busy += tx_ring->tx_stats.tx_busy;
5773 bytes += tx_ring->stats.bytes;
5774 packets += tx_ring->stats.packets;
5775 }
eb985f09 5776 adapter->restart_queue = restart_queue;
5b7da515
AD		 5777 adapter->tx_busy = tx_busy;
5778 netdev->stats.tx_bytes = bytes;
5779 netdev->stats.tx_packets = packets;
7ca3bc58 5780
7ca647bd 5781 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1a70db4b
ET		 5782
5783 /* 8 register reads */
6f11eef7
AV		 5784 for (i = 0; i < 8; i++) {
5785 /* for packet buffers not used, the register should read 0 */
5786 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5787 missed_rx += mpc;
7ca647bd
JP		 5788 hwstats->mpc[i] += mpc;
5789 total_mpc += hwstats->mpc[i];
1a70db4b
ET		 5790 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5791 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
bd508178
AD		 5792 switch (hw->mac.type) {
5793 case ixgbe_mac_82598EB:
1a70db4b
ET		 5794 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5795 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5796 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
7ca647bd
JP		 5797 hwstats->pxonrxc[i] +=
5798 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
bd508178
AD		 5799 break;
5800 case ixgbe_mac_82599EB:
b93a2226 5801 case ixgbe_mac_X540:
bd508178
AD		 5802 hwstats->pxonrxc[i] +=
5803 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
bd508178
AD		 5804 break;
5805 default:
5806 break;
e8e26350 5807 }
6f11eef7 5808 }
1a70db4b
ET		 5809
5810 /*16 register reads */
5811 for (i = 0; i < 16; i++) {
5812 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5813 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5814 if ((hw->mac.type == ixgbe_mac_82599EB) ||
5815 (hw->mac.type == ixgbe_mac_X540)) {
5816 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
5817 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
5818 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
5819 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
5820 }
5821 }
5822
7ca647bd 5823 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
6f11eef7 5824 /* work around hardware counting issue */
7ca647bd 5825 hwstats->gprc -= missed_rx;
6f11eef7 5826
c84d324c
JF		 5827 ixgbe_update_xoff_received(adapter);
5828
6f11eef7 5829 /* 82598 hardware only has a 32 bit counter in the high register */
bd508178
AD		 5830 switch (hw->mac.type) {
5831 case ixgbe_mac_82598EB:
5832 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
bd508178
AD		 5833 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5834 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5835 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5836 break;
b93a2226 5837 case ixgbe_mac_X540:
58f6bcf9
ET		 5838 /* OS2BMC stats are X540 only*/
5839 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
5840 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
5841 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
5842 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
5843 case ixgbe_mac_82599EB:
7ca647bd 5844 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
bd508178 5845 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
7ca647bd 5846 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
bd508178 5847 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
7ca647bd 5848 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
bd508178 5849 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
7ca647bd 5850 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
7ca647bd
JP		 5851 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
5852 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
6d45522c 5853#ifdef IXGBE_FCOE
7ca647bd
JP		 5854 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5855 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5856 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5857 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5858 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5859 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
7b859ebc
AH		 5860 /* Add up per cpu counters for total ddp aloc fail */
5861 if (fcoe->pcpu_noddp && fcoe->pcpu_noddp_ext_buff) {
5862 for_each_possible_cpu(cpu) {
5863 fcoe_noddp_counts_sum +=
5864 *per_cpu_ptr(fcoe->pcpu_noddp, cpu);
5865 fcoe_noddp_ext_buff_counts_sum +=
5866 *per_cpu_ptr(fcoe->
5867 pcpu_noddp_ext_buff, cpu);
5868 }
5869 }
5870 hwstats->fcoe_noddp = fcoe_noddp_counts_sum;
5871 hwstats->fcoe_noddp_ext_buff = fcoe_noddp_ext_buff_counts_sum;
6d45522c 5872#endif /* IXGBE_FCOE */
bd508178
AD		 5873 break;
5874 default:
5875 break;
e8e26350 5876 }
9a799d71 5877 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
7ca647bd
JP		 5878 hwstats->bprc += bprc;
5879 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
e8e26350 5880 if (hw->mac.type == ixgbe_mac_82598EB)
7ca647bd
JP		 5881 hwstats->mprc -= bprc;
5882 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5883 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5884 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5885 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5886 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5887 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5888 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5889 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
6f11eef7 5890 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
7ca647bd 5891 hwstats->lxontxc += lxon;
6f11eef7 5892 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
7ca647bd 5893 hwstats->lxofftxc += lxoff;
7ca647bd
JP		 5894 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5895 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
6f11eef7
AV		 5896 /*
5897 * 82598 errata - tx of flow control packets is included in tx counters
5898 */
5899 xon_off_tot = lxon + lxoff;
7ca647bd
JP		 5900 hwstats->gptc -= xon_off_tot;
5901 hwstats->mptc -= xon_off_tot;
5902 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
5903 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5904 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5905 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5906 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5907 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5908 hwstats->ptc64 -= xon_off_tot;
5909 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5910 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5911 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5912 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5913 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5914 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
9a799d71
AK		 5915
5916 /* Fill out the OS statistics structure */
7ca647bd 5917 netdev->stats.multicast = hwstats->mprc;
9a799d71
AK		 5918
5919 /* Rx Errors */
7ca647bd 5920 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
2d86f139 5921 netdev->stats.rx_dropped = 0;
7ca647bd
JP		 5922 netdev->stats.rx_length_errors = hwstats->rlec;
5923 netdev->stats.rx_crc_errors = hwstats->crcerrs;
2d86f139 5924 netdev->stats.rx_missed_errors = total_mpc;
9a799d71
AK		 5925}
5926
5927/**
d034acf1
AD		 5928 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
5929 * @adapter - pointer to the device adapter structure
9a799d71 5930 **/
d034acf1 5931static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
9a799d71 5932{
cf8280ee 5933 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a 5934 int i;
cf8280ee 5935
d034acf1
AD		 5936 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
5937 return;
5938
5939 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
22d5a71b 5940
d034acf1 5941 /* if interface is down do nothing */
fe49f04a 5942 if (test_bit(__IXGBE_DOWN, &adapter->state))
d034acf1
AD		 5943 return;
5944
5945 /* do nothing if we are not using signature filters */
5946 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
5947 return;
5948
5949 adapter->fdir_overflow++;
5950
93c52dd0
AD		 5951 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
5952 for (i = 0; i < adapter->num_tx_queues; i++)
5953 set_bit(__IXGBE_TX_FDIR_INIT_DONE,
f0f9778d 5954 &(adapter->tx_ring[i]->state));
d034acf1
AD		 5955 /* re-enable flow director interrupts */
5956 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
93c52dd0
AD		 5957 } else {
5958 e_err(probe, "failed to finish FDIR re-initialization, "
5959 "ignored adding FDIR ATR filters\n");
5960 }
93c52dd0
AD		 5961}
5962
5963/**
5964 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
5965 * @adapter - pointer to the device adapter structure
5966 *
5967 * This function serves two purposes. First it strobes the interrupt lines
52f33af8 5968 * in order to make certain interrupts are occurring. Secondly it sets the
93c52dd0 5969 * bits needed to check for TX hangs. As a result we should immediately
52f33af8 5970 * determine if a hang has occurred.
93c52dd0
AD		 5971 */
5972static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
9a799d71 5973{
cf8280ee 5974 struct ixgbe_hw *hw = &adapter->hw;
fe49f04a
AD		 5975 u64 eics = 0;
5976 int i;
cf8280ee 5977
93c52dd0
AD		 5978 /* If we're down or resetting, just bail */
5979 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5980 test_bit(__IXGBE_RESETTING, &adapter->state))
5981 return;
22d5a71b 5982
93c52dd0
AD		 5983 /* Force detection of hung controller */
5984 if (netif_carrier_ok(adapter->netdev)) {
5985 for (i = 0; i < adapter->num_tx_queues; i++)
5986 set_check_for_tx_hang(adapter->tx_ring[i]);
5987 }
22d5a71b 5988
fe49f04a
AD		 5989 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
5990 /*
5991 * for legacy and MSI interrupts don't set any bits
5992 * that are enabled for EIAM, because this operation
5993 * would set *both* EIMS and EICS for any bit in EIAM
5994 */
5995 IXGBE_WRITE_REG(hw, IXGBE_EICS,
5996 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
93c52dd0
AD		 5997 } else {
5998 /* get one bit for every active tx/rx interrupt vector */
5999 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
6000 struct ixgbe_q_vector *qv = adapter->q_vector[i];
efe3d3c8 6001 if (qv->rx.ring || qv->tx.ring)
93c52dd0
AD		 6002 eics |= ((u64)1 << i);
6003 }
cf8280ee 6004 }
9a799d71 6005
93c52dd0 6006 /* Cause software interrupt to ensure rings are cleaned */
fe49f04a
AD		 6007 ixgbe_irq_rearm_queues(adapter, eics);
6008
cf8280ee
JB		 6009}
6010
e8e26350 6011/**
93c52dd0
AD		 6012 * ixgbe_watchdog_update_link - update the link status
6013 * @adapter - pointer to the device adapter structure
6014 * @link_speed - pointer to a u32 to store the link_speed
e8e26350 6015 **/
93c52dd0 6016static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
e8e26350 6017{
e8e26350 6018 struct ixgbe_hw *hw = &adapter->hw;
93c52dd0
AD		 6019 u32 link_speed = adapter->link_speed;
6020 bool link_up = adapter->link_up;
c4cf55e5 6021 int i;
e8e26350 6022
93c52dd0
AD		 6023 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
6024 return;
6025
6026 if (hw->mac.ops.check_link) {
6027 hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
c4cf55e5 6028 } else {
93c52dd0
AD		 6029 /* always assume link is up, if no check link function */
6030 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
6031 link_up = true;
c4cf55e5 6032 }
93c52dd0
AD		 6033 if (link_up) {
6034 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
6035 for (i = 0; i < MAX_TRAFFIC_CLASS; i++)
6036 hw->mac.ops.fc_enable(hw, i);
6037 } else {
6038 hw->mac.ops.fc_enable(hw, 0);
6039 }
6040 }
6041
6042 if (link_up ||
6043 time_after(jiffies, (adapter->link_check_timeout +
6044 IXGBE_TRY_LINK_TIMEOUT))) {
6045 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
6046 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
6047 IXGBE_WRITE_FLUSH(hw);
6048 }
6049
6050 adapter->link_up = link_up;
6051 adapter->link_speed = link_speed;
e8e26350
PW		 6052}
6053
6054/**
93c52dd0
AD		 6055 * ixgbe_watchdog_link_is_up - update netif_carrier status and
6056 * print link up message
6057 * @adapter - pointer to the device adapter structure
e8e26350 6058 **/
93c52dd0 6059static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
e8e26350 6060{
93c52dd0 6061 struct net_device *netdev = adapter->netdev;
e8e26350 6062 struct ixgbe_hw *hw = &adapter->hw;
93c52dd0
AD		 6063 u32 link_speed = adapter->link_speed;
6064 bool flow_rx, flow_tx;
e8e26350 6065
93c52dd0
AD		 6066 /* only continue if link was previously down */
6067 if (netif_carrier_ok(netdev))
a985b6c3 6068 return;
63d6e1d8 6069
93c52dd0 6070 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
63d6e1d8 6071
93c52dd0
AD		 6072 switch (hw->mac.type) {
6073 case ixgbe_mac_82598EB: {
6074 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
6075 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
6076 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
6077 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
6078 }
6079 break;
6080 case ixgbe_mac_X540:
6081 case ixgbe_mac_82599EB: {
6082 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
6083 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
6084 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
6085 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
6086 }
6087 break;
6088 default:
6089 flow_tx = false;
6090 flow_rx = false;
6091 break;
e8e26350 6092 }
93c52dd0
AD		 6093 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
6094 (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
6095 "10 Gbps" :
6096 (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
6097 "1 Gbps" :
6098 (link_speed == IXGBE_LINK_SPEED_100_FULL ?
6099 "100 Mbps" :
6100 "unknown speed"))),
6101 ((flow_rx && flow_tx) ? "RX/TX" :
6102 (flow_rx ? "RX" :
6103 (flow_tx ? "TX" : "None"))));
e8e26350 6104
93c52dd0 6105 netif_carrier_on(netdev);
93c52dd0 6106 ixgbe_check_vf_rate_limit(adapter);
e8e26350
PW		 6107}
6108
c4cf55e5 6109/**
93c52dd0
AD		 6110 * ixgbe_watchdog_link_is_down - update netif_carrier status and
6111 * print link down message
6112 * @adapter - pointer to the adapter structure
c4cf55e5 6113 **/
93c52dd0 6114static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter* adapter)
c4cf55e5 6115{
cf8280ee 6116 struct net_device *netdev = adapter->netdev;
c4cf55e5 6117 struct ixgbe_hw *hw = &adapter->hw;
10eec955 6118
93c52dd0
AD		 6119 adapter->link_up = false;
6120 adapter->link_speed = 0;
cf8280ee 6121
93c52dd0
AD		 6122 /* only continue if link was up previously */
6123 if (!netif_carrier_ok(netdev))
6124 return;
264857b8 6125
93c52dd0
AD		 6126 /* poll for SFP+ cable when link is down */
6127 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
6128 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
9a799d71 6129
93c52dd0
AD		 6130 e_info(drv, "NIC Link is Down\n");
6131 netif_carrier_off(netdev);
6132}
e8e26350 6133
93c52dd0
AD		 6134/**
6135 * ixgbe_watchdog_flush_tx - flush queues on link down
6136 * @adapter - pointer to the device adapter structure
6137 **/
6138static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
6139{
c4cf55e5 6140 int i;
93c52dd0 6141 int some_tx_pending = 0;
c4cf55e5 6142
93c52dd0 6143 if (!netif_carrier_ok(adapter->netdev)) {
bc59fcda 6144 for (i = 0; i < adapter->num_tx_queues; i++) {
93c52dd0 6145 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
bc59fcda
NS		 6146 if (tx_ring->next_to_use != tx_ring->next_to_clean) {
6147 some_tx_pending = 1;
6148 break;
6149 }
6150 }
6151
6152 if (some_tx_pending) {
6153 /* We've lost link, so the controller stops DMA,
6154 * but we've got queued Tx work that's never going
6155 * to get done, so reset controller to flush Tx.
6156 * (Do the reset outside of interrupt context).
6157 */
c83c6cbd 6158 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
bc59fcda 6159 }
c4cf55e5 6160 }
c4cf55e5
PWJ		 6161}
6162
a985b6c3
GR		 6163static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
6164{
6165 u32 ssvpc;
6166
6167 /* Do not perform spoof check for 82598 */
6168 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
6169 return;
6170
6171 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
6172
6173 /*
6174 * ssvpc register is cleared on read, if zero then no
6175 * spoofed packets in the last interval.
6176 */
6177 if (!ssvpc)
6178 return;
6179
6180 e_warn(drv, "%d Spoofed packets detected\n", ssvpc);
6181}
6182
93c52dd0
AD		 6183/**
6184 * ixgbe_watchdog_subtask - check and bring link up
6185 * @adapter - pointer to the device adapter structure
6186 **/
6187static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
6188{
6189 /* if interface is down do nothing */
7edebf9a
ET		 6190 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6191 test_bit(__IXGBE_RESETTING, &adapter->state))
93c52dd0
AD		 6192 return;
6193
6194 ixgbe_watchdog_update_link(adapter);
6195
6196 if (adapter->link_up)
6197 ixgbe_watchdog_link_is_up(adapter);
6198 else
6199 ixgbe_watchdog_link_is_down(adapter);
bc59fcda 6200
a985b6c3 6201 ixgbe_spoof_check(adapter);
9a799d71 6202 ixgbe_update_stats(adapter);
93c52dd0
AD		 6203
6204 ixgbe_watchdog_flush_tx(adapter);
9a799d71 6205}
10eec955 6206
cf8280ee 6207/**
7086400d
AD		 6208 * ixgbe_sfp_detection_subtask - poll for SFP+ cable
6209 * @adapter - the ixgbe adapter structure
cf8280ee 6210 **/
7086400d 6211static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
cf8280ee 6212{
cf8280ee 6213 struct ixgbe_hw *hw = &adapter->hw;
7086400d 6214 s32 err;
cf8280ee 6215
7086400d
AD		 6216 /* not searching for SFP so there is nothing to do here */
6217 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
6218 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
6219 return;
10eec955 6220
7086400d
AD		 6221 /* someone else is in init, wait until next service event */
6222 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
6223 return;
cf8280ee 6224
7086400d
AD		 6225 err = hw->phy.ops.identify_sfp(hw);
6226 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
6227 goto sfp_out;
264857b8 6228
7086400d
AD		 6229 if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
6230 /* If no cable is present, then we need to reset
6231 * the next time we find a good cable. */
6232 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
cf8280ee 6233 }
9a799d71 6234
7086400d
AD		 6235 /* exit on error */
6236 if (err)
6237 goto sfp_out;
e8e26350 6238
7086400d
AD		 6239 /* exit if reset not needed */
6240 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
6241 goto sfp_out;
9a799d71 6242
7086400d 6243 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
bc59fcda 6244
7086400d
AD		 6245 /*
6246 * A module may be identified correctly, but the EEPROM may not have
6247 * support for that module. setup_sfp() will fail in that case, so
6248 * we should not allow that module to load.
6249 */
6250 if (hw->mac.type == ixgbe_mac_82598EB)
6251 err = hw->phy.ops.reset(hw);
6252 else
6253 err = hw->mac.ops.setup_sfp(hw);
6254
6255 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
6256 goto sfp_out;
6257
6258 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
6259 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
6260
6261sfp_out:
6262 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
6263
6264 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
6265 (adapter->netdev->reg_state == NETREG_REGISTERED)) {
6266 e_dev_err("failed to initialize because an unsupported "
6267 "SFP+ module type was detected.\n");
6268 e_dev_err("Reload the driver after installing a "
6269 "supported module.\n");
6270 unregister_netdev(adapter->netdev);
bc59fcda 6271 }
7086400d 6272}
bc59fcda 6273
7086400d
AD		 6274/**
6275 * ixgbe_sfp_link_config_subtask - set up link SFP after module install
6276 * @adapter - the ixgbe adapter structure
6277 **/
6278static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
6279{
6280 struct ixgbe_hw *hw = &adapter->hw;
6281 u32 autoneg;
6282 bool negotiation;
6283
6284 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
6285 return;
6286
6287 /* someone else is in init, wait until next service event */
6288 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
6289 return;
6290
6291 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
6292
6293 autoneg = hw->phy.autoneg_advertised;
6294 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
6295 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
7086400d
AD		 6296 if (hw->mac.ops.setup_link)
6297 hw->mac.ops.setup_link(hw, autoneg, negotiation, true);
6298
6299 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
6300 adapter->link_check_timeout = jiffies;
6301 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
6302}
6303
83c61fa9
GR		 6304#ifdef CONFIG_PCI_IOV
6305static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
6306{
6307 int vf;
6308 struct ixgbe_hw *hw = &adapter->hw;
6309 struct net_device *netdev = adapter->netdev;
6310 u32 gpc;
6311 u32 ciaa, ciad;
6312
6313 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
6314 if (gpc) /* If incrementing then no need for the check below */
6315 return;
6316 /*
6317 * Check to see if a bad DMA write target from an errant or
6318 * malicious VF has caused a PCIe error. If so then we can
6319 * issue a VFLR to the offending VF(s) and then resume without
6320 * requesting a full slot reset.
6321 */
6322
6323 for (vf = 0; vf < adapter->num_vfs; vf++) {
6324 ciaa = (vf << 16) | 0x80000000;
6325 /* 32 bit read so align, we really want status at offset 6 */
6326 ciaa |= PCI_COMMAND;
6327 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
6328 ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599);
6329 ciaa &= 0x7FFFFFFF;
6330 /* disable debug mode asap after reading data */
6331 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
6332 /* Get the upper 16 bits which will be the PCI status reg */
6333 ciad >>= 16;
6334 if (ciad & PCI_STATUS_REC_MASTER_ABORT) {
6335 netdev_err(netdev, "VF %d Hung DMA\n", vf);
6336 /* Issue VFLR */
6337 ciaa = (vf << 16) | 0x80000000;
6338 ciaa |= 0xA8;
6339 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
6340 ciad = 0x00008000; /* VFLR */
6341 IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad);
6342 ciaa &= 0x7FFFFFFF;
6343 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa);
6344 }
6345 }
6346}
6347
6348#endif
7086400d
AD		 6349/**
6350 * ixgbe_service_timer - Timer Call-back
6351 * @data: pointer to adapter cast into an unsigned long
6352 **/
6353static void ixgbe_service_timer(unsigned long data)
6354{
6355 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
6356 unsigned long next_event_offset;
83c61fa9 6357 bool ready = true;
7086400d 6358
83c61fa9
GR		 6359#ifdef CONFIG_PCI_IOV
6360 ready = false;
6361
6362 /*
6363 * don't bother with SR-IOV VF DMA hang check if there are
6364 * no VFs or the link is down
6365 */
6366 if (!adapter->num_vfs ||
6367 (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) {
6368 ready = true;
6369 goto normal_timer_service;
6370 }
6371
6372 /* If we have VFs allocated then we must check for DMA hangs */
6373 ixgbe_check_for_bad_vf(adapter);
6374 next_event_offset = HZ / 50;
6375 adapter->timer_event_accumulator++;
6376
6377 if (adapter->timer_event_accumulator >= 100) {
6378 ready = true;
6379 adapter->timer_event_accumulator = 0;
6380 }
6381
6382 goto schedule_event;
6383
6384normal_timer_service:
6385#endif
7086400d
AD		 6386 /* poll faster when waiting for link */
6387 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
6388 next_event_offset = HZ / 10;
6389 else
6390 next_event_offset = HZ * 2;
6391
83c61fa9
GR		 6392#ifdef CONFIG_PCI_IOV
6393schedule_event:
6394#endif
7086400d
AD		 6395 /* Reset the timer */
6396 mod_timer(&adapter->service_timer, next_event_offset + jiffies);
6397
83c61fa9
GR		 6398 if (ready)
6399 ixgbe_service_event_schedule(adapter);
7086400d
AD		 6400}
6401
c83c6cbd
AD		 6402static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
6403{
6404 if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
6405 return;
6406
6407 adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
6408
6409 /* If we're already down or resetting, just bail */
6410 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6411 test_bit(__IXGBE_RESETTING, &adapter->state))
6412 return;
6413
6414 ixgbe_dump(adapter);
6415 netdev_err(adapter->netdev, "Reset adapter\n");
6416 adapter->tx_timeout_count++;
6417
6418 ixgbe_reinit_locked(adapter);
6419}
6420
7086400d
AD		 6421/**
6422 * ixgbe_service_task - manages and runs subtasks
6423 * @work: pointer to work_struct containing our data
6424 **/
6425static void ixgbe_service_task(struct work_struct *work)
6426{
6427 struct ixgbe_adapter *adapter = container_of(work,
6428 struct ixgbe_adapter,
6429 service_task);
6430
c83c6cbd 6431 ixgbe_reset_subtask(adapter);
7086400d
AD		 6432 ixgbe_sfp_detection_subtask(adapter);
6433 ixgbe_sfp_link_config_subtask(adapter);
f0f9778d 6434 ixgbe_check_overtemp_subtask(adapter);
93c52dd0 6435 ixgbe_watchdog_subtask(adapter);
d034acf1 6436 ixgbe_fdir_reinit_subtask(adapter);
93c52dd0 6437 ixgbe_check_hang_subtask(adapter);
7086400d
AD		 6438
6439 ixgbe_service_event_complete(adapter);
9a799d71
AK		 6440}
6441
897ab156
AD		 6442void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens,
6443 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
9a799d71
AK		 6444{
6445 struct ixgbe_adv_tx_context_desc *context_desc;
897ab156 6446 u16 i = tx_ring->next_to_use;
9a799d71 6447
897ab156 6448 context_desc = IXGBE_TX_CTXTDESC_ADV(tx_ring, i);
9a799d71 6449
897ab156
AD		 6450 i++;
6451 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
9a799d71 6452
897ab156
AD		 6453 /* set bits to identify this as an advanced context descriptor */
6454 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
9a799d71 6455
897ab156
AD		 6456 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
6457 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);
6458 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
6459 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
6460}
9a799d71 6461
897ab156
AD		 6462static int ixgbe_tso(struct ixgbe_ring *tx_ring, struct sk_buff *skb,
6463 u32 tx_flags, __be16 protocol, u8 *hdr_len)
6464{
6465 int err;
6466 u32 vlan_macip_lens, type_tucmd;
6467 u32 mss_l4len_idx, l4len;
9a799d71 6468
897ab156
AD		 6469 if (!skb_is_gso(skb))
6470 return 0;
9a799d71 6471
897ab156
AD		 6472 if (skb_header_cloned(skb)) {
6473 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
6474 if (err)
6475 return err;
9a799d71 6476 }
9a799d71 6477
897ab156
AD		 6478 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
6479 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
6480
6481 if (protocol == __constant_htons(ETH_P_IP)) {
6482 struct iphdr *iph = ip_hdr(skb);
6483 iph->tot_len = 0;
6484 iph->check = 0;
6485 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
6486 iph->daddr, 0,
6487 IPPROTO_TCP,
6488 0);
6489 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
6490 } else if (skb_is_gso_v6(skb)) {
6491 ipv6_hdr(skb)->payload_len = 0;
6492 tcp_hdr(skb)->check =
6493 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
6494 &ipv6_hdr(skb)->daddr,
6495 0, IPPROTO_TCP, 0);
6496 }
6497
6498 l4len = tcp_hdrlen(skb);
6499 *hdr_len = skb_transport_offset(skb) + l4len;
6500
6501 /* mss_l4len_id: use 1 as index for TSO */
6502 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
6503 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
6504 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
6505
6506 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
6507 vlan_macip_lens = skb_network_header_len(skb);
6508 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
6509 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
6510
6511 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
6512 mss_l4len_idx);
6513
6514 return 1;
6515}
6516
6517static bool ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
6518 struct sk_buff *skb, u32 tx_flags,
6519 __be16 protocol)
7ca647bd 6520{
897ab156
AD		 6521 u32 vlan_macip_lens = 0;
6522 u32 mss_l4len_idx = 0;
6523 u32 type_tucmd = 0;
7ca647bd 6524
897ab156 6525 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7f9643fd
AD		 6526 if (!(tx_flags & IXGBE_TX_FLAGS_HW_VLAN) &&
6527 !(tx_flags & IXGBE_TX_FLAGS_TXSW))
897ab156
AD		 6528 return false;
6529 } else {
6530 u8 l4_hdr = 0;
6531 switch (protocol) {
6532 case __constant_htons(ETH_P_IP):
6533 vlan_macip_lens |= skb_network_header_len(skb);
6534 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
6535 l4_hdr = ip_hdr(skb)->protocol;
7ca647bd 6536 break;
897ab156
AD		 6537 case __constant_htons(ETH_P_IPV6):
6538 vlan_macip_lens |= skb_network_header_len(skb);
6539 l4_hdr = ipv6_hdr(skb)->nexthdr;
6540 break;
6541 default:
6542 if (unlikely(net_ratelimit())) {
6543 dev_warn(tx_ring->dev,
6544 "partial checksum but proto=%x!\n",
6545 skb->protocol);
6546 }
7ca647bd
JP		 6547 break;
6548 }
897ab156
AD		 6549
6550 switch (l4_hdr) {
7ca647bd 6551 case IPPROTO_TCP:
897ab156
AD		 6552 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
6553 mss_l4len_idx = tcp_hdrlen(skb) <<
6554 IXGBE_ADVTXD_L4LEN_SHIFT;
7ca647bd
JP		 6555 break;
6556 case IPPROTO_SCTP:
897ab156
AD		 6557 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
6558 mss_l4len_idx = sizeof(struct sctphdr) <<
6559 IXGBE_ADVTXD_L4LEN_SHIFT;
6560 break;
6561 case IPPROTO_UDP:
6562 mss_l4len_idx = sizeof(struct udphdr) <<
6563 IXGBE_ADVTXD_L4LEN_SHIFT;
6564 break;
6565 default:
6566 if (unlikely(net_ratelimit())) {
6567 dev_warn(tx_ring->dev,
6568 "partial checksum but l4 proto=%x!\n",
6569 skb->protocol);
6570 }
7ca647bd
JP		 6571 break;
6572 }
7ca647bd
JP		 6573 }
6574
897ab156
AD		 6575 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
6576 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
9a799d71 6577
897ab156
AD		 6578 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0,
6579 type_tucmd, mss_l4len_idx);
9a799d71 6580
897ab156 6581 return (skb->ip_summed == CHECKSUM_PARTIAL);
9a799d71
AK		 6582}
6583
d3d00239 6584static __le32 ixgbe_tx_cmd_type(u32 tx_flags)
9a799d71 6585{
d3d00239
AD		 6586 /* set type for advanced descriptor with frame checksum insertion */
6587 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
6588 IXGBE_ADVTXD_DCMD_IFCS |
6589 IXGBE_ADVTXD_DCMD_DEXT);
9a799d71 6590
d3d00239 6591 /* set HW vlan bit if vlan is present */
66f32a8b 6592 if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN)
d3d00239 6593 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
9a799d71 6594
d3d00239
AD		 6595 /* set segmentation enable bits for TSO/FSO */
6596#ifdef IXGBE_FCOE
6597 if ((tx_flags & IXGBE_TX_FLAGS_TSO) || (tx_flags & IXGBE_TX_FLAGS_FSO))
6598#else
6599 if (tx_flags & IXGBE_TX_FLAGS_TSO)
6600#endif
6601 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
eacd73f7 6602
d3d00239
AD		 6603 return cmd_type;
6604}
9a799d71 6605
d3d00239
AD		 6606static __le32 ixgbe_tx_olinfo_status(u32 tx_flags, unsigned int paylen)
6607{
6608 __le32 olinfo_status =
6609 cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
44df32c5 6610
d3d00239
AD		 6611 if (tx_flags & IXGBE_TX_FLAGS_TSO) {
6612 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM |
6613 (1 << IXGBE_ADVTXD_IDX_SHIFT));
6614 /* enble IPv4 checksum for TSO */
6615 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
6616 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
9a799d71
AK		 6617 }
6618
d3d00239
AD		 6619 /* enable L4 checksum for TSO and TX checksum offload */
6620 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
6621 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
9a799d71 6622
d3d00239
AD		 6623#ifdef IXGBE_FCOE
6624 /* use index 1 context for FCOE/FSO */
6625 if (tx_flags & IXGBE_TX_FLAGS_FCOE)
6626 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC |
6627 (1 << IXGBE_ADVTXD_IDX_SHIFT));
9a799d71 6628
d3d00239 6629#endif
7f9643fd
AD		 6630 /*
6631 * Check Context must be set if Tx switch is enabled, which it
6632 * always is for case where virtual functions are running
6633 */
6634 if (tx_flags & IXGBE_TX_FLAGS_TXSW)
6635 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
6636
d3d00239
AD		 6637 return olinfo_status;
6638}
44df32c5 6639
d3d00239
AD		 6640#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \
6641 IXGBE_TXD_CMD_RS)
6642
6643static void ixgbe_tx_map(struct ixgbe_ring *tx_ring,
6644 struct sk_buff *skb,
6645 struct ixgbe_tx_buffer *first,
6646 u32 tx_flags,
6647 const u8 hdr_len)
6648{
6649 struct device *dev = tx_ring->dev;
6650 struct ixgbe_tx_buffer *tx_buffer_info;
6651 union ixgbe_adv_tx_desc *tx_desc;
6652 dma_addr_t dma;
6653 __le32 cmd_type, olinfo_status;
6654 struct skb_frag_struct *frag;
6655 unsigned int f = 0;
6656 unsigned int data_len = skb->data_len;
6657 unsigned int size = skb_headlen(skb);
6658 u32 offset = 0;
6659 u32 paylen = skb->len - hdr_len;
6660 u16 i = tx_ring->next_to_use;
6661 u16 gso_segs;
6662
6663#ifdef IXGBE_FCOE
6664 if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
6665 if (data_len >= sizeof(struct fcoe_crc_eof)) {
6666 data_len -= sizeof(struct fcoe_crc_eof);
6667 } else {
6668 size -= sizeof(struct fcoe_crc_eof) - data_len;
6669 data_len = 0;
9a799d71
AK		 6670 }
6671 }
44df32c5 6672
d3d00239
AD		 6673#endif
6674 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
6675 if (dma_mapping_error(dev, dma))
6676 goto dma_error;
8ad494b0 6677
d3d00239
AD		 6678 cmd_type = ixgbe_tx_cmd_type(tx_flags);
6679 olinfo_status = ixgbe_tx_olinfo_status(tx_flags, paylen);
9a799d71 6680
d3d00239 6681 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
e5a43549 6682
d3d00239
AD		 6683 for (;;) {
6684 while (size > IXGBE_MAX_DATA_PER_TXD) {
6685 tx_desc->read.buffer_addr = cpu_to_le64(dma + offset);
6686 tx_desc->read.cmd_type_len =
6687 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
6688 tx_desc->read.olinfo_status = olinfo_status;
e5a43549 6689
d3d00239
AD		 6690 offset += IXGBE_MAX_DATA_PER_TXD;
6691 size -= IXGBE_MAX_DATA_PER_TXD;
e5a43549 6692
d3d00239
AD		 6693 tx_desc++;
6694 i++;
6695 if (i == tx_ring->count) {
6696 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
6697 i = 0;
6698 }
6699 }
e5a43549 6700
e5a43549 6701 tx_buffer_info = &tx_ring->tx_buffer_info[i];
d3d00239
AD		 6702 tx_buffer_info->length = offset + size;
6703 tx_buffer_info->tx_flags = tx_flags;
6704 tx_buffer_info->dma = dma;
9a799d71 6705
d3d00239
AD		 6706 tx_desc->read.buffer_addr = cpu_to_le64(dma + offset);
6707 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
6708 tx_desc->read.olinfo_status = olinfo_status;
9a799d71 6709
d3d00239
AD		 6710 if (!data_len)
6711 break;
9a799d71 6712
d3d00239
AD		 6713 frag = &skb_shinfo(skb)->frags[f];
6714#ifdef IXGBE_FCOE
9e903e08 6715 size = min_t(unsigned int, data_len, skb_frag_size(frag));
d3d00239 6716#else
9e903e08 6717 size = skb_frag_size(frag);
d3d00239
AD		 6718#endif
6719 data_len -= size;
6720 f++;
9a799d71 6721
d3d00239
AD		 6722 offset = 0;
6723 tx_flags |= IXGBE_TX_FLAGS_MAPPED_AS_PAGE;
9a799d71 6724
877749bf 6725 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
d3d00239
AD		 6726 if (dma_mapping_error(dev, dma))
6727 goto dma_error;
9a799d71 6728
d3d00239
AD		 6729 tx_desc++;
6730 i++;
6731 if (i == tx_ring->count) {
6732 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
6733 i = 0;
6734 }
6735 }
9a799d71 6736
d3d00239 6737 tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);
9a799d71 6738
d3d00239
AD		 6739 i++;
6740 if (i == tx_ring->count)
6741 i = 0;
9a799d71 6742
d3d00239 6743 tx_ring->next_to_use = i;
eacd73f7 6744
d3d00239
AD		 6745 if (tx_flags & IXGBE_TX_FLAGS_TSO)
6746 gso_segs = skb_shinfo(skb)->gso_segs;
6747#ifdef IXGBE_FCOE
6748 /* adjust for FCoE Sequence Offload */
6749 else if (tx_flags & IXGBE_TX_FLAGS_FSO)
6750 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
6751 skb_shinfo(skb)->gso_size);
6752#endif /* IXGBE_FCOE */
6753 else
6754 gso_segs = 1;
9a799d71 6755
d3d00239
AD		 6756 /* multiply data chunks by size of headers */
6757 tx_buffer_info->bytecount = paylen + (gso_segs * hdr_len);
6758 tx_buffer_info->gso_segs = gso_segs;
6759 tx_buffer_info->skb = skb;
9a799d71 6760
d3d00239
AD		 6761 /* set the timestamp */
6762 first->time_stamp = jiffies;
9a799d71
AK		 6763
6764 /*
6765 * Force memory writes to complete before letting h/w
6766 * know there are new descriptors to fetch. (Only
6767 * applicable for weak-ordered memory model archs,
6768 * such as IA-64).
6769 */
6770 wmb();
6771
d3d00239
AD		 6772 /* set next_to_watch value indicating a packet is present */
6773 first->next_to_watch = tx_desc;
6774
6775 /* notify HW of packet */
84ea2591 6776 writel(i, tx_ring->tail);
d3d00239
AD		 6777
6778 return;
6779dma_error:
6780 dev_err(dev, "TX DMA map failed\n");
6781
6782 /* clear dma mappings for failed tx_buffer_info map */
6783 for (;;) {
6784 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6785 ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
6786 if (tx_buffer_info == first)
6787 break;
6788 if (i == 0)
6789 i = tx_ring->count;
6790 i--;
6791 }
6792
6793 dev_kfree_skb_any(skb);
6794
6795 tx_ring->next_to_use = i;
9a799d71
AK		 6796}
6797
69830529
AD		 6798static void ixgbe_atr(struct ixgbe_ring *ring, struct sk_buff *skb,
6799 u32 tx_flags, __be16 protocol)
6800{
6801 struct ixgbe_q_vector *q_vector = ring->q_vector;
6802 union ixgbe_atr_hash_dword input = { .dword = 0 };
6803 union ixgbe_atr_hash_dword common = { .dword = 0 };
6804 union {
6805 unsigned char *network;
6806 struct iphdr *ipv4;
6807 struct ipv6hdr *ipv6;
6808 } hdr;
ee9e0f0b 6809 struct tcphdr *th;
905e4a41 6810 __be16 vlan_id;
c4cf55e5 6811
69830529
AD		 6812 /* if ring doesn't have a interrupt vector, cannot perform ATR */
6813 if (!q_vector)
6814 return;
6815
6816 /* do nothing if sampling is disabled */
6817 if (!ring->atr_sample_rate)
d3ead241 6818 return;
c4cf55e5 6819
69830529 6820 ring->atr_count++;
c4cf55e5 6821
69830529
AD		 6822 /* snag network header to get L4 type and address */
6823 hdr.network = skb_network_header(skb);
6824
6825 /* Currently only IPv4/IPv6 with TCP is supported */
6826 if ((protocol != __constant_htons(ETH_P_IPV6) ||
6827 hdr.ipv6->nexthdr != IPPROTO_TCP) &&
6828 (protocol != __constant_htons(ETH_P_IP) ||
6829 hdr.ipv4->protocol != IPPROTO_TCP))
6830 return;
ee9e0f0b
AD		 6831
6832 th = tcp_hdr(skb);
c4cf55e5 6833
66f32a8b
AD		 6834 /* skip this packet since it is invalid or the socket is closing */
6835 if (!th || th->fin)
69830529
AD		 6836 return;
6837
6838 /* sample on all syn packets or once every atr sample count */
6839 if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
6840 return;
6841
6842 /* reset sample count */
6843 ring->atr_count = 0;
6844
6845 vlan_id = htons(tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
6846
6847 /*
6848 * src and dst are inverted, think how the receiver sees them
6849 *
6850 * The input is broken into two sections, a non-compressed section
6851 * containing vm_pool, vlan_id, and flow_type. The rest of the data
6852 * is XORed together and stored in the compressed dword.
6853 */
6854 input.formatted.vlan_id = vlan_id;
6855
6856 /*
6857 * since src port and flex bytes occupy the same word XOR them together
6858 * and write the value to source port portion of compressed dword
6859 */
66f32a8b 6860 if (tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
69830529
AD		 6861 common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q);
6862 else
6863 common.port.src ^= th->dest ^ protocol;
6864 common.port.dst ^= th->source;
6865
6866 if (protocol == __constant_htons(ETH_P_IP)) {
6867 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
6868 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
6869 } else {
6870 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
6871 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
6872 hdr.ipv6->saddr.s6_addr32[1] ^
6873 hdr.ipv6->saddr.s6_addr32[2] ^
6874 hdr.ipv6->saddr.s6_addr32[3] ^
6875 hdr.ipv6->daddr.s6_addr32[0] ^
6876 hdr.ipv6->daddr.s6_addr32[1] ^
6877 hdr.ipv6->daddr.s6_addr32[2] ^
6878 hdr.ipv6->daddr.s6_addr32[3];
6879 }
c4cf55e5
PWJ		 6880
6881 /* This assumes the Rx queue and Tx queue are bound to the same CPU */
69830529
AD		 6882 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
6883 input, common, ring->queue_index);
c4cf55e5
PWJ		 6884}
6885
63544e9c 6886static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
e092be60 6887{
fc77dc3c 6888 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
e092be60
AV		 6889 /* Herbert's original patch had:
6890 * smp_mb__after_netif_stop_queue();
6891 * but since that doesn't exist yet, just open code it. */
6892 smp_mb();
6893
6894 /* We need to check again in a case another CPU has just
6895 * made room available. */
7d4987de 6896 if (likely(ixgbe_desc_unused(tx_ring) < size))
e092be60
AV		 6897 return -EBUSY;
6898
6899 /* A reprieve! - use start_queue because it doesn't call schedule */
fc77dc3c 6900 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
5b7da515 6901 ++tx_ring->tx_stats.restart_queue;
e092be60
AV		 6902 return 0;
6903}
6904
82d4e46e 6905static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
e092be60 6906{
7d4987de 6907 if (likely(ixgbe_desc_unused(tx_ring) >= size))
e092be60 6908 return 0;
fc77dc3c 6909 return __ixgbe_maybe_stop_tx(tx_ring, size);
e092be60
AV		 6910}
6911
09a3b1f8
SH		 6912static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb)
6913{
6914 struct ixgbe_adapter *adapter = netdev_priv(dev);
6440752c
AD		 6915 int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
6916 smp_processor_id();
56075a98 6917#ifdef IXGBE_FCOE
6440752c 6918 __be16 protocol = vlan_get_protocol(skb);
5e09a105 6919
e5b64635
JF		 6920 if (((protocol == htons(ETH_P_FCOE)) ||
6921 (protocol == htons(ETH_P_FIP))) &&
6922 (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
6923 txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1);
6924 txq += adapter->ring_feature[RING_F_FCOE].mask;
6925 return txq;
56075a98
JF		 6926 }
6927#endif
6928
fdd3d631
KK		 6929 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
6930 while (unlikely(txq >= dev->real_num_tx_queues))
6931 txq -= dev->real_num_tx_queues;
5f715823 6932 return txq;
fdd3d631 6933 }
c4cf55e5 6934
09a3b1f8
SH		 6935 return skb_tx_hash(dev, skb);
6936}
6937
fc77dc3c 6938netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
84418e3b
AD		 6939 struct ixgbe_adapter *adapter,
6940 struct ixgbe_ring *tx_ring)
9a799d71 6941{
d3d00239 6942 struct ixgbe_tx_buffer *first;
5f715823 6943 int tso;
d3d00239 6944 u32 tx_flags = 0;
a535c30e
AD		 6945#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
6946 unsigned short f;
6947#endif
a535c30e 6948 u16 count = TXD_USE_COUNT(skb_headlen(skb));
66f32a8b 6949 __be16 protocol = skb->protocol;
63544e9c 6950 u8 hdr_len = 0;
5e09a105 6951
a535c30e
AD		 6952 /*
6953 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
6954 * + 1 desc for skb_head_len/IXGBE_MAX_DATA_PER_TXD,
6955 * + 2 desc gap to keep tail from touching head,
6956 * + 1 desc for context descriptor,
6957 * otherwise try next time
6958 */
6959#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
6960 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6961 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
6962#else
6963 count += skb_shinfo(skb)->nr_frags;
6964#endif
6965 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
6966 tx_ring->tx_stats.tx_busy++;
6967 return NETDEV_TX_BUSY;
6968 }
6969
7f9643fd
AD		 6970#ifdef CONFIG_PCI_IOV
6971 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
6972 tx_flags |= IXGBE_TX_FLAGS_TXSW;
6973
6974#endif
66f32a8b 6975 /* if we have a HW VLAN tag being added default to the HW one */
eab6d18d 6976 if (vlan_tx_tag_present(skb)) {
66f32a8b
AD		 6977 tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
6978 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
6979 /* else if it is a SW VLAN check the next protocol and store the tag */
6980 } else if (protocol == __constant_htons(ETH_P_8021Q)) {
6981 struct vlan_hdr *vhdr, _vhdr;
6982 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
6983 if (!vhdr)
6984 goto out_drop;
6985
6986 protocol = vhdr->h_vlan_encapsulated_proto;
6987 tx_flags |= ntohs(vhdr->h_vlan_TCI) << IXGBE_TX_FLAGS_VLAN_SHIFT;
6988 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
6989 }
6990
32701dc2 6991 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
66f32a8b 6992 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
09dca476
AD		 6993 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
6994 (skb->priority != TC_PRIO_CONTROL))) {
66f32a8b 6995 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
32701dc2
JF		 6996 tx_flags |= (skb->priority & 0x7) <<
6997 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
66f32a8b
AD		 6998 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
6999 struct vlan_ethhdr *vhdr;
7000 if (skb_header_cloned(skb) &&
7001 pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
7002 goto out_drop;
7003 vhdr = (struct vlan_ethhdr *)skb->data;
7004 vhdr->h_vlan_TCI = htons(tx_flags >>
7005 IXGBE_TX_FLAGS_VLAN_SHIFT);
7006 } else {
7007 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
2f90b865 7008 }
9a799d71 7009 }
eacd73f7 7010
a535c30e 7011 /* record the location of the first descriptor for this packet */
d3d00239 7012 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
a535c30e 7013
eacd73f7 7014#ifdef IXGBE_FCOE
66f32a8b
AD		 7015 /* setup tx offload for FCoE */
7016 if ((protocol == __constant_htons(ETH_P_FCOE)) &&
7017 (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
897ab156
AD		 7018 tso = ixgbe_fso(tx_ring, skb, tx_flags, &hdr_len);
7019 if (tso < 0)
7020 goto out_drop;
7021 else if (tso)
66f32a8b
AD		 7022 tx_flags |= IXGBE_TX_FLAGS_FSO |
7023 IXGBE_TX_FLAGS_FCOE;
7024 else
7025 tx_flags |= IXGBE_TX_FLAGS_FCOE;
9a799d71 7026
66f32a8b 7027 goto xmit_fcoe;
eacd73f7 7028 }
9a799d71 7029
66f32a8b
AD		 7030#endif /* IXGBE_FCOE */
7031 /* setup IPv4/IPv6 offloads */
7032 if (protocol == __constant_htons(ETH_P_IP))
7033 tx_flags |= IXGBE_TX_FLAGS_IPV4;
9a799d71 7034
66f32a8b
AD		 7035 tso = ixgbe_tso(tx_ring, skb, tx_flags, protocol, &hdr_len);
7036 if (tso < 0)
897ab156 7037 goto out_drop;
66f32a8b
AD		 7038 else if (tso)
7039 tx_flags |= IXGBE_TX_FLAGS_TSO;
7040 else if (ixgbe_tx_csum(tx_ring, skb, tx_flags, protocol))
7041 tx_flags |= IXGBE_TX_FLAGS_CSUM;
7042
7043 /* add the ATR filter if ATR is on */
7044 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
7045 ixgbe_atr(tx_ring, skb, tx_flags, protocol);
7046
7047#ifdef IXGBE_FCOE
7048xmit_fcoe:
7049#endif /* IXGBE_FCOE */
d3d00239
AD		 7050 ixgbe_tx_map(tx_ring, skb, first, tx_flags, hdr_len);
7051
7052 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
9a799d71
AK		 7053
7054 return NETDEV_TX_OK;
897ab156
AD		 7055
7056out_drop:
7057 dev_kfree_skb_any(skb);
7058 return NETDEV_TX_OK;
9a799d71
AK		 7059}
7060
84418e3b
AD		 7061static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
7062{
7063 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7064 struct ixgbe_ring *tx_ring;
7065
7066 tx_ring = adapter->tx_ring[skb->queue_mapping];
fc77dc3c 7067 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
84418e3b
AD		 7068}
7069
9a799d71
AK		 7070/**
7071 * ixgbe_set_mac - Change the Ethernet Address of the NIC
7072 * @netdev: network interface device structure
7073 * @p: pointer to an address structure
7074 *
7075 * Returns 0 on success, negative on failure
7076 **/
7077static int ixgbe_set_mac(struct net_device *netdev, void *p)
7078{
7079 struct ixgbe_adapter *adapter = netdev_priv(netdev);
b4617240 7080 struct ixgbe_hw *hw = &adapter->hw;
9a799d71
AK		 7081 struct sockaddr *addr = p;
7082
7083 if (!is_valid_ether_addr(addr->sa_data))
7084 return -EADDRNOTAVAIL;
7085
7086 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
b4617240 7087 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9a799d71 7088
1cdd1ec8
GR		 7089 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
7090 IXGBE_RAH_AV);
9a799d71
AK		 7091
7092 return 0;
7093}
7094
6b73e10d
BH		 7095static int
7096ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
7097{
7098 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7099 struct ixgbe_hw *hw = &adapter->hw;
7100 u16 value;
7101 int rc;
7102
7103 if (prtad != hw->phy.mdio.prtad)
7104 return -EINVAL;
7105 rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
7106 if (!rc)
7107 rc = value;
7108 return rc;
7109}
7110
7111static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
7112 u16 addr, u16 value)
7113{
7114 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7115 struct ixgbe_hw *hw = &adapter->hw;
7116
7117 if (prtad != hw->phy.mdio.prtad)
7118 return -EINVAL;
7119 return hw->phy.ops.write_reg(hw, addr, devad, value);
7120}
7121
7122static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
7123{
7124 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7125
7126 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
7127}
7128
0365e6e4
PW		 7129/**
7130 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
31278e71 7131 * netdev->dev_addrs
0365e6e4
PW		 7132 * @netdev: network interface device structure
7133 *
7134 * Returns non-zero on failure
7135 **/
7136static int ixgbe_add_sanmac_netdev(struct net_device *dev)
7137{
7138 int err = 0;
7139 struct ixgbe_adapter *adapter = netdev_priv(dev);
7140 struct ixgbe_mac_info *mac = &adapter->hw.mac;
7141
7142 if (is_valid_ether_addr(mac->san_addr)) {
7143 rtnl_lock();
7144 err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
7145 rtnl_unlock();
7146 }
7147 return err;
7148}
7149
7150/**
7151 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
31278e71 7152 * netdev->dev_addrs
0365e6e4
PW		 7153 * @netdev: network interface device structure
7154 *
7155 * Returns non-zero on failure
7156 **/
7157static int ixgbe_del_sanmac_netdev(struct net_device *dev)
7158{
7159 int err = 0;
7160 struct ixgbe_adapter *adapter = netdev_priv(dev);
7161 struct ixgbe_mac_info *mac = &adapter->hw.mac;
7162
7163 if (is_valid_ether_addr(mac->san_addr)) {
7164 rtnl_lock();
7165 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
7166 rtnl_unlock();
7167 }
7168 return err;
7169}
7170
9a799d71
AK		 7171#ifdef CONFIG_NET_POLL_CONTROLLER
7172/*
7173 * Polling 'interrupt' - used by things like netconsole to send skbs
7174 * without having to re-enable interrupts. It's not called while
7175 * the interrupt routine is executing.
7176 */
7177static void ixgbe_netpoll(struct net_device *netdev)
7178{
7179 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8f9a7167 7180 int i;
9a799d71 7181
1a647bd2
AD		 7182 /* if interface is down do nothing */
7183 if (test_bit(__IXGBE_DOWN, &adapter->state))
7184 return;
7185
9a799d71 7186 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
8f9a7167
PWJ		 7187 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
7188 int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
7189 for (i = 0; i < num_q_vectors; i++) {
7190 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
4ff7fb12 7191 ixgbe_msix_clean_rings(0, q_vector);
8f9a7167
PWJ		 7192 }
7193 } else {
7194 ixgbe_intr(adapter->pdev->irq, netdev);
7195 }
9a799d71 7196 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
9a799d71
AK		 7197}
7198#endif
7199
de1036b1
ED		 7200static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
7201 struct rtnl_link_stats64 *stats)
7202{
7203 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7204 int i;
7205
1a51502b 7206 rcu_read_lock();
de1036b1 7207 for (i = 0; i < adapter->num_rx_queues; i++) {
1a51502b 7208 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
de1036b1
ED		 7209 u64 bytes, packets;
7210 unsigned int start;
7211
1a51502b
ED		 7212 if (ring) {
7213 do {
7214 start = u64_stats_fetch_begin_bh(&ring->syncp);
7215 packets = ring->stats.packets;
7216 bytes = ring->stats.bytes;
7217 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
7218 stats->rx_packets += packets;
7219 stats->rx_bytes += bytes;
7220 }
de1036b1 7221 }
1ac9ad13
ED		 7222
7223 for (i = 0; i < adapter->num_tx_queues; i++) {
7224 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
7225 u64 bytes, packets;
7226 unsigned int start;
7227
7228 if (ring) {
7229 do {
7230 start = u64_stats_fetch_begin_bh(&ring->syncp);
7231 packets = ring->stats.packets;
7232 bytes = ring->stats.bytes;
7233 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
7234 stats->tx_packets += packets;
7235 stats->tx_bytes += bytes;
7236 }
7237 }
1a51502b 7238 rcu_read_unlock();
de1036b1
ED		 7239 /* following stats updated by ixgbe_watchdog_task() */
7240 stats->multicast = netdev->stats.multicast;
7241 stats->rx_errors = netdev->stats.rx_errors;
7242 stats->rx_length_errors = netdev->stats.rx_length_errors;
7243 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
7244 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
7245 return stats;
7246}
7247
8b1c0b24
JF		 7248/* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
7249 * #adapter: pointer to ixgbe_adapter
7250 * @tc: number of traffic classes currently enabled
7251 *
7252 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
7253 * 802.1Q priority maps to a packet buffer that exists.
7254 */
7255static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
7256{
7257 struct ixgbe_hw *hw = &adapter->hw;
7258 u32 reg, rsave;
7259 int i;
7260
7261 /* 82598 have a static priority to TC mapping that can not
7262 * be changed so no validation is needed.
7263 */
7264 if (hw->mac.type == ixgbe_mac_82598EB)
7265 return;
7266
7267 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
7268 rsave = reg;
7269
7270 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
7271 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
7272
7273 /* If up2tc is out of bounds default to zero */
7274 if (up2tc > tc)
7275 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
7276 }
7277
7278 if (reg != rsave)
7279 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
7280
7281 return;
7282}
7283
7284
7285/* ixgbe_setup_tc - routine to configure net_device for multiple traffic
7286 * classes.
7287 *
7288 * @netdev: net device to configure
7289 * @tc: number of traffic classes to enable
7290 */
7291int ixgbe_setup_tc(struct net_device *dev, u8 tc)
7292{
8b1c0b24
JF		 7293 struct ixgbe_adapter *adapter = netdev_priv(dev);
7294 struct ixgbe_hw *hw = &adapter->hw;
8b1c0b24 7295
e7589eab
JF		 7296 /* Multiple traffic classes requires multiple queues */
7297 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
7298 e_err(drv, "Enable failed, needs MSI-X\n");
7299 return -EINVAL;
7300 }
8b1c0b24
JF		 7301
7302 /* Hardware supports up to 8 traffic classes */
4de2a022 7303 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs ||
8b1c0b24
JF		 7304 (hw->mac.type == ixgbe_mac_82598EB && tc < MAX_TRAFFIC_CLASS))
7305 return -EINVAL;
7306
7307 /* Hardware has to reinitialize queues and interrupts to
52f33af8 7308 * match packet buffer alignment. Unfortunately, the
8b1c0b24
JF		 7309 * hardware is not flexible enough to do this dynamically.
7310 */
7311 if (netif_running(dev))
7312 ixgbe_close(dev);
7313 ixgbe_clear_interrupt_scheme(adapter);
7314
e7589eab 7315 if (tc) {
8b1c0b24 7316 netdev_set_num_tc(dev, tc);
e7589eab
JF		 7317 adapter->last_lfc_mode = adapter->hw.fc.current_mode;
7318
7319 adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
7320 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
7321
7322 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
7323 adapter->hw.fc.requested_mode = ixgbe_fc_none;
7324 } else {
8b1c0b24
JF		 7325 netdev_reset_tc(dev);
7326
e7589eab
JF		 7327 adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
7328
7329 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
7330 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
7331
7332 adapter->temp_dcb_cfg.pfc_mode_enable = false;
7333 adapter->dcb_cfg.pfc_mode_enable = false;
7334 }
7335
8b1c0b24
JF		 7336 ixgbe_init_interrupt_scheme(adapter);
7337 ixgbe_validate_rtr(adapter, tc);
7338 if (netif_running(dev))
7339 ixgbe_open(dev);
7340
7341 return 0;
7342}
de1036b1 7343
082757af
DS		 7344void ixgbe_do_reset(struct net_device *netdev)
7345{
7346 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7347
7348 if (netif_running(netdev))
7349 ixgbe_reinit_locked(adapter);
7350 else
7351 ixgbe_reset(adapter);
7352}
7353
c8f44aff
MM		 7354static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
7355 netdev_features_t data)
082757af
DS		 7356{
7357 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7358
7359#ifdef CONFIG_DCB
7360 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)
7361 data &= ~NETIF_F_HW_VLAN_RX;
7362#endif
7363
7364 /* return error if RXHASH is being enabled when RSS is not supported */
7365 if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
7366 data &= ~NETIF_F_RXHASH;
7367
7368 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
7369 if (!(data & NETIF_F_RXCSUM))
7370 data &= ~NETIF_F_LRO;
7371
7372 /* Turn off LRO if not RSC capable or invalid ITR settings */
7373 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) {
7374 data &= ~NETIF_F_LRO;
7375 } else if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
7376 (adapter->rx_itr_setting != 1 &&
7377 adapter->rx_itr_setting > IXGBE_MAX_RSC_INT_RATE)) {
7378 data &= ~NETIF_F_LRO;
7379 e_info(probe, "rx-usecs set too low, not enabling RSC\n");
7380 }
7381
7382 return data;
7383}
7384
c8f44aff
MM		 7385static int ixgbe_set_features(struct net_device *netdev,
7386 netdev_features_t data)
082757af
DS		 7387{
7388 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7389 bool need_reset = false;
7390
7391 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
7392 if (!(data & NETIF_F_RXCSUM))
7393 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED;
7394 else
7395 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
7396
7397 /* Make sure RSC matches LRO, reset if change */
7398 if (!!(data & NETIF_F_LRO) !=
7399 !!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
7400 adapter->flags2 ^= IXGBE_FLAG2_RSC_ENABLED;
7401 switch (adapter->hw.mac.type) {
7402 case ixgbe_mac_X540:
7403 case ixgbe_mac_82599EB:
7404 need_reset = true;
7405 break;
7406 default:
7407 break;
7408 }
7409 }
7410
7411 /*
7412 * Check if Flow Director n-tuple support was enabled or disabled. If
7413 * the state changed, we need to reset.
7414 */
7415 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) {
7416 /* turn off ATR, enable perfect filters and reset */
7417 if (data & NETIF_F_NTUPLE) {
7418 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
7419 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
7420 need_reset = true;
7421 }
7422 } else if (!(data & NETIF_F_NTUPLE)) {
7423 /* turn off Flow Director, set ATR and reset */
7424 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
7425 if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
7426 !(adapter->flags & IXGBE_FLAG_DCB_ENABLED))
7427 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
7428 need_reset = true;
7429 }
7430
7431 if (need_reset)
7432 ixgbe_do_reset(netdev);
7433
7434 return 0;
7435
7436}
7437
0edc3527 7438static const struct net_device_ops ixgbe_netdev_ops = {
e8e9f696 7439 .ndo_open = ixgbe_open,
0edc3527 7440 .ndo_stop = ixgbe_close,
00829823 7441 .ndo_start_xmit = ixgbe_xmit_frame,
09a3b1f8 7442 .ndo_select_queue = ixgbe_select_queue,
e90d400c 7443 .ndo_set_rx_mode = ixgbe_set_rx_mode,
0edc3527
SH		 7444 .ndo_validate_addr = eth_validate_addr,
7445 .ndo_set_mac_address = ixgbe_set_mac,
7446 .ndo_change_mtu = ixgbe_change_mtu,
7447 .ndo_tx_timeout = ixgbe_tx_timeout,
0edc3527
SH		 7448 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
7449 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
6b73e10d 7450 .ndo_do_ioctl = ixgbe_ioctl,
7f01648a
GR		 7451 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
7452 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
7453 .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw,
de4c7f65 7454 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
7f01648a 7455 .ndo_get_vf_config = ixgbe_ndo_get_vf_config,
de1036b1 7456 .ndo_get_stats64 = ixgbe_get_stats64,
24095aa3 7457 .ndo_setup_tc = ixgbe_setup_tc,
0edc3527
SH		 7458#ifdef CONFIG_NET_POLL_CONTROLLER
7459 .ndo_poll_controller = ixgbe_netpoll,
7460#endif
332d4a7d
YZ		 7461#ifdef IXGBE_FCOE
7462 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
68a683cf 7463 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
332d4a7d 7464 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
8450ff8c
YZ		 7465 .ndo_fcoe_enable = ixgbe_fcoe_enable,
7466 .ndo_fcoe_disable = ixgbe_fcoe_disable,
61a1fa10 7467 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
ea81875a 7468 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,
332d4a7d 7469#endif /* IXGBE_FCOE */
082757af
DS		 7470 .ndo_set_features = ixgbe_set_features,
7471 .ndo_fix_features = ixgbe_fix_features,
0edc3527
SH		 7472};
7473
1cdd1ec8
GR		 7474static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
7475 const struct ixgbe_info *ii)
7476{
7477#ifdef CONFIG_PCI_IOV
7478 struct ixgbe_hw *hw = &adapter->hw;
1cdd1ec8 7479
c6bda30a 7480 if (hw->mac.type == ixgbe_mac_82598EB)
1cdd1ec8
GR		 7481 return;
7482
7483 /* The 82599 supports up to 64 VFs per physical function
7484 * but this implementation limits allocation to 63 so that
7485 * basic networking resources are still available to the
7486 * physical function
7487 */
7488 adapter->num_vfs = (max_vfs > 63) ? 63 : max_vfs;
c6bda30a 7489 ixgbe_enable_sriov(adapter, ii);
1cdd1ec8
GR		 7490#endif /* CONFIG_PCI_IOV */
7491}
7492
9a799d71
AK		 7493/**
7494 * ixgbe_probe - Device Initialization Routine
7495 * @pdev: PCI device information struct
7496 * @ent: entry in ixgbe_pci_tbl
7497 *
7498 * Returns 0 on success, negative on failure
7499 *
7500 * ixgbe_probe initializes an adapter identified by a pci_dev structure.
7501 * The OS initialization, configuring of the adapter private structure,
7502 * and a hardware reset occur.
7503 **/
7504static int __devinit ixgbe_probe(struct pci_dev *pdev,
e8e9f696 7505 const struct pci_device_id *ent)
9a799d71
AK		 7506{
7507 struct net_device *netdev;
7508 struct ixgbe_adapter *adapter = NULL;
7509 struct ixgbe_hw *hw;
7510 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
9a799d71
AK		 7511 static int cards_found;
7512 int i, err, pci_using_dac;
289700db 7513 u8 part_str[IXGBE_PBANUM_LENGTH];
c85a2618 7514 unsigned int indices = num_possible_cpus();
eacd73f7
YZ		 7515#ifdef IXGBE_FCOE
7516 u16 device_caps;
7517#endif
289700db 7518 u32 eec;
c23f5b6b 7519 u16 wol_cap;
9a799d71 7520
bded64a7
AG		 7521 /* Catch broken hardware that put the wrong VF device ID in
7522 * the PCIe SR-IOV capability.
7523 */
7524 if (pdev->is_virtfn) {
7525 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
7526 pci_name(pdev), pdev->vendor, pdev->device);
7527 return -EINVAL;
7528 }
7529
9ce77666 7530 err = pci_enable_device_mem(pdev);
9a799d71
AK		 7531 if (err)
7532 return err;
7533
1b507730
NN		 7534 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
7535 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
9a799d71
AK		 7536 pci_using_dac = 1;
7537 } else {
1b507730 7538 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9a799d71 7539 if (err) {
1b507730
NN		 7540 err = dma_set_coherent_mask(&pdev->dev,
7541 DMA_BIT_MASK(32));
9a799d71 7542 if (err) {
b8bc0421
DC		 7543 dev_err(&pdev->dev,
7544 "No usable DMA configuration, aborting\n");
9a799d71
AK		 7545 goto err_dma;
7546 }
7547 }
7548 pci_using_dac = 0;
7549 }
7550
9ce77666 7551 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
e8e9f696 7552 IORESOURCE_MEM), ixgbe_driver_name);
9a799d71 7553 if (err) {
b8bc0421
DC		 7554 dev_err(&pdev->dev,
7555 "pci_request_selected_regions failed 0x%x\n", err);
9a799d71
AK		 7556 goto err_pci_reg;
7557 }
7558
19d5afd4 7559 pci_enable_pcie_error_reporting(pdev);
6fabd715 7560
9a799d71 7561 pci_set_master(pdev);
fb3b27bc 7562 pci_save_state(pdev);
9a799d71 7563
e901acd6
JF		 7564#ifdef CONFIG_IXGBE_DCB
7565 indices *= MAX_TRAFFIC_CLASS;
7566#endif
7567
c85a2618
JF		 7568 if (ii->mac == ixgbe_mac_82598EB)
7569 indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES);
7570 else
7571 indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES);
7572
e901acd6 7573#ifdef IXGBE_FCOE
c85a2618
JF		 7574 indices += min_t(unsigned int, num_possible_cpus(),
7575 IXGBE_MAX_FCOE_INDICES);
7576#endif
c85a2618 7577 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
9a799d71
AK		 7578 if (!netdev) {
7579 err = -ENOMEM;
7580 goto err_alloc_etherdev;
7581 }
7582
9a799d71
AK		 7583 SET_NETDEV_DEV(netdev, &pdev->dev);
7584
9a799d71 7585 adapter = netdev_priv(netdev);
c60fbb00 7586 pci_set_drvdata(pdev, adapter);
9a799d71
AK		 7587
7588 adapter->netdev = netdev;
7589 adapter->pdev = pdev;
7590 hw = &adapter->hw;
7591 hw->back = adapter;
7592 adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
7593
05857980 7594 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
e8e9f696 7595 pci_resource_len(pdev, 0));
9a799d71
AK		 7596 if (!hw->hw_addr) {
7597 err = -EIO;
7598 goto err_ioremap;
7599 }
7600
7601 for (i = 1; i <= 5; i++) {
7602 if (pci_resource_len(pdev, i) == 0)
7603 continue;
7604 }
7605
0edc3527 7606 netdev->netdev_ops = &ixgbe_netdev_ops;
9a799d71 7607 ixgbe_set_ethtool_ops(netdev);
9a799d71 7608 netdev->watchdog_timeo = 5 * HZ;
9fe93afd 7609 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
9a799d71 7610
9a799d71
AK		 7611 adapter->bd_number = cards_found;
7612
9a799d71
AK		 7613 /* Setup hw api */
7614 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
021230d4 7615 hw->mac.type = ii->mac;
9a799d71 7616
c44ade9e
JB		 7617 /* EEPROM */
7618 memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops));
7619 eec = IXGBE_READ_REG(hw, IXGBE_EEC);
7620 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
7621 if (!(eec & (1 << 8)))
7622 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
7623
7624 /* PHY */
7625 memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops));
c4900be0 7626 hw->phy.sfp_type = ixgbe_sfp_type_unknown;
6b73e10d
BH		 7627 /* ixgbe_identify_phy_generic will set prtad and mmds properly */
7628 hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
7629 hw->phy.mdio.mmds = 0;
7630 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
7631 hw->phy.mdio.dev = netdev;
7632 hw->phy.mdio.mdio_read = ixgbe_mdio_read;
7633 hw->phy.mdio.mdio_write = ixgbe_mdio_write;
c4900be0 7634
8ca783ab 7635 ii->get_invariants(hw);
9a799d71
AK		 7636
7637 /* setup the private structure */
7638 err = ixgbe_sw_init(adapter);
7639 if (err)
7640 goto err_sw_init;
7641
e86bff0e 7642 /* Make it possible the adapter to be woken up via WOL */
b93a2226
DS		 7643 switch (adapter->hw.mac.type) {
7644 case ixgbe_mac_82599EB:
7645 case ixgbe_mac_X540:
e86bff0e 7646 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
b93a2226
DS		 7647 break;
7648 default:
7649 break;
7650 }
e86bff0e 7651
bf069c97
DS		 7652 /*
7653 * If there is a fan on this device and it has failed log the
7654 * failure.
7655 */
7656 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
7657 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
7658 if (esdp & IXGBE_ESDP_SDP1)
396e799c 7659 e_crit(probe, "Fan has stopped, replace the adapter\n");
bf069c97
DS		 7660 }
7661
8ef78adc
PWJ		 7662 if (allow_unsupported_sfp)
7663 hw->allow_unsupported_sfp = allow_unsupported_sfp;
7664
c44ade9e 7665 /* reset_hw fills in the perm_addr as well */
119fc60a 7666 hw->phy.reset_if_overtemp = true;
c44ade9e 7667 err = hw->mac.ops.reset_hw(hw);
119fc60a 7668 hw->phy.reset_if_overtemp = false;
8ca783ab
DS		 7669 if (err == IXGBE_ERR_SFP_NOT_PRESENT &&
7670 hw->mac.type == ixgbe_mac_82598EB) {
8ca783ab
DS		 7671 err = 0;
7672 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
7086400d 7673 e_dev_err("failed to load because an unsupported SFP+ "
849c4542
ET		 7674 "module type was detected.\n");
7675 e_dev_err("Reload the driver after installing a supported "
7676 "module.\n");
04f165ef
PW		 7677 goto err_sw_init;
7678 } else if (err) {
849c4542 7679 e_dev_err("HW Init failed: %d\n", err);
c44ade9e
JB		 7680 goto err_sw_init;
7681 }
7682
1cdd1ec8
GR		 7683 ixgbe_probe_vf(adapter, ii);
7684
396e799c 7685 netdev->features = NETIF_F_SG |
e8e9f696 7686 NETIF_F_IP_CSUM |
082757af 7687 NETIF_F_IPV6_CSUM |
e8e9f696
JP		 7688 NETIF_F_HW_VLAN_TX |
7689 NETIF_F_HW_VLAN_RX |
082757af
DS		 7690 NETIF_F_HW_VLAN_FILTER |
7691 NETIF_F_TSO |
7692 NETIF_F_TSO6 |
082757af
DS		 7693 NETIF_F_RXHASH |
7694 NETIF_F_RXCSUM;
9a799d71 7695
082757af 7696 netdev->hw_features = netdev->features;
ad31c402 7697
58be7666
DS		 7698 switch (adapter->hw.mac.type) {
7699 case ixgbe_mac_82599EB:
7700 case ixgbe_mac_X540:
45a5ead0 7701 netdev->features |= NETIF_F_SCTP_CSUM;
082757af
DS		 7702 netdev->hw_features |= NETIF_F_SCTP_CSUM |
7703 NETIF_F_NTUPLE;
58be7666
DS		 7704 break;
7705 default:
7706 break;
7707 }
45a5ead0 7708
ad31c402
JK		 7709 netdev->vlan_features |= NETIF_F_TSO;
7710 netdev->vlan_features |= NETIF_F_TSO6;
22f32b7a 7711 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 7712 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
ad31c402
JK		 7713 netdev->vlan_features |= NETIF_F_SG;
7714
01789349
JP		 7715 netdev->priv_flags |= IFF_UNICAST_FLT;
7716
1cdd1ec8
GR		 7717 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7718 adapter->flags &= ~(IXGBE_FLAG_RSS_ENABLED |
7719 IXGBE_FLAG_DCB_ENABLED);
2f90b865 7720
7a6b6f51 7721#ifdef CONFIG_IXGBE_DCB
2f90b865
AD		 7722 netdev->dcbnl_ops = &dcbnl_ops;
7723#endif
7724
eacd73f7 7725#ifdef IXGBE_FCOE
0d551589 7726 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
eacd73f7
YZ		 7727 if (hw->mac.ops.get_device_caps) {
7728 hw->mac.ops.get_device_caps(hw, &device_caps);
0d551589
YZ		 7729 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
7730 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
eacd73f7
YZ		 7731 }
7732 }
5e09d7f6
YZ		 7733 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
7734 netdev->vlan_features |= NETIF_F_FCOE_CRC;
7735 netdev->vlan_features |= NETIF_F_FSO;
7736 netdev->vlan_features |= NETIF_F_FCOE_MTU;
7737 }
eacd73f7 7738#endif /* IXGBE_FCOE */
7b872a55 7739 if (pci_using_dac) {
9a799d71 7740 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 7741 netdev->vlan_features |= NETIF_F_HIGHDMA;
7742 }
9a799d71 7743
082757af
DS		 7744 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
7745 netdev->hw_features |= NETIF_F_LRO;
0c19d6af 7746 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
f8212f97
AD		 7747 netdev->features |= NETIF_F_LRO;
7748
9a799d71 7749 /* make sure the EEPROM is good */
c44ade9e 7750 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
849c4542 7751 e_dev_err("The EEPROM Checksum Is Not Valid\n");
9a799d71
AK		 7752 err = -EIO;
7753 goto err_eeprom;
7754 }
7755
7756 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
7757 memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len);
7758
c44ade9e 7759 if (ixgbe_validate_mac_addr(netdev->perm_addr)) {
849c4542 7760 e_dev_err("invalid MAC address\n");
9a799d71
AK		 7761 err = -EIO;
7762 goto err_eeprom;
7763 }
7764
7086400d
AD		 7765 setup_timer(&adapter->service_timer, &ixgbe_service_timer,
7766 (unsigned long) adapter);
9a799d71 7767
7086400d
AD		 7768 INIT_WORK(&adapter->service_task, ixgbe_service_task);
7769 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
9a799d71 7770
021230d4
AV		 7771 err = ixgbe_init_interrupt_scheme(adapter);
7772 if (err)
7773 goto err_sw_init;
9a799d71 7774
082757af
DS		 7775 if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
7776 netdev->hw_features &= ~NETIF_F_RXHASH;
67a74ee2 7777 netdev->features &= ~NETIF_F_RXHASH;
082757af 7778 }
67a74ee2 7779
c23f5b6b
ET		 7780 /* WOL not supported for all but the following */
7781 adapter->wol = 0;
e8e26350 7782 switch (pdev->device) {
0b077fea 7783 case IXGBE_DEV_ID_82599_SFP:
0e22d043
DS		 7784 /* Only these subdevice supports WOL */
7785 switch (pdev->subsystem_device) {
7786 case IXGBE_SUBDEV_ID_82599_560FLR:
7787 /* only support first port */
7788 if (hw->bus.func != 0)
7789 break;
7790 case IXGBE_SUBDEV_ID_82599_SFP:
9417c464 7791 adapter->wol = IXGBE_WUFC_MAG;
0e22d043
DS		 7792 break;
7793 }
0b077fea 7794 break;
50d6c681
AD		 7795 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
7796 /* All except this subdevice support WOL */
0b077fea 7797 if (pdev->subsystem_device != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
9417c464 7798 adapter->wol = IXGBE_WUFC_MAG;
0b077fea 7799 break;
e8e26350 7800 case IXGBE_DEV_ID_82599_KX4:
9417c464 7801 adapter->wol = IXGBE_WUFC_MAG;
e8e26350 7802 break;
c23f5b6b
ET		 7803 case IXGBE_DEV_ID_X540T:
7804 /* Check eeprom to see if it is enabled */
7805 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
7806 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
7807
7808 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
7809 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
7810 (hw->bus.func == 0)))
7811 adapter->wol = IXGBE_WUFC_MAG;
e8e26350
PW		 7812 break;
7813 }
e8e26350
PW		 7814 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
7815
15e5209f
ET		 7816 /* save off EEPROM version number */
7817 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh);
7818 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl);
7819
04f165ef
PW		 7820 /* pick up the PCI bus settings for reporting later */
7821 hw->mac.ops.get_bus_info(hw);
7822
9a799d71 7823 /* print bus type/speed/width info */
849c4542 7824 e_dev_info("(PCI Express:%s:%s) %pM\n",
6716344c
DS		 7825 (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" :
7826 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" :
e8e9f696
JP		 7827 "Unknown"),
7828 (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
7829 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
7830 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" :
7831 "Unknown"),
7832 netdev->dev_addr);
289700db
DS		 7833
7834 err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH);
7835 if (err)
9fe93afd 7836 strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH);
e8e26350 7837 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
289700db 7838 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
849c4542 7839 hw->mac.type, hw->phy.type, hw->phy.sfp_type,
289700db 7840 part_str);
e8e26350 7841 else
289700db
DS		 7842 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
7843 hw->mac.type, hw->phy.type, part_str);
9a799d71 7844
e8e26350 7845 if (hw->bus.width <= ixgbe_bus_width_pcie_x4) {
849c4542
ET		 7846 e_dev_warn("PCI-Express bandwidth available for this card is "
7847 "not sufficient for optimal performance.\n");
7848 e_dev_warn("For optimal performance a x8 PCI-Express slot "
7849 "is required.\n");
0c254d86
AK		 7850 }
7851
9a799d71 7852 /* reset the hardware with the new settings */
794caeb2 7853 err = hw->mac.ops.start_hw(hw);
c44ade9e 7854
794caeb2
PWJ		 7855 if (err == IXGBE_ERR_EEPROM_VERSION) {
7856 /* We are running on a pre-production device, log a warning */
849c4542
ET		 7857 e_dev_warn("This device is a pre-production adapter/LOM. "
7858 "Please be aware there may be issues associated "
7859 "with your hardware. If you are experiencing "
7860 "problems please contact your Intel or hardware "
7861 "representative who provided you with this "
7862 "hardware.\n");
794caeb2 7863 }
9a799d71
AK		 7864 strcpy(netdev->name, "eth%d");
7865 err = register_netdev(netdev);
7866 if (err)
7867 goto err_register;
7868
93d3ce8f
ET		 7869 /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
7870 if (hw->mac.ops.disable_tx_laser &&
7871 ((hw->phy.multispeed_fiber) ||
7872 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
7873 (hw->mac.type == ixgbe_mac_82599EB))))
7874 hw->mac.ops.disable_tx_laser(hw);
7875
54386467
JB		 7876 /* carrier off reporting is important to ethtool even BEFORE open */
7877 netif_carrier_off(netdev);
7878
5dd2d332 7879#ifdef CONFIG_IXGBE_DCA
652f093f 7880 if (dca_add_requester(&pdev->dev) == 0) {
bd0362dd 7881 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
bd0362dd
JC		 7882 ixgbe_setup_dca(adapter);
7883 }
7884#endif
1cdd1ec8 7885 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
396e799c 7886 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
1cdd1ec8
GR		 7887 for (i = 0; i < adapter->num_vfs; i++)
7888 ixgbe_vf_configuration(pdev, (i | 0x10000000));
7889 }
7890
2466dd9c
JK		 7891 /* firmware requires driver version to be 0xFFFFFFFF
7892 * since os does not support feature
7893 */
9612de92 7894 if (hw->mac.ops.set_fw_drv_ver)
2466dd9c
JK		 7895 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF,
7896 0xFF);
9612de92 7897
0365e6e4
PW		 7898 /* add san mac addr to netdev */
7899 ixgbe_add_sanmac_netdev(netdev);
9a799d71 7900
ea81875a 7901 e_dev_info("%s\n", ixgbe_default_device_descr);
9a799d71
AK		 7902 cards_found++;
7903 return 0;
7904
7905err_register:
5eba3699 7906 ixgbe_release_hw_control(adapter);
7a921c93 7907 ixgbe_clear_interrupt_scheme(adapter);
9a799d71
AK		 7908err_sw_init:
7909err_eeprom:
1cdd1ec8
GR		 7910 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7911 ixgbe_disable_sriov(adapter);
7086400d 7912 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
9a799d71
AK		 7913 iounmap(hw->hw_addr);
7914err_ioremap:
7915 free_netdev(netdev);
7916err_alloc_etherdev:
e8e9f696
JP		 7917 pci_release_selected_regions(pdev,
7918 pci_select_bars(pdev, IORESOURCE_MEM));
9a799d71
AK		 7919err_pci_reg:
7920err_dma:
7921 pci_disable_device(pdev);
7922 return err;
7923}
7924
7925/**
7926 * ixgbe_remove - Device Removal Routine
7927 * @pdev: PCI device information struct
7928 *
7929 * ixgbe_remove is called by the PCI subsystem to alert the driver
7930 * that it should release a PCI device. The could be caused by a
7931 * Hot-Plug event, or because the driver is going to be removed from
7932 * memory.
7933 **/
7934static void __devexit ixgbe_remove(struct pci_dev *pdev)
7935{
c60fbb00
AD		 7936 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7937 struct net_device *netdev = adapter->netdev;
9a799d71
AK		 7938
7939 set_bit(__IXGBE_DOWN, &adapter->state);
7086400d 7940 cancel_work_sync(&adapter->service_task);
9a799d71 7941
5dd2d332 7942#ifdef CONFIG_IXGBE_DCA
bd0362dd
JC		 7943 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
7944 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
7945 dca_remove_requester(&pdev->dev);
7946 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
7947 }
7948
7949#endif
332d4a7d
YZ		 7950#ifdef IXGBE_FCOE
7951 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
7952 ixgbe_cleanup_fcoe(adapter);
7953
7954#endif /* IXGBE_FCOE */
0365e6e4
PW		 7955
7956 /* remove the added san mac */
7957 ixgbe_del_sanmac_netdev(netdev);
7958
c4900be0
DS		 7959 if (netdev->reg_state == NETREG_REGISTERED)
7960 unregister_netdev(netdev);
9a799d71 7961
c6bda30a
GR		 7962 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
7963 if (!(ixgbe_check_vf_assignment(adapter)))
7964 ixgbe_disable_sriov(adapter);
7965 else
7966 e_dev_warn("Unloading driver while VFs are assigned "
7967 "- VFs will not be deallocated\n");
7968 }
1cdd1ec8 7969
7a921c93 7970 ixgbe_clear_interrupt_scheme(adapter);
5eba3699 7971
021230d4 7972 ixgbe_release_hw_control(adapter);
9a799d71
AK		 7973
7974 iounmap(adapter->hw.hw_addr);
9ce77666 7975 pci_release_selected_regions(pdev, pci_select_bars(pdev,
e8e9f696 7976 IORESOURCE_MEM));
9a799d71 7977
849c4542 7978 e_dev_info("complete\n");
021230d4 7979
9a799d71
AK		 7980 free_netdev(netdev);
7981
19d5afd4 7982 pci_disable_pcie_error_reporting(pdev);
6fabd715 7983
9a799d71
AK		 7984 pci_disable_device(pdev);
7985}
7986
7987/**
7988 * ixgbe_io_error_detected - called when PCI error is detected
7989 * @pdev: Pointer to PCI device
7990 * @state: The current pci connection state
7991 *
7992 * This function is called after a PCI bus error affecting
7993 * this device has been detected.
7994 */
7995static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
e8e9f696 7996 pci_channel_state_t state)
9a799d71 7997{
c60fbb00
AD		 7998 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7999 struct net_device *netdev = adapter->netdev;
9a799d71 8000
83c61fa9
GR		 8001#ifdef CONFIG_PCI_IOV
8002 struct pci_dev *bdev, *vfdev;
8003 u32 dw0, dw1, dw2, dw3;
8004 int vf, pos;
8005 u16 req_id, pf_func;
8006
8007 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
8008 adapter->num_vfs == 0)
8009 goto skip_bad_vf_detection;
8010
8011 bdev = pdev->bus->self;
8012 while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT))
8013 bdev = bdev->bus->self;
8014
8015 if (!bdev)
8016 goto skip_bad_vf_detection;
8017
8018 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
8019 if (!pos)
8020 goto skip_bad_vf_detection;
8021
8022 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
8023 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
8024 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
8025 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
8026
8027 req_id = dw1 >> 16;
8028 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
8029 if (!(req_id & 0x0080))
8030 goto skip_bad_vf_detection;
8031
8032 pf_func = req_id & 0x01;
8033 if ((pf_func & 1) == (pdev->devfn & 1)) {
8034 unsigned int device_id;
8035
8036 vf = (req_id & 0x7F) >> 1;
8037 e_dev_err("VF %d has caused a PCIe error\n", vf);
8038 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
8039 "%8.8x\tdw3: %8.8x\n",
8040 dw0, dw1, dw2, dw3);
8041 switch (adapter->hw.mac.type) {
8042 case ixgbe_mac_82599EB:
8043 device_id = IXGBE_82599_VF_DEVICE_ID;
8044 break;
8045 case ixgbe_mac_X540:
8046 device_id = IXGBE_X540_VF_DEVICE_ID;
8047 break;
8048 default:
8049 device_id = 0;
8050 break;
8051 }
8052
8053 /* Find the pci device of the offending VF */
8054 vfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID, device_id, NULL);
8055 while (vfdev) {
8056 if (vfdev->devfn == (req_id & 0xFF))
8057 break;
8058 vfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID,
8059 device_id, vfdev);
8060 }
8061 /*
8062 * There's a slim chance the VF could have been hot plugged,
8063 * so if it is no longer present we don't need to issue the
8064 * VFLR. Just clean up the AER in that case.
8065 */
8066 if (vfdev) {
8067 e_dev_err("Issuing VFLR to VF %d\n", vf);
8068 pci_write_config_dword(vfdev, 0xA8, 0x00008000);
8069 }
8070
8071 pci_cleanup_aer_uncorrect_error_status(pdev);
8072 }
8073
8074 /*
8075 * Even though the error may have occurred on the other port
8076 * we still need to increment the vf error reference count for
8077 * both ports because the I/O resume function will be called
8078 * for both of them.
8079 */
8080 adapter->vferr_refcount++;
8081
8082 return PCI_ERS_RESULT_RECOVERED;
8083
8084skip_bad_vf_detection:
8085#endif /* CONFIG_PCI_IOV */
9a799d71
AK		 8086 netif_device_detach(netdev);
8087
3044b8d1
BL		 8088 if (state == pci_channel_io_perm_failure)
8089 return PCI_ERS_RESULT_DISCONNECT;
8090
9a799d71
AK		 8091 if (netif_running(netdev))
8092 ixgbe_down(adapter);
8093 pci_disable_device(pdev);
8094
b4617240 8095 /* Request a slot reset. */
9a799d71
AK		 8096 return PCI_ERS_RESULT_NEED_RESET;
8097}
8098
8099/**
8100 * ixgbe_io_slot_reset - called after the pci bus has been reset.
8101 * @pdev: Pointer to PCI device
8102 *
8103 * Restart the card from scratch, as if from a cold-boot.
8104 */
8105static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
8106{
c60fbb00 8107 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
6fabd715
PWJ		 8108 pci_ers_result_t result;
8109 int err;
9a799d71 8110
9ce77666 8111 if (pci_enable_device_mem(pdev)) {
396e799c 8112 e_err(probe, "Cannot re-enable PCI device after reset.\n");
6fabd715
PWJ		 8113 result = PCI_ERS_RESULT_DISCONNECT;
8114 } else {
8115 pci_set_master(pdev);
8116 pci_restore_state(pdev);
c0e1f68b 8117 pci_save_state(pdev);
9a799d71 8118
dd4d8ca6 8119 pci_wake_from_d3(pdev, false);
9a799d71 8120
6fabd715 8121 ixgbe_reset(adapter);
88512539 8122 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
6fabd715
PWJ		 8123 result = PCI_ERS_RESULT_RECOVERED;
8124 }
8125
8126 err = pci_cleanup_aer_uncorrect_error_status(pdev);
8127 if (err) {
849c4542
ET		 8128 e_dev_err("pci_cleanup_aer_uncorrect_error_status "
8129 "failed 0x%0x\n", err);
6fabd715
PWJ		 8130 /* non-fatal, continue */
8131 }
9a799d71 8132
6fabd715 8133 return result;
9a799d71
AK		 8134}
8135
8136/**
8137 * ixgbe_io_resume - called when traffic can start flowing again.
8138 * @pdev: Pointer to PCI device
8139 *
8140 * This callback is called when the error recovery driver tells us that
8141 * its OK to resume normal operation.
8142 */
8143static void ixgbe_io_resume(struct pci_dev *pdev)
8144{
c60fbb00
AD		 8145 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
8146 struct net_device *netdev = adapter->netdev;
9a799d71 8147
83c61fa9
GR		 8148#ifdef CONFIG_PCI_IOV
8149 if (adapter->vferr_refcount) {
8150 e_info(drv, "Resuming after VF err\n");
8151 adapter->vferr_refcount--;
8152 return;
8153 }
8154
8155#endif
c7ccde0f
AD		 8156 if (netif_running(netdev))
8157 ixgbe_up(adapter);
9a799d71
AK		 8158
8159 netif_device_attach(netdev);
9a799d71
AK		 8160}
8161
8162static struct pci_error_handlers ixgbe_err_handler = {
8163 .error_detected = ixgbe_io_error_detected,
8164 .slot_reset = ixgbe_io_slot_reset,
8165 .resume = ixgbe_io_resume,
8166};
8167
8168static struct pci_driver ixgbe_driver = {
8169 .name = ixgbe_driver_name,
8170 .id_table = ixgbe_pci_tbl,
8171 .probe = ixgbe_probe,
8172 .remove = __devexit_p(ixgbe_remove),
8173#ifdef CONFIG_PM
8174 .suspend = ixgbe_suspend,
8175 .resume = ixgbe_resume,
8176#endif
8177 .shutdown = ixgbe_shutdown,
8178 .err_handler = &ixgbe_err_handler
8179};
8180
8181/**
8182 * ixgbe_init_module - Driver Registration Routine
8183 *
8184 * ixgbe_init_module is the first routine called when the driver is
8185 * loaded. All it does is register with the PCI subsystem.
8186 **/
8187static int __init ixgbe_init_module(void)
8188{
8189 int ret;
c7689578 8190 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
849c4542 8191 pr_info("%s\n", ixgbe_copyright);
9a799d71 8192
5dd2d332 8193#ifdef CONFIG_IXGBE_DCA
bd0362dd 8194 dca_register_notify(&dca_notifier);
bd0362dd 8195#endif
5dd2d332 8196
9a799d71
AK		 8197 ret = pci_register_driver(&ixgbe_driver);
8198 return ret;
8199}
b4617240 8200
9a799d71
AK		 8201module_init(ixgbe_init_module);
8202
8203/**
8204 * ixgbe_exit_module - Driver Exit Cleanup Routine
8205 *
8206 * ixgbe_exit_module is called just before the driver is removed
8207 * from memory.
8208 **/
8209static void __exit ixgbe_exit_module(void)
8210{
5dd2d332 8211#ifdef CONFIG_IXGBE_DCA
bd0362dd
JC		 8212 dca_unregister_notify(&dca_notifier);
8213#endif
9a799d71 8214 pci_unregister_driver(&ixgbe_driver);
1a51502b 8215 rcu_barrier(); /* Wait for completion of call_rcu()'s */
9a799d71 8216}
bd0362dd 8217
5dd2d332 8218#ifdef CONFIG_IXGBE_DCA
bd0362dd 8219static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
e8e9f696 8220 void *p)
bd0362dd
JC		 8221{
8222 int ret_val;
8223
8224 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
e8e9f696 8225 __ixgbe_notify_dca);
bd0362dd
JC		 8226
8227 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
8228}
b453368d 8229
5dd2d332 8230#endif /* CONFIG_IXGBE_DCA */
849c4542 8231
9a799d71
AK		 8232module_exit(ixgbe_exit_module);
8233
8234/* ixgbe_main.c */
Linux 6.x block layer and io_uring tree(s)