projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4/iw_cxgb4: Doorbell Drop Avoidance Bug Fixes
[linux-2.6-block.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
b8ff05a9
DM		 64#include <asm/uaccess.h>
65
66#include "cxgb4.h"
67#include "t4_regs.h"
68#include "t4_msg.h"
69#include "t4fw_api.h"
70#include "l2t.h"
71
01bcca68
VP		 72#include <../drivers/net/bonding/bonding.h>
73
74#ifdef DRV_VERSION
75#undef DRV_VERSION
76#endif
3a7f8554
SR		 77#define DRV_VERSION "2.0.0-ko"
78#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9
DM		 79
80/*
81 * Max interrupt hold-off timer value in us. Queues fall back to this value
82 * under extreme memory pressure so it's largish to give the system time to
83 * recover.
84 */
85#define MAX_SGE_TIMERVAL 200U
86
7ee9ff94 87enum {
13ee15d3
VP		 88 /*
89 * Physical Function provisioning constants.
90 */
91 PFRES_NVI = 4, /* # of Virtual Interfaces */
92 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */
93 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr
94 */
95 PFRES_NEQ = 256, /* # of egress queues */
96 PFRES_NIQ = 0, /* # of ingress queues */
97 PFRES_TC = 0, /* PCI-E traffic class */
98 PFRES_NEXACTF = 128, /* # of exact MPS filters */
99
100 PFRES_R_CAPS = FW_CMD_CAP_PF,
101 PFRES_WX_CAPS = FW_CMD_CAP_PF,
102
103#ifdef CONFIG_PCI_IOV
104 /*
105 * Virtual Function provisioning constants. We need two extra Ingress
106 * Queues with Interrupt capability to serve as the VF's Firmware
107 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
108 * neither will have Free Lists associated with them). For each
109 * Ethernet/Control Egress Queue and for each Free List, we need an
110 * Egress Context.
111 */
7ee9ff94
CL		 112 VFRES_NPORTS = 1, /* # of "ports" per VF */
113 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
114
115 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
116 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
117 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
7ee9ff94 118 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
13ee15d3 119 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
7ee9ff94
CL		 120 VFRES_TC = 0, /* PCI-E traffic class */
121 VFRES_NEXACTF = 16, /* # of exact MPS filters */
122
123 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
124 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
13ee15d3 125#endif
7ee9ff94
CL		 126};
127
128/*
129 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
130 * static and likely not to be useful in the long run. We really need to
131 * implement some form of persistent configuration which the firmware
132 * controls.
133 */
134static unsigned int pfvfres_pmask(struct adapter *adapter,
135 unsigned int pf, unsigned int vf)
136{
137 unsigned int portn, portvec;
138
139 /*
140 * Give PF's access to all of the ports.
141 */
142 if (vf == 0)
143 return FW_PFVF_CMD_PMASK_MASK;
144
145 /*
146 * For VFs, we'll assign them access to the ports based purely on the
147 * PF. We assign active ports in order, wrapping around if there are
148 * fewer active ports than PFs: e.g. active port[pf % nports].
149 * Unfortunately the adapter's port_info structs haven't been
150 * initialized yet so we have to compute this.
151 */
152 if (adapter->params.nports == 0)
153 return 0;
154
155 portn = pf % adapter->params.nports;
156 portvec = adapter->params.portvec;
157 for (;;) {
158 /*
159 * Isolate the lowest set bit in the port vector. If we're at
160 * the port number that we want, return that as the pmask.
161 * otherwise mask that bit out of the port vector and
162 * decrement our port number ...
163 */
164 unsigned int pmask = portvec ^ (portvec & (portvec-1));
165 if (portn == 0)
166 return pmask;
167 portn--;
168 portvec &= ~pmask;
169 }
170 /*NOTREACHED*/
171}
7ee9ff94 172
b8ff05a9
DM		 173enum {
174 MAX_TXQ_ENTRIES = 16384,
175 MAX_CTRL_TXQ_ENTRIES = 1024,
176 MAX_RSPQ_ENTRIES = 16384,
177 MAX_RX_BUFFERS = 16384,
178 MIN_TXQ_ENTRIES = 32,
179 MIN_CTRL_TXQ_ENTRIES = 32,
180 MIN_RSPQ_ENTRIES = 128,
181 MIN_FL_ENTRIES = 16
182};
183
f2b7e78d
VP		 184/* Host shadow copy of ingress filter entry. This is in host native format
185 * and doesn't match the ordering or bit order, etc. of the hardware of the
186 * firmware command. The use of bit-field structure elements is purely to
187 * remind ourselves of the field size limitations and save memory in the case
188 * where the filter table is large.
189 */
190struct filter_entry {
191 /* Administrative fields for filter.
192 */
193 u32 valid:1; /* filter allocated and valid */
194 u32 locked:1; /* filter is administratively locked */
195
196 u32 pending:1; /* filter action is pending firmware reply */
197 u32 smtidx:8; /* Source MAC Table index for smac */
198 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
199
200 /* The filter itself. Most of this is a straight copy of information
201 * provided by the extended ioctl(). Some fields are translated to
202 * internal forms -- for instance the Ingress Queue ID passed in from
203 * the ioctl() is translated into the Absolute Ingress Queue ID.
204 */
205 struct ch_filter_specification fs;
206};
207
b8ff05a9
DM		 208#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
209 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
210 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
211
060e0c75 212#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
b8ff05a9
DM		 213
214static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
060e0c75 215 CH_DEVICE(0xa000, 0), /* PE10K */
ccea790e
DM		 216 CH_DEVICE(0x4001, -1),
217 CH_DEVICE(0x4002, -1),
218 CH_DEVICE(0x4003, -1),
219 CH_DEVICE(0x4004, -1),
220 CH_DEVICE(0x4005, -1),
221 CH_DEVICE(0x4006, -1),
222 CH_DEVICE(0x4007, -1),
223 CH_DEVICE(0x4008, -1),
224 CH_DEVICE(0x4009, -1),
225 CH_DEVICE(0x400a, -1),
226 CH_DEVICE(0x4401, 4),
227 CH_DEVICE(0x4402, 4),
228 CH_DEVICE(0x4403, 4),
229 CH_DEVICE(0x4404, 4),
230 CH_DEVICE(0x4405, 4),
231 CH_DEVICE(0x4406, 4),
232 CH_DEVICE(0x4407, 4),
233 CH_DEVICE(0x4408, 4),
234 CH_DEVICE(0x4409, 4),
235 CH_DEVICE(0x440a, 4),
f637d577
VP		 236 CH_DEVICE(0x440d, 4),
237 CH_DEVICE(0x440e, 4),
9ef603a0
VP		 238 CH_DEVICE(0x5001, 4),
239 CH_DEVICE(0x5002, 4),
240 CH_DEVICE(0x5003, 4),
241 CH_DEVICE(0x5004, 4),
242 CH_DEVICE(0x5005, 4),
243 CH_DEVICE(0x5006, 4),
244 CH_DEVICE(0x5007, 4),
245 CH_DEVICE(0x5008, 4),
246 CH_DEVICE(0x5009, 4),
247 CH_DEVICE(0x500A, 4),
248 CH_DEVICE(0x500B, 4),
249 CH_DEVICE(0x500C, 4),
250 CH_DEVICE(0x500D, 4),
251 CH_DEVICE(0x500E, 4),
252 CH_DEVICE(0x500F, 4),
253 CH_DEVICE(0x5010, 4),
254 CH_DEVICE(0x5011, 4),
255 CH_DEVICE(0x5012, 4),
256 CH_DEVICE(0x5013, 4),
f0a8e6de
HS		 257 CH_DEVICE(0x5014, 4),
258 CH_DEVICE(0x5015, 4),
9ef603a0
VP		 259 CH_DEVICE(0x5401, 4),
260 CH_DEVICE(0x5402, 4),
261 CH_DEVICE(0x5403, 4),
262 CH_DEVICE(0x5404, 4),
263 CH_DEVICE(0x5405, 4),
264 CH_DEVICE(0x5406, 4),
265 CH_DEVICE(0x5407, 4),
266 CH_DEVICE(0x5408, 4),
267 CH_DEVICE(0x5409, 4),
268 CH_DEVICE(0x540A, 4),
269 CH_DEVICE(0x540B, 4),
270 CH_DEVICE(0x540C, 4),
271 CH_DEVICE(0x540D, 4),
272 CH_DEVICE(0x540E, 4),
273 CH_DEVICE(0x540F, 4),
274 CH_DEVICE(0x5410, 4),
275 CH_DEVICE(0x5411, 4),
276 CH_DEVICE(0x5412, 4),
277 CH_DEVICE(0x5413, 4),
f0a8e6de
HS		 278 CH_DEVICE(0x5414, 4),
279 CH_DEVICE(0x5415, 4),
b8ff05a9
DM		 280 { 0, }
281};
282
16e47624 283#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 284#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 285#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 286#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 287
288MODULE_DESCRIPTION(DRV_DESC);
289MODULE_AUTHOR("Chelsio Communications");
290MODULE_LICENSE("Dual BSD/GPL");
291MODULE_VERSION(DRV_VERSION);
292MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 293MODULE_FIRMWARE(FW4_FNAME);
0a57a536 294MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 295
636f9d37
VP		 296/*
297 * Normally we're willing to become the firmware's Master PF but will be happy
298 * if another PF has already become the Master and initialized the adapter.
299 * Setting "force_init" will cause this driver to forcibly establish itself as
300 * the Master PF and initialize the adapter.
301 */
302static uint force_init;
303
304module_param(force_init, uint, 0644);
305MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
306
13ee15d3
VP		 307/*
308 * Normally if the firmware we connect to has Configuration File support, we
309 * use that and only fall back to the old Driver-based initialization if the
310 * Configuration File fails for some reason. If force_old_init is set, then
311 * we'll always use the old Driver-based initialization sequence.
312 */
313static uint force_old_init;
314
315module_param(force_old_init, uint, 0644);
316MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
317
b8ff05a9
DM		 318static int dflt_msg_enable = DFLT_MSG_ENABLE;
319
320module_param(dflt_msg_enable, int, 0644);
321MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
322
323/*
324 * The driver uses the best interrupt scheme available on a platform in the
325 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
326 * of these schemes the driver may consider as follows:
327 *
328 * msi = 2: choose from among all three options
329 * msi = 1: only consider MSI and INTx interrupts
330 * msi = 0: force INTx interrupts
331 */
332static int msi = 2;
333
334module_param(msi, int, 0644);
335MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
336
337/*
338 * Queue interrupt hold-off timer values. Queues default to the first of these
339 * upon creation.
340 */
341static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
342
343module_param_array(intr_holdoff, uint, NULL, 0644);
344MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
345 "0..4 in microseconds");
346
347static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
348
349module_param_array(intr_cnt, uint, NULL, 0644);
350MODULE_PARM_DESC(intr_cnt,
351 "thresholds 1..3 for queue interrupt packet counters");
352
636f9d37
VP		 353/*
354 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
355 * offset by 2 bytes in order to have the IP headers line up on 4-byte
356 * boundaries. This is a requirement for many architectures which will throw
357 * a machine check fault if an attempt is made to access one of the 4-byte IP
358 * header fields on a non-4-byte boundary. And it's a major performance issue
359 * even on some architectures which allow it like some implementations of the
360 * x86 ISA. However, some architectures don't mind this and for some very
361 * edge-case performance sensitive applications (like forwarding large volumes
362 * of small packets), setting this DMA offset to 0 will decrease the number of
363 * PCI-E Bus transfers enough to measurably affect performance.
364 */
365static int rx_dma_offset = 2;
366
eb939922 367static bool vf_acls;
b8ff05a9
DM		 368
369#ifdef CONFIG_PCI_IOV
370module_param(vf_acls, bool, 0644);
371MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
372
7d6727cf
SR		 373/* Configure the number of PCI-E Virtual Function which are to be instantiated
374 * on SR-IOV Capable Physical Functions.
0a57a536 375 */
7d6727cf 376static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 377
378module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 379MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 380#endif
381
13ee15d3
VP		 382/*
383 * The filter TCAM has a fixed portion and a variable portion. The fixed
384 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
385 * ports. The variable portion is 36 bits which can include things like Exact
386 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
387 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
388 * far exceed the 36-bit budget for this "compressed" header portion of the
389 * filter. Thus, we have a scarce resource which must be carefully managed.
390 *
391 * By default we set this up to mostly match the set of filter matching
392 * capabilities of T3 but with accommodations for some of T4's more
393 * interesting features:
394 *
395 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
396 * [Inner] VLAN (17), Port (3), FCoE (1) }
397 */
398enum {
399 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
400 TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
401 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
402};
403
404static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
405
f2b7e78d
VP		 406module_param(tp_vlan_pri_map, uint, 0644);
407MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
408
b8ff05a9
DM		 409static struct dentry *cxgb4_debugfs_root;
410
411static LIST_HEAD(adapter_list);
412static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 413/* Adapter list to be accessed from atomic context */
414static LIST_HEAD(adap_rcu_list);
415static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 416static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
417static const char *uld_str[] = { "RDMA", "iSCSI" };
418
419static void link_report(struct net_device *dev)
420{
421 if (!netif_carrier_ok(dev))
422 netdev_info(dev, "link down\n");
423 else {
424 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
425
426 const char *s = "10Mbps";
427 const struct port_info *p = netdev_priv(dev);
428
429 switch (p->link_cfg.speed) {
e8b39015 430 case 10000:
b8ff05a9
DM		 431 s = "10Gbps";
432 break;
e8b39015 433 case 1000:
b8ff05a9
DM		 434 s = "1000Mbps";
435 break;
e8b39015 436 case 100:
b8ff05a9
DM		 437 s = "100Mbps";
438 break;
e8b39015 439 case 40000:
72aca4bf
KS		 440 s = "40Gbps";
441 break;
b8ff05a9
DM		 442 }
443
444 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
445 fc[p->link_cfg.fc]);
446 }
447}
448
449void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
450{
451 struct net_device *dev = adapter->port[port_id];
452
453 /* Skip changes from disabled ports. */
454 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
455 if (link_stat)
456 netif_carrier_on(dev);
457 else
458 netif_carrier_off(dev);
459
460 link_report(dev);
461 }
462}
463
464void t4_os_portmod_changed(const struct adapter *adap, int port_id)
465{
466 static const char *mod_str[] = {
a0881cab 467 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 468 };
469
470 const struct net_device *dev = adap->port[port_id];
471 const struct port_info *pi = netdev_priv(dev);
472
473 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
474 netdev_info(dev, "port module unplugged\n");
a0881cab 475 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 476 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
477}
478
479/*
480 * Configure the exact and hash address filters to handle a port's multicast
481 * and secondary unicast MAC addresses.
482 */
483static int set_addr_filters(const struct net_device *dev, bool sleep)
484{
485 u64 mhash = 0;
486 u64 uhash = 0;
487 bool free = true;
488 u16 filt_idx[7];
489 const u8 *addr[7];
490 int ret, naddr = 0;
b8ff05a9
DM		 491 const struct netdev_hw_addr *ha;
492 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 493 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 494 const struct port_info *pi = netdev_priv(dev);
060e0c75 495 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 496
497 /* first do the secondary unicast addresses */
498 netdev_for_each_uc_addr(ha, dev) {
499 addr[naddr++] = ha->addr;
500 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 501 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 502 naddr, addr, filt_idx, &uhash, sleep);
503 if (ret < 0)
504 return ret;
505
506 free = false;
507 naddr = 0;
508 }
509 }
510
511 /* next set up the multicast addresses */
4a35ecf8
DM		 512 netdev_for_each_mc_addr(ha, dev) {
513 addr[naddr++] = ha->addr;
514 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 515 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 516 naddr, addr, filt_idx, &mhash, sleep);
517 if (ret < 0)
518 return ret;
519
520 free = false;
521 naddr = 0;
522 }
523 }
524
060e0c75 525 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 526 uhash | mhash, sleep);
527}
528
3069ee9b
VP		 529int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
530module_param(dbfifo_int_thresh, int, 0644);
531MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
532
404d9e3f
VP		 533/*
534 * usecs to sleep while draining the dbfifo
535 */
536static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 537module_param(dbfifo_drain_delay, int, 0644);
538MODULE_PARM_DESC(dbfifo_drain_delay,
539 "usecs to sleep while draining the dbfifo");
540
b8ff05a9
DM		 541/*
542 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
543 * If @mtu is -1 it is left unchanged.
544 */
545static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
546{
547 int ret;
548 struct port_info *pi = netdev_priv(dev);
549
550 ret = set_addr_filters(dev, sleep_ok);
551 if (ret == 0)
060e0c75 552 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 553 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 554 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 555 sleep_ok);
556 return ret;
557}
558
3069ee9b
VP		 559static struct workqueue_struct *workq;
560
b8ff05a9
DM		 561/**
562 * link_start - enable a port
563 * @dev: the port to enable
564 *
565 * Performs the MAC and PHY actions needed to enable a port.
566 */
567static int link_start(struct net_device *dev)
568{
569 int ret;
570 struct port_info *pi = netdev_priv(dev);
060e0c75 571 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 572
573 /*
574 * We do not set address filters and promiscuity here, the stack does
575 * that step explicitly.
576 */
060e0c75 577 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 578 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 579 if (ret == 0) {
060e0c75 580 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 581 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 582 true);
b8ff05a9
DM		 583 if (ret >= 0) {
584 pi->xact_addr_filt = ret;
585 ret = 0;
586 }
587 }
588 if (ret == 0)
060e0c75
DM		 589 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
590 &pi->link_cfg);
b8ff05a9 591 if (ret == 0)
060e0c75 592 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
b8ff05a9
DM		 593 return ret;
594}
595
f2b7e78d
VP		 596/* Clear a filter and release any of its resources that we own. This also
597 * clears the filter's "pending" status.
598 */
599static void clear_filter(struct adapter *adap, struct filter_entry *f)
600{
601 /* If the new or old filter have loopback rewriteing rules then we'll
602 * need to free any existing Layer Two Table (L2T) entries of the old
603 * filter rule. The firmware will handle freeing up any Source MAC
604 * Table (SMT) entries used for rewriting Source MAC Addresses in
605 * loopback rules.
606 */
607 if (f->l2t)
608 cxgb4_l2t_release(f->l2t);
609
610 /* The zeroing of the filter rule below clears the filter valid,
611 * pending, locked flags, l2t pointer, etc. so it's all we need for
612 * this operation.
613 */
614 memset(f, 0, sizeof(*f));
615}
616
617/* Handle a filter write/deletion reply.
618 */
619static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
620{
621 unsigned int idx = GET_TID(rpl);
622 unsigned int nidx = idx - adap->tids.ftid_base;
623 unsigned int ret;
624 struct filter_entry *f;
625
626 if (idx >= adap->tids.ftid_base && nidx <
627 (adap->tids.nftids + adap->tids.nsftids)) {
628 idx = nidx;
629 ret = GET_TCB_COOKIE(rpl->cookie);
630 f = &adap->tids.ftid_tab[idx];
631
632 if (ret == FW_FILTER_WR_FLT_DELETED) {
633 /* Clear the filter when we get confirmation from the
634 * hardware that the filter has been deleted.
635 */
636 clear_filter(adap, f);
637 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
638 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
639 idx);
640 clear_filter(adap, f);
641 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
642 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
643 f->pending = 0; /* asynchronous setup completed */
644 f->valid = 1;
645 } else {
646 /* Something went wrong. Issue a warning about the
647 * problem and clear everything out.
648 */
649 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
650 idx, ret);
651 clear_filter(adap, f);
652 }
653 }
654}
655
656/* Response queue handler for the FW event queue.
b8ff05a9
DM		 657 */
658static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
659 const struct pkt_gl *gl)
660{
661 u8 opcode = ((const struct rss_header *)rsp)->opcode;
662
663 rsp++; /* skip RSS header */
b407a4a9
VP		 664
665 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
666 */
667 if (unlikely(opcode == CPL_FW4_MSG &&
668 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
669 rsp++;
670 opcode = ((const struct rss_header *)rsp)->opcode;
671 rsp++;
672 if (opcode != CPL_SGE_EGR_UPDATE) {
673 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
674 , opcode);
675 goto out;
676 }
677 }
678
b8ff05a9
DM		 679 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
680 const struct cpl_sge_egr_update *p = (void *)rsp;
681 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 682 struct sge_txq *txq;
b8ff05a9 683
e46dab4d 684 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 685 txq->restarts++;
e46dab4d 686 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 687 struct sge_eth_txq *eq;
688
689 eq = container_of(txq, struct sge_eth_txq, q);
690 netif_tx_wake_queue(eq->txq);
691 } else {
692 struct sge_ofld_txq *oq;
693
694 oq = container_of(txq, struct sge_ofld_txq, q);
695 tasklet_schedule(&oq->qresume_tsk);
696 }
697 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
698 const struct cpl_fw6_msg *p = (void *)rsp;
699
700 if (p->type == 0)
701 t4_handle_fw_rpl(q->adap, p->data);
702 } else if (opcode == CPL_L2T_WRITE_RPL) {
703 const struct cpl_l2t_write_rpl *p = (void *)rsp;
704
705 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 706 } else if (opcode == CPL_SET_TCB_RPL) {
707 const struct cpl_set_tcb_rpl *p = (void *)rsp;
708
709 filter_rpl(q->adap, p);
b8ff05a9
DM		 710 } else
711 dev_err(q->adap->pdev_dev,
712 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 713out:
b8ff05a9
DM		 714 return 0;
715}
716
717/**
718 * uldrx_handler - response queue handler for ULD queues
719 * @q: the response queue that received the packet
720 * @rsp: the response queue descriptor holding the offload message
721 * @gl: the gather list of packet fragments
722 *
723 * Deliver an ingress offload packet to a ULD. All processing is done by
724 * the ULD, we just maintain statistics.
725 */
726static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
727 const struct pkt_gl *gl)
728{
729 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
730
b407a4a9
VP		 731 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
732 */
733 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
734 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
735 rsp += 2;
736
b8ff05a9
DM		 737 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
738 rxq->stats.nomem++;
739 return -1;
740 }
741 if (gl == NULL)
742 rxq->stats.imm++;
743 else if (gl == CXGB4_MSG_AN)
744 rxq->stats.an++;
745 else
746 rxq->stats.pkts++;
747 return 0;
748}
749
750static void disable_msi(struct adapter *adapter)
751{
752 if (adapter->flags & USING_MSIX) {
753 pci_disable_msix(adapter->pdev);
754 adapter->flags &= ~USING_MSIX;
755 } else if (adapter->flags & USING_MSI) {
756 pci_disable_msi(adapter->pdev);
757 adapter->flags &= ~USING_MSI;
758 }
759}
760
761/*
762 * Interrupt handler for non-data events used with MSI-X.
763 */
764static irqreturn_t t4_nondata_intr(int irq, void *cookie)
765{
766 struct adapter *adap = cookie;
767
768 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
769 if (v & PFSW) {
770 adap->swintr = 1;
771 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
772 }
773 t4_slow_intr_handler(adap);
774 return IRQ_HANDLED;
775}
776
777/*
778 * Name the MSI-X interrupts.
779 */
780static void name_msix_vecs(struct adapter *adap)
781{
ba27816c 782 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 783
784 /* non-data interrupts */
b1a3c2b6 785 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 786
787 /* FW events */
b1a3c2b6
DM		 788 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
789 adap->port[0]->name);
b8ff05a9
DM		 790
791 /* Ethernet queues */
792 for_each_port(adap, j) {
793 struct net_device *d = adap->port[j];
794 const struct port_info *pi = netdev_priv(d);
795
ba27816c 796 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 797 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
798 d->name, i);
b8ff05a9
DM		 799 }
800
801 /* offload queues */
ba27816c
DM		 802 for_each_ofldrxq(&adap->sge, i)
803 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 804 adap->port[0]->name, i);
ba27816c
DM		 805
806 for_each_rdmarxq(&adap->sge, i)
807 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 808 adap->port[0]->name, i);
b8ff05a9
DM		 809}
810
811static int request_msix_queue_irqs(struct adapter *adap)
812{
813 struct sge *s = &adap->sge;
404d9e3f 814 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;
b8ff05a9
DM		 815
816 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
817 adap->msix_info[1].desc, &s->fw_evtq);
818 if (err)
819 return err;
820
821 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 822 err = request_irq(adap->msix_info[msi_index].vec,
823 t4_sge_intr_msix, 0,
824 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 825 &s->ethrxq[ethqidx].rspq);
826 if (err)
827 goto unwind;
404d9e3f 828 msi_index++;
b8ff05a9
DM		 829 }
830 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 831 err = request_irq(adap->msix_info[msi_index].vec,
832 t4_sge_intr_msix, 0,
833 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 834 &s->ofldrxq[ofldqidx].rspq);
835 if (err)
836 goto unwind;
404d9e3f 837 msi_index++;
b8ff05a9
DM		 838 }
839 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 840 err = request_irq(adap->msix_info[msi_index].vec,
841 t4_sge_intr_msix, 0,
842 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 843 &s->rdmarxq[rdmaqidx].rspq);
844 if (err)
845 goto unwind;
404d9e3f 846 msi_index++;
b8ff05a9
DM		 847 }
848 return 0;
849
850unwind:
851 while (--rdmaqidx >= 0)
404d9e3f 852 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 853 &s->rdmarxq[rdmaqidx].rspq);
854 while (--ofldqidx >= 0)
404d9e3f 855 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 856 &s->ofldrxq[ofldqidx].rspq);
857 while (--ethqidx >= 0)
404d9e3f
VP		 858 free_irq(adap->msix_info[--msi_index].vec,
859 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 860 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
861 return err;
862}
863
864static void free_msix_queue_irqs(struct adapter *adap)
865{
404d9e3f 866 int i, msi_index = 2;
b8ff05a9
DM		 867 struct sge *s = &adap->sge;
868
869 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
870 for_each_ethrxq(s, i)
404d9e3f 871 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 872 for_each_ofldrxq(s, i)
404d9e3f 873 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 874 for_each_rdmarxq(s, i)
404d9e3f 875 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
b8ff05a9
DM		 876}
877
671b0060
DM		 878/**
879 * write_rss - write the RSS table for a given port
880 * @pi: the port
881 * @queues: array of queue indices for RSS
882 *
883 * Sets up the portion of the HW RSS table for the port's VI to distribute
884 * packets to the Rx queues in @queues.
885 */
886static int write_rss(const struct port_info *pi, const u16 *queues)
887{
888 u16 *rss;
889 int i, err;
890 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
891
892 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
893 if (!rss)
894 return -ENOMEM;
895
896 /* map the queue indices to queue ids */
897 for (i = 0; i < pi->rss_size; i++, queues++)
898 rss[i] = q[*queues].rspq.abs_id;
899
060e0c75
DM		 900 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
901 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 902 kfree(rss);
903 return err;
904}
905
b8ff05a9
DM		 906/**
907 * setup_rss - configure RSS
908 * @adap: the adapter
909 *
671b0060 910 * Sets up RSS for each port.
b8ff05a9
DM		 911 */
912static int setup_rss(struct adapter *adap)
913{
671b0060 914 int i, err;
b8ff05a9
DM		 915
916 for_each_port(adap, i) {
917 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 918
671b0060 919 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 920 if (err)
921 return err;
922 }
923 return 0;
924}
925
e46dab4d
DM		 926/*
927 * Return the channel of the ingress queue with the given qid.
928 */
929static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
930{
931 qid -= p->ingr_start;
932 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
933}
934
b8ff05a9
DM		 935/*
936 * Wait until all NAPI handlers are descheduled.
937 */
938static void quiesce_rx(struct adapter *adap)
939{
940 int i;
941
942 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
943 struct sge_rspq *q = adap->sge.ingr_map[i];
944
945 if (q && q->handler)
946 napi_disable(&q->napi);
947 }
948}
949
950/*
951 * Enable NAPI scheduling and interrupt generation for all Rx queues.
952 */
953static void enable_rx(struct adapter *adap)
954{
955 int i;
956
957 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
958 struct sge_rspq *q = adap->sge.ingr_map[i];
959
960 if (!q)
961 continue;
962 if (q->handler)
963 napi_enable(&q->napi);
964 /* 0-increment GTS to start the timer and enable interrupts */
965 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
966 SEINTARM(q->intr_params) |
967 INGRESSQID(q->cntxt_id));
968 }
969}
970
971/**
972 * setup_sge_queues - configure SGE Tx/Rx/response queues
973 * @adap: the adapter
974 *
975 * Determines how many sets of SGE queues to use and initializes them.
976 * We support multiple queue sets per port if we have MSI-X, otherwise
977 * just one queue set per port.
978 */
979static int setup_sge_queues(struct adapter *adap)
980{
981 int err, msi_idx, i, j;
982 struct sge *s = &adap->sge;
983
984 bitmap_zero(s->starving_fl, MAX_EGRQ);
985 bitmap_zero(s->txq_maperr, MAX_EGRQ);
986
987 if (adap->flags & USING_MSIX)
988 msi_idx = 1; /* vector 0 is for non-queue interrupts */
989 else {
990 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
991 NULL, NULL);
992 if (err)
993 return err;
994 msi_idx = -((int)s->intrq.abs_id + 1);
995 }
996
997 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
998 msi_idx, NULL, fwevtq_handler);
999 if (err) {
1000freeout: t4_free_sge_resources(adap);
1001 return err;
1002 }
1003
1004 for_each_port(adap, i) {
1005 struct net_device *dev = adap->port[i];
1006 struct port_info *pi = netdev_priv(dev);
1007 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1008 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1009
1010 for (j = 0; j < pi->nqsets; j++, q++) {
1011 if (msi_idx > 0)
1012 msi_idx++;
1013 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1014 msi_idx, &q->fl,
1015 t4_ethrx_handler);
1016 if (err)
1017 goto freeout;
1018 q->rspq.idx = j;
1019 memset(&q->stats, 0, sizeof(q->stats));
1020 }
1021 for (j = 0; j < pi->nqsets; j++, t++) {
1022 err = t4_sge_alloc_eth_txq(adap, t, dev,
1023 netdev_get_tx_queue(dev, j),
1024 s->fw_evtq.cntxt_id);
1025 if (err)
1026 goto freeout;
1027 }
1028 }
1029
1030 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1031 for_each_ofldrxq(s, i) {
1032 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1033 struct net_device *dev = adap->port[i / j];
1034
1035 if (msi_idx > 0)
1036 msi_idx++;
1037 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1038 &q->fl, uldrx_handler);
1039 if (err)
1040 goto freeout;
1041 memset(&q->stats, 0, sizeof(q->stats));
1042 s->ofld_rxq[i] = q->rspq.abs_id;
1043 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1044 s->fw_evtq.cntxt_id);
1045 if (err)
1046 goto freeout;
1047 }
1048
1049 for_each_rdmarxq(s, i) {
1050 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1051
1052 if (msi_idx > 0)
1053 msi_idx++;
1054 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1055 msi_idx, &q->fl, uldrx_handler);
1056 if (err)
1057 goto freeout;
1058 memset(&q->stats, 0, sizeof(q->stats));
1059 s->rdma_rxq[i] = q->rspq.abs_id;
1060 }
1061
1062 for_each_port(adap, i) {
1063 /*
1064 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1065 * have RDMA queues, and that's the right value.
1066 */
1067 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1068 s->fw_evtq.cntxt_id,
1069 s->rdmarxq[i].rspq.cntxt_id);
1070 if (err)
1071 goto freeout;
1072 }
1073
1074 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1075 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1076 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1077 return 0;
1078}
1079
b8ff05a9
DM		 1080/*
1081 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1082 * The allocated memory is cleared.
1083 */
1084void *t4_alloc_mem(size_t size)
1085{
8be04b93 1086 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1087
1088 if (!p)
89bf67f1 1089 p = vzalloc(size);
b8ff05a9
DM		 1090 return p;
1091}
1092
1093/*
1094 * Free memory allocated through alloc_mem().
1095 */
31b9c19b 1096static void t4_free_mem(void *addr)
b8ff05a9
DM		 1097{
1098 if (is_vmalloc_addr(addr))
1099 vfree(addr);
1100 else
1101 kfree(addr);
1102}
1103
f2b7e78d
VP		 1104/* Send a Work Request to write the filter at a specified index. We construct
1105 * a Firmware Filter Work Request to have the work done and put the indicated
1106 * filter into "pending" mode which will prevent any further actions against
1107 * it till we get a reply from the firmware on the completion status of the
1108 * request.
1109 */
1110static int set_filter_wr(struct adapter *adapter, int fidx)
1111{
1112 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1113 struct sk_buff *skb;
1114 struct fw_filter_wr *fwr;
1115 unsigned int ftid;
1116
1117 /* If the new filter requires loopback Destination MAC and/or VLAN
1118 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1119 * the filter.
1120 */
1121 if (f->fs.newdmac || f->fs.newvlan) {
1122 /* allocate L2T entry for new filter */
1123 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1124 if (f->l2t == NULL)
1125 return -EAGAIN;
1126 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1127 f->fs.eport, f->fs.dmac)) {
1128 cxgb4_l2t_release(f->l2t);
1129 f->l2t = NULL;
1130 return -ENOMEM;
1131 }
1132 }
1133
1134 ftid = adapter->tids.ftid_base + fidx;
1135
1136 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1137 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1138 memset(fwr, 0, sizeof(*fwr));
1139
1140 /* It would be nice to put most of the following in t4_hw.c but most
1141 * of the work is translating the cxgbtool ch_filter_specification
1142 * into the Work Request and the definition of that structure is
1143 * currently in cxgbtool.h which isn't appropriate to pull into the
1144 * common code. We may eventually try to come up with a more neutral
1145 * filter specification structure but for now it's easiest to simply
1146 * put this fairly direct code in line ...
1147 */
1148 fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1149 fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1150 fwr->tid_to_iq =
1151 htonl(V_FW_FILTER_WR_TID(ftid) |
1152 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1153 V_FW_FILTER_WR_NOREPLY(0) |
1154 V_FW_FILTER_WR_IQ(f->fs.iq));
1155 fwr->del_filter_to_l2tix =
1156 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1157 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1158 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1159 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1160 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1161 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1162 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1163 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1164 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1165 f->fs.newvlan == VLAN_REWRITE) |
1166 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1167 f->fs.newvlan == VLAN_REWRITE) |
1168 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1169 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1170 V_FW_FILTER_WR_PRIO(f->fs.prio) |
1171 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1172 fwr->ethtype = htons(f->fs.val.ethtype);
1173 fwr->ethtypem = htons(f->fs.mask.ethtype);
1174 fwr->frag_to_ovlan_vldm =
1175 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1176 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1177 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1178 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1179 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1180 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1181 fwr->smac_sel = 0;
1182 fwr->rx_chan_rx_rpl_iq =
1183 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1184 V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1185 fwr->maci_to_matchtypem =
1186 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1187 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1188 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1189 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1190 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1191 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1192 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1193 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1194 fwr->ptcl = f->fs.val.proto;
1195 fwr->ptclm = f->fs.mask.proto;
1196 fwr->ttyp = f->fs.val.tos;
1197 fwr->ttypm = f->fs.mask.tos;
1198 fwr->ivlan = htons(f->fs.val.ivlan);
1199 fwr->ivlanm = htons(f->fs.mask.ivlan);
1200 fwr->ovlan = htons(f->fs.val.ovlan);
1201 fwr->ovlanm = htons(f->fs.mask.ovlan);
1202 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1203 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1204 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1205 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1206 fwr->lp = htons(f->fs.val.lport);
1207 fwr->lpm = htons(f->fs.mask.lport);
1208 fwr->fp = htons(f->fs.val.fport);
1209 fwr->fpm = htons(f->fs.mask.fport);
1210 if (f->fs.newsmac)
1211 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1212
1213 /* Mark the filter as "pending" and ship off the Filter Work Request.
1214 * When we get the Work Request Reply we'll clear the pending status.
1215 */
1216 f->pending = 1;
1217 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1218 t4_ofld_send(adapter, skb);
1219 return 0;
1220}
1221
1222/* Delete the filter at a specified index.
1223 */
1224static int del_filter_wr(struct adapter *adapter, int fidx)
1225{
1226 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1227 struct sk_buff *skb;
1228 struct fw_filter_wr *fwr;
1229 unsigned int len, ftid;
1230
1231 len = sizeof(*fwr);
1232 ftid = adapter->tids.ftid_base + fidx;
1233
1234 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1235 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1236 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1237
1238 /* Mark the filter as "pending" and ship off the Filter Work Request.
1239 * When we get the Work Request Reply we'll clear the pending status.
1240 */
1241 f->pending = 1;
1242 t4_mgmt_tx(adapter, skb);
1243 return 0;
1244}
1245
b8ff05a9
DM		 1246static inline int is_offload(const struct adapter *adap)
1247{
1248 return adap->params.offload;
1249}
1250
1251/*
1252 * Implementation of ethtool operations.
1253 */
1254
1255static u32 get_msglevel(struct net_device *dev)
1256{
1257 return netdev2adap(dev)->msg_enable;
1258}
1259
1260static void set_msglevel(struct net_device *dev, u32 val)
1261{
1262 netdev2adap(dev)->msg_enable = val;
1263}
1264
1265static char stats_strings[][ETH_GSTRING_LEN] = {
1266 "TxOctetsOK ",
1267 "TxFramesOK ",
1268 "TxBroadcastFrames ",
1269 "TxMulticastFrames ",
1270 "TxUnicastFrames ",
1271 "TxErrorFrames ",
1272
1273 "TxFrames64 ",
1274 "TxFrames65To127 ",
1275 "TxFrames128To255 ",
1276 "TxFrames256To511 ",
1277 "TxFrames512To1023 ",
1278 "TxFrames1024To1518 ",
1279 "TxFrames1519ToMax ",
1280
1281 "TxFramesDropped ",
1282 "TxPauseFrames ",
1283 "TxPPP0Frames ",
1284 "TxPPP1Frames ",
1285 "TxPPP2Frames ",
1286 "TxPPP3Frames ",
1287 "TxPPP4Frames ",
1288 "TxPPP5Frames ",
1289 "TxPPP6Frames ",
1290 "TxPPP7Frames ",
1291
1292 "RxOctetsOK ",
1293 "RxFramesOK ",
1294 "RxBroadcastFrames ",
1295 "RxMulticastFrames ",
1296 "RxUnicastFrames ",
1297
1298 "RxFramesTooLong ",
1299 "RxJabberErrors ",
1300 "RxFCSErrors ",
1301 "RxLengthErrors ",
1302 "RxSymbolErrors ",
1303 "RxRuntFrames ",
1304
1305 "RxFrames64 ",
1306 "RxFrames65To127 ",
1307 "RxFrames128To255 ",
1308 "RxFrames256To511 ",
1309 "RxFrames512To1023 ",
1310 "RxFrames1024To1518 ",
1311 "RxFrames1519ToMax ",
1312
1313 "RxPauseFrames ",
1314 "RxPPP0Frames ",
1315 "RxPPP1Frames ",
1316 "RxPPP2Frames ",
1317 "RxPPP3Frames ",
1318 "RxPPP4Frames ",
1319 "RxPPP5Frames ",
1320 "RxPPP6Frames ",
1321 "RxPPP7Frames ",
1322
1323 "RxBG0FramesDropped ",
1324 "RxBG1FramesDropped ",
1325 "RxBG2FramesDropped ",
1326 "RxBG3FramesDropped ",
1327 "RxBG0FramesTrunc ",
1328 "RxBG1FramesTrunc ",
1329 "RxBG2FramesTrunc ",
1330 "RxBG3FramesTrunc ",
1331
1332 "TSO ",
1333 "TxCsumOffload ",
1334 "RxCsumGood ",
1335 "VLANextractions ",
1336 "VLANinsertions ",
4a6346d4
DM		 1337 "GROpackets ",
1338 "GROmerged ",
22adfe0a
SR		 1339 "WriteCoalSuccess ",
1340 "WriteCoalFail ",
b8ff05a9
DM		 1341};
1342
1343static int get_sset_count(struct net_device *dev, int sset)
1344{
1345 switch (sset) {
1346 case ETH_SS_STATS:
1347 return ARRAY_SIZE(stats_strings);
1348 default:
1349 return -EOPNOTSUPP;
1350 }
1351}
1352
1353#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1354#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1355
1356static int get_regs_len(struct net_device *dev)
1357{
251f9e88 1358 struct adapter *adap = netdev2adap(dev);
d14807dd 1359 if (is_t4(adap->params.chip))
251f9e88
SR		 1360 return T4_REGMAP_SIZE;
1361 else
1362 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1363}
1364
1365static int get_eeprom_len(struct net_device *dev)
1366{
1367 return EEPROMSIZE;
1368}
1369
1370static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1371{
1372 struct adapter *adapter = netdev2adap(dev);
1373
23020ab3
RJ		 1374 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1375 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1376 strlcpy(info->bus_info, pci_name(adapter->pdev),
1377 sizeof(info->bus_info));
b8ff05a9 1378
84b40501 1379 if (adapter->params.fw_vers)
b8ff05a9
DM		 1380 snprintf(info->fw_version, sizeof(info->fw_version),
1381 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1382 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1383 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1384 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1385 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1386 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1387 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1388 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1389 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1390}
1391
1392static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1393{
1394 if (stringset == ETH_SS_STATS)
1395 memcpy(data, stats_strings, sizeof(stats_strings));
1396}
1397
1398/*
1399 * port stats maintained per queue of the port. They should be in the same
1400 * order as in stats_strings above.
1401 */
1402struct queue_port_stats {
1403 u64 tso;
1404 u64 tx_csum;
1405 u64 rx_csum;
1406 u64 vlan_ex;
1407 u64 vlan_ins;
4a6346d4
DM		 1408 u64 gro_pkts;
1409 u64 gro_merged;
b8ff05a9
DM		 1410};
1411
1412static void collect_sge_port_stats(const struct adapter *adap,
1413 const struct port_info *p, struct queue_port_stats *s)
1414{
1415 int i;
1416 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1417 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1418
1419 memset(s, 0, sizeof(*s));
1420 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1421 s->tso += tx->tso;
1422 s->tx_csum += tx->tx_cso;
1423 s->rx_csum += rx->stats.rx_cso;
1424 s->vlan_ex += rx->stats.vlan_ex;
1425 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1426 s->gro_pkts += rx->stats.lro_pkts;
1427 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1428 }
1429}
1430
1431static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1432 u64 *data)
1433{
1434 struct port_info *pi = netdev_priv(dev);
1435 struct adapter *adapter = pi->adapter;
22adfe0a 1436 u32 val1, val2;
b8ff05a9
DM		 1437
1438 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1439
1440 data += sizeof(struct port_stats) / sizeof(u64);
1441 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1442 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1443 if (!is_t4(adapter->params.chip)) {
22adfe0a
SR		 1444 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1445 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1446 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1447 *data = val1 - val2;
1448 data++;
1449 *data = val2;
1450 data++;
1451 } else {
1452 memset(data, 0, 2 * sizeof(u64));
1453 *data += 2;
1454 }
b8ff05a9
DM		 1455}
1456
1457/*
1458 * Return a version number to identify the type of adapter. The scheme is:
1459 * - bits 0..9: chip version
1460 * - bits 10..15: chip revision
835bb606 1461 * - bits 16..23: register dump version
b8ff05a9
DM		 1462 */
1463static inline unsigned int mk_adap_vers(const struct adapter *ap)
1464{
d14807dd
HS		 1465 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1466 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1467}
1468
1469static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1470 unsigned int end)
1471{
1472 u32 *p = buf + start;
1473
1474 for ( ; start <= end; start += sizeof(u32))
1475 *p++ = t4_read_reg(ap, start);
1476}
1477
1478static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1479 void *buf)
1480{
251f9e88 1481 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1482 0x1008, 0x1108,
1483 0x1180, 0x11b4,
1484 0x11fc, 0x123c,
1485 0x1300, 0x173c,
1486 0x1800, 0x18fc,
1487 0x3000, 0x30d8,
1488 0x30e0, 0x5924,
1489 0x5960, 0x59d4,
1490 0x5a00, 0x5af8,
1491 0x6000, 0x6098,
1492 0x6100, 0x6150,
1493 0x6200, 0x6208,
1494 0x6240, 0x6248,
1495 0x6280, 0x6338,
1496 0x6370, 0x638c,
1497 0x6400, 0x643c,
1498 0x6500, 0x6524,
1499 0x6a00, 0x6a38,
1500 0x6a60, 0x6a78,
1501 0x6b00, 0x6b84,
1502 0x6bf0, 0x6c84,
1503 0x6cf0, 0x6d84,
1504 0x6df0, 0x6e84,
1505 0x6ef0, 0x6f84,
1506 0x6ff0, 0x7084,
1507 0x70f0, 0x7184,
1508 0x71f0, 0x7284,
1509 0x72f0, 0x7384,
1510 0x73f0, 0x7450,
1511 0x7500, 0x7530,
1512 0x7600, 0x761c,
1513 0x7680, 0x76cc,
1514 0x7700, 0x7798,
1515 0x77c0, 0x77fc,
1516 0x7900, 0x79fc,
1517 0x7b00, 0x7c38,
1518 0x7d00, 0x7efc,
1519 0x8dc0, 0x8e1c,
1520 0x8e30, 0x8e78,
1521 0x8ea0, 0x8f6c,
1522 0x8fc0, 0x9074,
1523 0x90fc, 0x90fc,
1524 0x9400, 0x9458,
1525 0x9600, 0x96bc,
1526 0x9800, 0x9808,
1527 0x9820, 0x983c,
1528 0x9850, 0x9864,
1529 0x9c00, 0x9c6c,
1530 0x9c80, 0x9cec,
1531 0x9d00, 0x9d6c,
1532 0x9d80, 0x9dec,
1533 0x9e00, 0x9e6c,
1534 0x9e80, 0x9eec,
1535 0x9f00, 0x9f6c,
1536 0x9f80, 0x9fec,
1537 0xd004, 0xd03c,
1538 0xdfc0, 0xdfe0,
1539 0xe000, 0xea7c,
1540 0xf000, 0x11190,
835bb606
DM		 1541 0x19040, 0x1906c,
1542 0x19078, 0x19080,
1543 0x1908c, 0x19124,
b8ff05a9
DM		 1544 0x19150, 0x191b0,
1545 0x191d0, 0x191e8,
1546 0x19238, 0x1924c,
1547 0x193f8, 0x19474,
1548 0x19490, 0x194f8,
1549 0x19800, 0x19f30,
1550 0x1a000, 0x1a06c,
1551 0x1a0b0, 0x1a120,
1552 0x1a128, 0x1a138,
1553 0x1a190, 0x1a1c4,
1554 0x1a1fc, 0x1a1fc,
1555 0x1e040, 0x1e04c,
835bb606 1556 0x1e284, 0x1e28c,
b8ff05a9
DM		 1557 0x1e2c0, 0x1e2c0,
1558 0x1e2e0, 0x1e2e0,
1559 0x1e300, 0x1e384,
1560 0x1e3c0, 0x1e3c8,
1561 0x1e440, 0x1e44c,
835bb606 1562 0x1e684, 0x1e68c,
b8ff05a9
DM		 1563 0x1e6c0, 0x1e6c0,
1564 0x1e6e0, 0x1e6e0,
1565 0x1e700, 0x1e784,
1566 0x1e7c0, 0x1e7c8,
1567 0x1e840, 0x1e84c,
835bb606 1568 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1569 0x1eac0, 0x1eac0,
1570 0x1eae0, 0x1eae0,
1571 0x1eb00, 0x1eb84,
1572 0x1ebc0, 0x1ebc8,
1573 0x1ec40, 0x1ec4c,
835bb606 1574 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1575 0x1eec0, 0x1eec0,
1576 0x1eee0, 0x1eee0,
1577 0x1ef00, 0x1ef84,
1578 0x1efc0, 0x1efc8,
1579 0x1f040, 0x1f04c,
835bb606 1580 0x1f284, 0x1f28c,
b8ff05a9
DM		 1581 0x1f2c0, 0x1f2c0,
1582 0x1f2e0, 0x1f2e0,
1583 0x1f300, 0x1f384,
1584 0x1f3c0, 0x1f3c8,
1585 0x1f440, 0x1f44c,
835bb606 1586 0x1f684, 0x1f68c,
b8ff05a9
DM		 1587 0x1f6c0, 0x1f6c0,
1588 0x1f6e0, 0x1f6e0,
1589 0x1f700, 0x1f784,
1590 0x1f7c0, 0x1f7c8,
1591 0x1f840, 0x1f84c,
835bb606 1592 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1593 0x1fac0, 0x1fac0,
1594 0x1fae0, 0x1fae0,
1595 0x1fb00, 0x1fb84,
1596 0x1fbc0, 0x1fbc8,
1597 0x1fc40, 0x1fc4c,
835bb606 1598 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1599 0x1fec0, 0x1fec0,
1600 0x1fee0, 0x1fee0,
1601 0x1ff00, 0x1ff84,
1602 0x1ffc0, 0x1ffc8,
1603 0x20000, 0x2002c,
1604 0x20100, 0x2013c,
1605 0x20190, 0x201c8,
1606 0x20200, 0x20318,
1607 0x20400, 0x20528,
1608 0x20540, 0x20614,
1609 0x21000, 0x21040,
1610 0x2104c, 0x21060,
1611 0x210c0, 0x210ec,
1612 0x21200, 0x21268,
1613 0x21270, 0x21284,
1614 0x212fc, 0x21388,
1615 0x21400, 0x21404,
1616 0x21500, 0x21518,
1617 0x2152c, 0x2153c,
1618 0x21550, 0x21554,
1619 0x21600, 0x21600,
1620 0x21608, 0x21628,
1621 0x21630, 0x2163c,
1622 0x21700, 0x2171c,
1623 0x21780, 0x2178c,
1624 0x21800, 0x21c38,
1625 0x21c80, 0x21d7c,
1626 0x21e00, 0x21e04,
1627 0x22000, 0x2202c,
1628 0x22100, 0x2213c,
1629 0x22190, 0x221c8,
1630 0x22200, 0x22318,
1631 0x22400, 0x22528,
1632 0x22540, 0x22614,
1633 0x23000, 0x23040,
1634 0x2304c, 0x23060,
1635 0x230c0, 0x230ec,
1636 0x23200, 0x23268,
1637 0x23270, 0x23284,
1638 0x232fc, 0x23388,
1639 0x23400, 0x23404,
1640 0x23500, 0x23518,
1641 0x2352c, 0x2353c,
1642 0x23550, 0x23554,
1643 0x23600, 0x23600,
1644 0x23608, 0x23628,
1645 0x23630, 0x2363c,
1646 0x23700, 0x2371c,
1647 0x23780, 0x2378c,
1648 0x23800, 0x23c38,
1649 0x23c80, 0x23d7c,
1650 0x23e00, 0x23e04,
1651 0x24000, 0x2402c,
1652 0x24100, 0x2413c,
1653 0x24190, 0x241c8,
1654 0x24200, 0x24318,
1655 0x24400, 0x24528,
1656 0x24540, 0x24614,
1657 0x25000, 0x25040,
1658 0x2504c, 0x25060,
1659 0x250c0, 0x250ec,
1660 0x25200, 0x25268,
1661 0x25270, 0x25284,
1662 0x252fc, 0x25388,
1663 0x25400, 0x25404,
1664 0x25500, 0x25518,
1665 0x2552c, 0x2553c,
1666 0x25550, 0x25554,
1667 0x25600, 0x25600,
1668 0x25608, 0x25628,
1669 0x25630, 0x2563c,
1670 0x25700, 0x2571c,
1671 0x25780, 0x2578c,
1672 0x25800, 0x25c38,
1673 0x25c80, 0x25d7c,
1674 0x25e00, 0x25e04,
1675 0x26000, 0x2602c,
1676 0x26100, 0x2613c,
1677 0x26190, 0x261c8,
1678 0x26200, 0x26318,
1679 0x26400, 0x26528,
1680 0x26540, 0x26614,
1681 0x27000, 0x27040,
1682 0x2704c, 0x27060,
1683 0x270c0, 0x270ec,
1684 0x27200, 0x27268,
1685 0x27270, 0x27284,
1686 0x272fc, 0x27388,
1687 0x27400, 0x27404,
1688 0x27500, 0x27518,
1689 0x2752c, 0x2753c,
1690 0x27550, 0x27554,
1691 0x27600, 0x27600,
1692 0x27608, 0x27628,
1693 0x27630, 0x2763c,
1694 0x27700, 0x2771c,
1695 0x27780, 0x2778c,
1696 0x27800, 0x27c38,
1697 0x27c80, 0x27d7c,
1698 0x27e00, 0x27e04
1699 };
1700
251f9e88
SR		 1701 static const unsigned int t5_reg_ranges[] = {
1702 0x1008, 0x1148,
1703 0x1180, 0x11b4,
1704 0x11fc, 0x123c,
1705 0x1280, 0x173c,
1706 0x1800, 0x18fc,
1707 0x3000, 0x3028,
1708 0x3060, 0x30d8,
1709 0x30e0, 0x30fc,
1710 0x3140, 0x357c,
1711 0x35a8, 0x35cc,
1712 0x35ec, 0x35ec,
1713 0x3600, 0x5624,
1714 0x56cc, 0x575c,
1715 0x580c, 0x5814,
1716 0x5890, 0x58bc,
1717 0x5940, 0x59dc,
1718 0x59fc, 0x5a18,
1719 0x5a60, 0x5a9c,
1720 0x5b9c, 0x5bfc,
1721 0x6000, 0x6040,
1722 0x6058, 0x614c,
1723 0x7700, 0x7798,
1724 0x77c0, 0x78fc,
1725 0x7b00, 0x7c54,
1726 0x7d00, 0x7efc,
1727 0x8dc0, 0x8de0,
1728 0x8df8, 0x8e84,
1729 0x8ea0, 0x8f84,
1730 0x8fc0, 0x90f8,
1731 0x9400, 0x9470,
1732 0x9600, 0x96f4,
1733 0x9800, 0x9808,
1734 0x9820, 0x983c,
1735 0x9850, 0x9864,
1736 0x9c00, 0x9c6c,
1737 0x9c80, 0x9cec,
1738 0x9d00, 0x9d6c,
1739 0x9d80, 0x9dec,
1740 0x9e00, 0x9e6c,
1741 0x9e80, 0x9eec,
1742 0x9f00, 0x9f6c,
1743 0x9f80, 0xa020,
1744 0xd004, 0xd03c,
1745 0xdfc0, 0xdfe0,
1746 0xe000, 0x11088,
1747 0x1109c, 0x1117c,
1748 0x11190, 0x11204,
1749 0x19040, 0x1906c,
1750 0x19078, 0x19080,
1751 0x1908c, 0x19124,
1752 0x19150, 0x191b0,
1753 0x191d0, 0x191e8,
1754 0x19238, 0x19290,
1755 0x193f8, 0x19474,
1756 0x19490, 0x194cc,
1757 0x194f0, 0x194f8,
1758 0x19c00, 0x19c60,
1759 0x19c94, 0x19e10,
1760 0x19e50, 0x19f34,
1761 0x19f40, 0x19f50,
1762 0x19f90, 0x19fe4,
1763 0x1a000, 0x1a06c,
1764 0x1a0b0, 0x1a120,
1765 0x1a128, 0x1a138,
1766 0x1a190, 0x1a1c4,
1767 0x1a1fc, 0x1a1fc,
1768 0x1e008, 0x1e00c,
1769 0x1e040, 0x1e04c,
1770 0x1e284, 0x1e290,
1771 0x1e2c0, 0x1e2c0,
1772 0x1e2e0, 0x1e2e0,
1773 0x1e300, 0x1e384,
1774 0x1e3c0, 0x1e3c8,
1775 0x1e408, 0x1e40c,
1776 0x1e440, 0x1e44c,
1777 0x1e684, 0x1e690,
1778 0x1e6c0, 0x1e6c0,
1779 0x1e6e0, 0x1e6e0,
1780 0x1e700, 0x1e784,
1781 0x1e7c0, 0x1e7c8,
1782 0x1e808, 0x1e80c,
1783 0x1e840, 0x1e84c,
1784 0x1ea84, 0x1ea90,
1785 0x1eac0, 0x1eac0,
1786 0x1eae0, 0x1eae0,
1787 0x1eb00, 0x1eb84,
1788 0x1ebc0, 0x1ebc8,
1789 0x1ec08, 0x1ec0c,
1790 0x1ec40, 0x1ec4c,
1791 0x1ee84, 0x1ee90,
1792 0x1eec0, 0x1eec0,
1793 0x1eee0, 0x1eee0,
1794 0x1ef00, 0x1ef84,
1795 0x1efc0, 0x1efc8,
1796 0x1f008, 0x1f00c,
1797 0x1f040, 0x1f04c,
1798 0x1f284, 0x1f290,
1799 0x1f2c0, 0x1f2c0,
1800 0x1f2e0, 0x1f2e0,
1801 0x1f300, 0x1f384,
1802 0x1f3c0, 0x1f3c8,
1803 0x1f408, 0x1f40c,
1804 0x1f440, 0x1f44c,
1805 0x1f684, 0x1f690,
1806 0x1f6c0, 0x1f6c0,
1807 0x1f6e0, 0x1f6e0,
1808 0x1f700, 0x1f784,
1809 0x1f7c0, 0x1f7c8,
1810 0x1f808, 0x1f80c,
1811 0x1f840, 0x1f84c,
1812 0x1fa84, 0x1fa90,
1813 0x1fac0, 0x1fac0,
1814 0x1fae0, 0x1fae0,
1815 0x1fb00, 0x1fb84,
1816 0x1fbc0, 0x1fbc8,
1817 0x1fc08, 0x1fc0c,
1818 0x1fc40, 0x1fc4c,
1819 0x1fe84, 0x1fe90,
1820 0x1fec0, 0x1fec0,
1821 0x1fee0, 0x1fee0,
1822 0x1ff00, 0x1ff84,
1823 0x1ffc0, 0x1ffc8,
1824 0x30000, 0x30030,
1825 0x30100, 0x30144,
1826 0x30190, 0x301d0,
1827 0x30200, 0x30318,
1828 0x30400, 0x3052c,
1829 0x30540, 0x3061c,
1830 0x30800, 0x30834,
1831 0x308c0, 0x30908,
1832 0x30910, 0x309ac,
1833 0x30a00, 0x30a04,
1834 0x30a0c, 0x30a2c,
1835 0x30a44, 0x30a50,
1836 0x30a74, 0x30c24,
1837 0x30d08, 0x30d14,
1838 0x30d1c, 0x30d20,
1839 0x30d3c, 0x30d50,
1840 0x31200, 0x3120c,
1841 0x31220, 0x31220,
1842 0x31240, 0x31240,
1843 0x31600, 0x31600,
1844 0x31608, 0x3160c,
1845 0x31a00, 0x31a1c,
1846 0x31e04, 0x31e20,
1847 0x31e38, 0x31e3c,
1848 0x31e80, 0x31e80,
1849 0x31e88, 0x31ea8,
1850 0x31eb0, 0x31eb4,
1851 0x31ec8, 0x31ed4,
1852 0x31fb8, 0x32004,
1853 0x32208, 0x3223c,
1854 0x32600, 0x32630,
1855 0x32a00, 0x32abc,
1856 0x32b00, 0x32b70,
1857 0x33000, 0x33048,
1858 0x33060, 0x3309c,
1859 0x330f0, 0x33148,
1860 0x33160, 0x3319c,
1861 0x331f0, 0x332e4,
1862 0x332f8, 0x333e4,
1863 0x333f8, 0x33448,
1864 0x33460, 0x3349c,
1865 0x334f0, 0x33548,
1866 0x33560, 0x3359c,
1867 0x335f0, 0x336e4,
1868 0x336f8, 0x337e4,
1869 0x337f8, 0x337fc,
1870 0x33814, 0x33814,
1871 0x3382c, 0x3382c,
1872 0x33880, 0x3388c,
1873 0x338e8, 0x338ec,
1874 0x33900, 0x33948,
1875 0x33960, 0x3399c,
1876 0x339f0, 0x33ae4,
1877 0x33af8, 0x33b10,
1878 0x33b28, 0x33b28,
1879 0x33b3c, 0x33b50,
1880 0x33bf0, 0x33c10,
1881 0x33c28, 0x33c28,
1882 0x33c3c, 0x33c50,
1883 0x33cf0, 0x33cfc,
1884 0x34000, 0x34030,
1885 0x34100, 0x34144,
1886 0x34190, 0x341d0,
1887 0x34200, 0x34318,
1888 0x34400, 0x3452c,
1889 0x34540, 0x3461c,
1890 0x34800, 0x34834,
1891 0x348c0, 0x34908,
1892 0x34910, 0x349ac,
1893 0x34a00, 0x34a04,
1894 0x34a0c, 0x34a2c,
1895 0x34a44, 0x34a50,
1896 0x34a74, 0x34c24,
1897 0x34d08, 0x34d14,
1898 0x34d1c, 0x34d20,
1899 0x34d3c, 0x34d50,
1900 0x35200, 0x3520c,
1901 0x35220, 0x35220,
1902 0x35240, 0x35240,
1903 0x35600, 0x35600,
1904 0x35608, 0x3560c,
1905 0x35a00, 0x35a1c,
1906 0x35e04, 0x35e20,
1907 0x35e38, 0x35e3c,
1908 0x35e80, 0x35e80,
1909 0x35e88, 0x35ea8,
1910 0x35eb0, 0x35eb4,
1911 0x35ec8, 0x35ed4,
1912 0x35fb8, 0x36004,
1913 0x36208, 0x3623c,
1914 0x36600, 0x36630,
1915 0x36a00, 0x36abc,
1916 0x36b00, 0x36b70,
1917 0x37000, 0x37048,
1918 0x37060, 0x3709c,
1919 0x370f0, 0x37148,
1920 0x37160, 0x3719c,
1921 0x371f0, 0x372e4,
1922 0x372f8, 0x373e4,
1923 0x373f8, 0x37448,
1924 0x37460, 0x3749c,
1925 0x374f0, 0x37548,
1926 0x37560, 0x3759c,
1927 0x375f0, 0x376e4,
1928 0x376f8, 0x377e4,
1929 0x377f8, 0x377fc,
1930 0x37814, 0x37814,
1931 0x3782c, 0x3782c,
1932 0x37880, 0x3788c,
1933 0x378e8, 0x378ec,
1934 0x37900, 0x37948,
1935 0x37960, 0x3799c,
1936 0x379f0, 0x37ae4,
1937 0x37af8, 0x37b10,
1938 0x37b28, 0x37b28,
1939 0x37b3c, 0x37b50,
1940 0x37bf0, 0x37c10,
1941 0x37c28, 0x37c28,
1942 0x37c3c, 0x37c50,
1943 0x37cf0, 0x37cfc,
1944 0x38000, 0x38030,
1945 0x38100, 0x38144,
1946 0x38190, 0x381d0,
1947 0x38200, 0x38318,
1948 0x38400, 0x3852c,
1949 0x38540, 0x3861c,
1950 0x38800, 0x38834,
1951 0x388c0, 0x38908,
1952 0x38910, 0x389ac,
1953 0x38a00, 0x38a04,
1954 0x38a0c, 0x38a2c,
1955 0x38a44, 0x38a50,
1956 0x38a74, 0x38c24,
1957 0x38d08, 0x38d14,
1958 0x38d1c, 0x38d20,
1959 0x38d3c, 0x38d50,
1960 0x39200, 0x3920c,
1961 0x39220, 0x39220,
1962 0x39240, 0x39240,
1963 0x39600, 0x39600,
1964 0x39608, 0x3960c,
1965 0x39a00, 0x39a1c,
1966 0x39e04, 0x39e20,
1967 0x39e38, 0x39e3c,
1968 0x39e80, 0x39e80,
1969 0x39e88, 0x39ea8,
1970 0x39eb0, 0x39eb4,
1971 0x39ec8, 0x39ed4,
1972 0x39fb8, 0x3a004,
1973 0x3a208, 0x3a23c,
1974 0x3a600, 0x3a630,
1975 0x3aa00, 0x3aabc,
1976 0x3ab00, 0x3ab70,
1977 0x3b000, 0x3b048,
1978 0x3b060, 0x3b09c,
1979 0x3b0f0, 0x3b148,
1980 0x3b160, 0x3b19c,
1981 0x3b1f0, 0x3b2e4,
1982 0x3b2f8, 0x3b3e4,
1983 0x3b3f8, 0x3b448,
1984 0x3b460, 0x3b49c,
1985 0x3b4f0, 0x3b548,
1986 0x3b560, 0x3b59c,
1987 0x3b5f0, 0x3b6e4,
1988 0x3b6f8, 0x3b7e4,
1989 0x3b7f8, 0x3b7fc,
1990 0x3b814, 0x3b814,
1991 0x3b82c, 0x3b82c,
1992 0x3b880, 0x3b88c,
1993 0x3b8e8, 0x3b8ec,
1994 0x3b900, 0x3b948,
1995 0x3b960, 0x3b99c,
1996 0x3b9f0, 0x3bae4,
1997 0x3baf8, 0x3bb10,
1998 0x3bb28, 0x3bb28,
1999 0x3bb3c, 0x3bb50,
2000 0x3bbf0, 0x3bc10,
2001 0x3bc28, 0x3bc28,
2002 0x3bc3c, 0x3bc50,
2003 0x3bcf0, 0x3bcfc,
2004 0x3c000, 0x3c030,
2005 0x3c100, 0x3c144,
2006 0x3c190, 0x3c1d0,
2007 0x3c200, 0x3c318,
2008 0x3c400, 0x3c52c,
2009 0x3c540, 0x3c61c,
2010 0x3c800, 0x3c834,
2011 0x3c8c0, 0x3c908,
2012 0x3c910, 0x3c9ac,
2013 0x3ca00, 0x3ca04,
2014 0x3ca0c, 0x3ca2c,
2015 0x3ca44, 0x3ca50,
2016 0x3ca74, 0x3cc24,
2017 0x3cd08, 0x3cd14,
2018 0x3cd1c, 0x3cd20,
2019 0x3cd3c, 0x3cd50,
2020 0x3d200, 0x3d20c,
2021 0x3d220, 0x3d220,
2022 0x3d240, 0x3d240,
2023 0x3d600, 0x3d600,
2024 0x3d608, 0x3d60c,
2025 0x3da00, 0x3da1c,
2026 0x3de04, 0x3de20,
2027 0x3de38, 0x3de3c,
2028 0x3de80, 0x3de80,
2029 0x3de88, 0x3dea8,
2030 0x3deb0, 0x3deb4,
2031 0x3dec8, 0x3ded4,
2032 0x3dfb8, 0x3e004,
2033 0x3e208, 0x3e23c,
2034 0x3e600, 0x3e630,
2035 0x3ea00, 0x3eabc,
2036 0x3eb00, 0x3eb70,
2037 0x3f000, 0x3f048,
2038 0x3f060, 0x3f09c,
2039 0x3f0f0, 0x3f148,
2040 0x3f160, 0x3f19c,
2041 0x3f1f0, 0x3f2e4,
2042 0x3f2f8, 0x3f3e4,
2043 0x3f3f8, 0x3f448,
2044 0x3f460, 0x3f49c,
2045 0x3f4f0, 0x3f548,
2046 0x3f560, 0x3f59c,
2047 0x3f5f0, 0x3f6e4,
2048 0x3f6f8, 0x3f7e4,
2049 0x3f7f8, 0x3f7fc,
2050 0x3f814, 0x3f814,
2051 0x3f82c, 0x3f82c,
2052 0x3f880, 0x3f88c,
2053 0x3f8e8, 0x3f8ec,
2054 0x3f900, 0x3f948,
2055 0x3f960, 0x3f99c,
2056 0x3f9f0, 0x3fae4,
2057 0x3faf8, 0x3fb10,
2058 0x3fb28, 0x3fb28,
2059 0x3fb3c, 0x3fb50,
2060 0x3fbf0, 0x3fc10,
2061 0x3fc28, 0x3fc28,
2062 0x3fc3c, 0x3fc50,
2063 0x3fcf0, 0x3fcfc,
2064 0x40000, 0x4000c,
2065 0x40040, 0x40068,
2066 0x40080, 0x40144,
2067 0x40180, 0x4018c,
2068 0x40200, 0x40298,
2069 0x402ac, 0x4033c,
2070 0x403f8, 0x403fc,
c1f49e3e 2071 0x41304, 0x413c4,
251f9e88
SR		 2072 0x41400, 0x4141c,
2073 0x41480, 0x414d0,
2074 0x44000, 0x44078,
2075 0x440c0, 0x44278,
2076 0x442c0, 0x44478,
2077 0x444c0, 0x44678,
2078 0x446c0, 0x44878,
2079 0x448c0, 0x449fc,
2080 0x45000, 0x45068,
2081 0x45080, 0x45084,
2082 0x450a0, 0x450b0,
2083 0x45200, 0x45268,
2084 0x45280, 0x45284,
2085 0x452a0, 0x452b0,
2086 0x460c0, 0x460e4,
2087 0x47000, 0x4708c,
2088 0x47200, 0x47250,
2089 0x47400, 0x47420,
2090 0x47600, 0x47618,
2091 0x47800, 0x47814,
2092 0x48000, 0x4800c,
2093 0x48040, 0x48068,
2094 0x48080, 0x48144,
2095 0x48180, 0x4818c,
2096 0x48200, 0x48298,
2097 0x482ac, 0x4833c,
2098 0x483f8, 0x483fc,
c1f49e3e 2099 0x49304, 0x493c4,
251f9e88
SR		 2100 0x49400, 0x4941c,
2101 0x49480, 0x494d0,
2102 0x4c000, 0x4c078,
2103 0x4c0c0, 0x4c278,
2104 0x4c2c0, 0x4c478,
2105 0x4c4c0, 0x4c678,
2106 0x4c6c0, 0x4c878,
2107 0x4c8c0, 0x4c9fc,
2108 0x4d000, 0x4d068,
2109 0x4d080, 0x4d084,
2110 0x4d0a0, 0x4d0b0,
2111 0x4d200, 0x4d268,
2112 0x4d280, 0x4d284,
2113 0x4d2a0, 0x4d2b0,
2114 0x4e0c0, 0x4e0e4,
2115 0x4f000, 0x4f08c,
2116 0x4f200, 0x4f250,
2117 0x4f400, 0x4f420,
2118 0x4f600, 0x4f618,
2119 0x4f800, 0x4f814,
2120 0x50000, 0x500cc,
2121 0x50400, 0x50400,
2122 0x50800, 0x508cc,
2123 0x50c00, 0x50c00,
2124 0x51000, 0x5101c,
2125 0x51300, 0x51308,
2126 };
2127
b8ff05a9
DM		 2128 int i;
2129 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2130 static const unsigned int *reg_ranges;
2131 int arr_size = 0, buf_size = 0;
2132
d14807dd 2133 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2134 reg_ranges = &t4_reg_ranges[0];
2135 arr_size = ARRAY_SIZE(t4_reg_ranges);
2136 buf_size = T4_REGMAP_SIZE;
2137 } else {
2138 reg_ranges = &t5_reg_ranges[0];
2139 arr_size = ARRAY_SIZE(t5_reg_ranges);
2140 buf_size = T5_REGMAP_SIZE;
2141 }
b8ff05a9
DM		 2142
2143 regs->version = mk_adap_vers(ap);
2144
251f9e88
SR		 2145 memset(buf, 0, buf_size);
2146 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2147 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2148}
2149
2150static int restart_autoneg(struct net_device *dev)
2151{
2152 struct port_info *p = netdev_priv(dev);
2153
2154 if (!netif_running(dev))
2155 return -EAGAIN;
2156 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2157 return -EINVAL;
060e0c75 2158 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2159 return 0;
2160}
2161
c5e06360
DM		 2162static int identify_port(struct net_device *dev,
2163 enum ethtool_phys_id_state state)
b8ff05a9 2164{
c5e06360 2165 unsigned int val;
060e0c75
DM		 2166 struct adapter *adap = netdev2adap(dev);
2167
c5e06360
DM		 2168 if (state == ETHTOOL_ID_ACTIVE)
2169 val = 0xffff;
2170 else if (state == ETHTOOL_ID_INACTIVE)
2171 val = 0;
2172 else
2173 return -EINVAL;
b8ff05a9 2174
c5e06360 2175 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2176}
2177
2178static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
2179{
2180 unsigned int v = 0;
2181
a0881cab
DM		 2182 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2183 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2184 v |= SUPPORTED_TP;
2185 if (caps & FW_PORT_CAP_SPEED_100M)
2186 v |= SUPPORTED_100baseT_Full;
2187 if (caps & FW_PORT_CAP_SPEED_1G)
2188 v |= SUPPORTED_1000baseT_Full;
2189 if (caps & FW_PORT_CAP_SPEED_10G)
2190 v |= SUPPORTED_10000baseT_Full;
2191 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2192 v |= SUPPORTED_Backplane;
2193 if (caps & FW_PORT_CAP_SPEED_1G)
2194 v |= SUPPORTED_1000baseKX_Full;
2195 if (caps & FW_PORT_CAP_SPEED_10G)
2196 v |= SUPPORTED_10000baseKX4_Full;
2197 } else if (type == FW_PORT_TYPE_KR)
2198 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2199 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2200 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2201 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2202 else if (type == FW_PORT_TYPE_BP4_AP)
2203 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2204 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2205 SUPPORTED_10000baseKX4_Full;
a0881cab
DM		 2206 else if (type == FW_PORT_TYPE_FIBER_XFI ||
2207 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9 2208 v |= SUPPORTED_FIBRE;
72aca4bf
KS		 2209 else if (type == FW_PORT_TYPE_BP40_BA)
2210 v |= SUPPORTED_40000baseSR4_Full;
b8ff05a9
DM		 2211
2212 if (caps & FW_PORT_CAP_ANEG)
2213 v |= SUPPORTED_Autoneg;
2214 return v;
2215}
2216
2217static unsigned int to_fw_linkcaps(unsigned int caps)
2218{
2219 unsigned int v = 0;
2220
2221 if (caps & ADVERTISED_100baseT_Full)
2222 v |= FW_PORT_CAP_SPEED_100M;
2223 if (caps & ADVERTISED_1000baseT_Full)
2224 v |= FW_PORT_CAP_SPEED_1G;
2225 if (caps & ADVERTISED_10000baseT_Full)
2226 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2227 if (caps & ADVERTISED_40000baseSR4_Full)
2228 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2229 return v;
2230}
2231
2232static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2233{
2234 const struct port_info *p = netdev_priv(dev);
2235
2236 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2237 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2238 p->port_type == FW_PORT_TYPE_BT_XAUI)
2239 cmd->port = PORT_TP;
a0881cab
DM		 2240 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2241 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2242 cmd->port = PORT_FIBRE;
a0881cab
DM		 2243 else if (p->port_type == FW_PORT_TYPE_SFP) {
2244 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2245 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2246 cmd->port = PORT_DA;
2247 else
2248 cmd->port = PORT_FIBRE;
2249 } else
b8ff05a9
DM		 2250 cmd->port = PORT_OTHER;
2251
2252 if (p->mdio_addr >= 0) {
2253 cmd->phy_address = p->mdio_addr;
2254 cmd->transceiver = XCVR_EXTERNAL;
2255 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2256 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2257 } else {
2258 cmd->phy_address = 0; /* not really, but no better option */
2259 cmd->transceiver = XCVR_INTERNAL;
2260 cmd->mdio_support = 0;
2261 }
2262
2263 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2264 cmd->advertising = from_fw_linkcaps(p->port_type,
2265 p->link_cfg.advertising);
70739497
DD		 2266 ethtool_cmd_speed_set(cmd,
2267 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2268 cmd->duplex = DUPLEX_FULL;
2269 cmd->autoneg = p->link_cfg.autoneg;
2270 cmd->maxtxpkt = 0;
2271 cmd->maxrxpkt = 0;
2272 return 0;
2273}
2274
2275static unsigned int speed_to_caps(int speed)
2276{
e8b39015 2277 if (speed == 100)
b8ff05a9 2278 return FW_PORT_CAP_SPEED_100M;
e8b39015 2279 if (speed == 1000)
b8ff05a9 2280 return FW_PORT_CAP_SPEED_1G;
e8b39015 2281 if (speed == 10000)
b8ff05a9 2282 return FW_PORT_CAP_SPEED_10G;
e8b39015 2283 if (speed == 40000)
72aca4bf 2284 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2285 return 0;
2286}
2287
2288static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2289{
2290 unsigned int cap;
2291 struct port_info *p = netdev_priv(dev);
2292 struct link_config *lc = &p->link_cfg;
25db0338 2293 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2294
2295 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2296 return -EINVAL;
2297
2298 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2299 /*
2300 * PHY offers a single speed. See if that's what's
2301 * being requested.
2302 */
2303 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2304 (lc->supported & speed_to_caps(speed)))
2305 return 0;
b8ff05a9
DM		 2306 return -EINVAL;
2307 }
2308
2309 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2310 cap = speed_to_caps(speed);
b8ff05a9 2311
72aca4bf 2312 if (!(lc->supported & cap) ||
e8b39015
BH		 2313 (speed == 1000) ||
2314 (speed == 10000) ||
72aca4bf 2315 (speed == 40000))
b8ff05a9
DM		 2316 return -EINVAL;
2317 lc->requested_speed = cap;
2318 lc->advertising = 0;
2319 } else {
2320 cap = to_fw_linkcaps(cmd->advertising);
2321 if (!(lc->supported & cap))
2322 return -EINVAL;
2323 lc->requested_speed = 0;
2324 lc->advertising = cap | FW_PORT_CAP_ANEG;
2325 }
2326 lc->autoneg = cmd->autoneg;
2327
2328 if (netif_running(dev))
060e0c75
DM		 2329 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2330 lc);
b8ff05a9
DM		 2331 return 0;
2332}
2333
2334static void get_pauseparam(struct net_device *dev,
2335 struct ethtool_pauseparam *epause)
2336{
2337 struct port_info *p = netdev_priv(dev);
2338
2339 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2340 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2341 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2342}
2343
2344static int set_pauseparam(struct net_device *dev,
2345 struct ethtool_pauseparam *epause)
2346{
2347 struct port_info *p = netdev_priv(dev);
2348 struct link_config *lc = &p->link_cfg;
2349
2350 if (epause->autoneg == AUTONEG_DISABLE)
2351 lc->requested_fc = 0;
2352 else if (lc->supported & FW_PORT_CAP_ANEG)
2353 lc->requested_fc = PAUSE_AUTONEG;
2354 else
2355 return -EINVAL;
2356
2357 if (epause->rx_pause)
2358 lc->requested_fc |= PAUSE_RX;
2359 if (epause->tx_pause)
2360 lc->requested_fc |= PAUSE_TX;
2361 if (netif_running(dev))
060e0c75
DM		 2362 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2363 lc);
b8ff05a9
DM		 2364 return 0;
2365}
2366
b8ff05a9
DM		 2367static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2368{
2369 const struct port_info *pi = netdev_priv(dev);
2370 const struct sge *s = &pi->adapter->sge;
2371
2372 e->rx_max_pending = MAX_RX_BUFFERS;
2373 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2374 e->rx_jumbo_max_pending = 0;
2375 e->tx_max_pending = MAX_TXQ_ENTRIES;
2376
2377 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2378 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2379 e->rx_jumbo_pending = 0;
2380 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2381}
2382
2383static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2384{
2385 int i;
2386 const struct port_info *pi = netdev_priv(dev);
2387 struct adapter *adapter = pi->adapter;
2388 struct sge *s = &adapter->sge;
2389
2390 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2391 e->tx_pending > MAX_TXQ_ENTRIES ||
2392 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2393 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2394 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2395 return -EINVAL;
2396
2397 if (adapter->flags & FULL_INIT_DONE)
2398 return -EBUSY;
2399
2400 for (i = 0; i < pi->nqsets; ++i) {
2401 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2402 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2403 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2404 }
2405 return 0;
2406}
2407
2408static int closest_timer(const struct sge *s, int time)
2409{
2410 int i, delta, match = 0, min_delta = INT_MAX;
2411
2412 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2413 delta = time - s->timer_val[i];
2414 if (delta < 0)
2415 delta = -delta;
2416 if (delta < min_delta) {
2417 min_delta = delta;
2418 match = i;
2419 }
2420 }
2421 return match;
2422}
2423
2424static int closest_thres(const struct sge *s, int thres)
2425{
2426 int i, delta, match = 0, min_delta = INT_MAX;
2427
2428 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2429 delta = thres - s->counter_val[i];
2430 if (delta < 0)
2431 delta = -delta;
2432 if (delta < min_delta) {
2433 min_delta = delta;
2434 match = i;
2435 }
2436 }
2437 return match;
2438}
2439
2440/*
2441 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2442 */
2443static unsigned int qtimer_val(const struct adapter *adap,
2444 const struct sge_rspq *q)
2445{
2446 unsigned int idx = q->intr_params >> 1;
2447
2448 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2449}
2450
2451/**
2452 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
2453 * @adap: the adapter
2454 * @q: the Rx queue
2455 * @us: the hold-off time in us, or 0 to disable timer
2456 * @cnt: the hold-off packet count, or 0 to disable counter
2457 *
2458 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2459 * one of the two needs to be enabled for the queue to generate interrupts.
2460 */
2461static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
2462 unsigned int us, unsigned int cnt)
2463{
2464 if ((us | cnt) == 0)
2465 cnt = 1;
2466
2467 if (cnt) {
2468 int err;
2469 u32 v, new_idx;
2470
2471 new_idx = closest_thres(&adap->sge, cnt);
2472 if (q->desc && q->pktcnt_idx != new_idx) {
2473 /* the queue has already been created, update it */
2474 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2475 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2476 FW_PARAMS_PARAM_YZ(q->cntxt_id);
060e0c75
DM		 2477 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2478 &new_idx);
b8ff05a9
DM		 2479 if (err)
2480 return err;
2481 }
2482 q->pktcnt_idx = new_idx;
2483 }
2484
2485 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2486 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2487 return 0;
2488}
2489
2490static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2491{
2492 const struct port_info *pi = netdev_priv(dev);
2493 struct adapter *adap = pi->adapter;
d4fc9dc2
TLSC		 2494 struct sge_rspq *q;
2495 int i;
2496 int r = 0;
2497
2498 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2499 q = &adap->sge.ethrxq[i].rspq;
2500 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2501 c->rx_max_coalesced_frames);
2502 if (r) {
2503 dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2504 break;
2505 }
2506 }
2507 return r;
b8ff05a9
DM		 2508}
2509
2510static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2511{
2512 const struct port_info *pi = netdev_priv(dev);
2513 const struct adapter *adap = pi->adapter;
2514 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2515
2516 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2517 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2518 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2519 return 0;
2520}
2521
1478b3ee
DM		 2522/**
2523 * eeprom_ptov - translate a physical EEPROM address to virtual
2524 * @phys_addr: the physical EEPROM address
2525 * @fn: the PCI function number
2526 * @sz: size of function-specific area
2527 *
2528 * Translate a physical EEPROM address to virtual. The first 1K is
2529 * accessed through virtual addresses starting at 31K, the rest is
2530 * accessed through virtual addresses starting at 0.
2531 *
2532 * The mapping is as follows:
2533 * [0..1K) -> [31K..32K)
2534 * [1K..1K+A) -> [31K-A..31K)
2535 * [1K+A..ES) -> [0..ES-A-1K)
2536 *
2537 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2538 */
1478b3ee 2539static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2540{
1478b3ee 2541 fn *= sz;
b8ff05a9
DM		 2542 if (phys_addr < 1024)
2543 return phys_addr + (31 << 10);
1478b3ee
DM		 2544 if (phys_addr < 1024 + fn)
2545 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2546 if (phys_addr < EEPROMSIZE)
1478b3ee 2547 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2548 return -EINVAL;
2549}
2550
2551/*
2552 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2553 */
2554static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2555{
1478b3ee 2556 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2557
2558 if (vaddr >= 0)
2559 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2560 return vaddr < 0 ? vaddr : 0;
2561}
2562
2563static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2564{
1478b3ee 2565 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2566
2567 if (vaddr >= 0)
2568 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2569 return vaddr < 0 ? vaddr : 0;
2570}
2571
2572#define EEPROM_MAGIC 0x38E2F10C
2573
2574static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2575 u8 *data)
2576{
2577 int i, err = 0;
2578 struct adapter *adapter = netdev2adap(dev);
2579
2580 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2581 if (!buf)
2582 return -ENOMEM;
2583
2584 e->magic = EEPROM_MAGIC;
2585 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2586 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2587
2588 if (!err)
2589 memcpy(data, buf + e->offset, e->len);
2590 kfree(buf);
2591 return err;
2592}
2593
2594static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2595 u8 *data)
2596{
2597 u8 *buf;
2598 int err = 0;
2599 u32 aligned_offset, aligned_len, *p;
2600 struct adapter *adapter = netdev2adap(dev);
2601
2602 if (eeprom->magic != EEPROM_MAGIC)
2603 return -EINVAL;
2604
2605 aligned_offset = eeprom->offset & ~3;
2606 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2607
1478b3ee
DM		 2608 if (adapter->fn > 0) {
2609 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2610
2611 if (aligned_offset < start ||
2612 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2613 return -EPERM;
2614 }
2615
b8ff05a9
DM		 2616 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2617 /*
2618 * RMW possibly needed for first or last words.
2619 */
2620 buf = kmalloc(aligned_len, GFP_KERNEL);
2621 if (!buf)
2622 return -ENOMEM;
2623 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2624 if (!err && aligned_len > 4)
2625 err = eeprom_rd_phys(adapter,
2626 aligned_offset + aligned_len - 4,
2627 (u32 *)&buf[aligned_len - 4]);
2628 if (err)
2629 goto out;
2630 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2631 } else
2632 buf = data;
2633
2634 err = t4_seeprom_wp(adapter, false);
2635 if (err)
2636 goto out;
2637
2638 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2639 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2640 aligned_offset += 4;
2641 }
2642
2643 if (!err)
2644 err = t4_seeprom_wp(adapter, true);
2645out:
2646 if (buf != data)
2647 kfree(buf);
2648 return err;
2649}
2650
2651static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2652{
2653 int ret;
2654 const struct firmware *fw;
2655 struct adapter *adap = netdev2adap(netdev);
2656
2657 ef->data[sizeof(ef->data) - 1] = '\0';
2658 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2659 if (ret < 0)
2660 return ret;
2661
2662 ret = t4_load_fw(adap, fw->data, fw->size);
2663 release_firmware(fw);
2664 if (!ret)
2665 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2666 return ret;
2667}
2668
2669#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2670#define BCAST_CRC 0xa0ccc1a6
2671
2672static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2673{
2674 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2675 wol->wolopts = netdev2adap(dev)->wol;
2676 memset(&wol->sopass, 0, sizeof(wol->sopass));
2677}
2678
2679static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2680{
2681 int err = 0;
2682 struct port_info *pi = netdev_priv(dev);
2683
2684 if (wol->wolopts & ~WOL_SUPPORTED)
2685 return -EINVAL;
2686 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2687 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2688 if (wol->wolopts & WAKE_BCAST) {
2689 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2690 ~0ULL, 0, false);
2691 if (!err)
2692 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2693 ~6ULL, ~0ULL, BCAST_CRC, true);
2694 } else
2695 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2696 return err;
2697}
2698
c8f44aff 2699static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2700{
2ed28baa 2701 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2702 netdev_features_t changed = dev->features ^ features;
19ecae2c 2703 int err;
19ecae2c 2704
f646968f 2705 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2706 return 0;
19ecae2c 2707
2ed28baa
MM		 2708 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2709 -1, -1, -1,
f646968f 2710 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2711 if (unlikely(err))
f646968f 2712 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2713 return err;
87b6cf51
DM		 2714}
2715
7850f63f 2716static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2717{
2718 const struct port_info *pi = netdev_priv(dev);
671b0060 2719
7850f63f
BH		 2720 return pi->rss_size;
2721}
2722
2723static int get_rss_table(struct net_device *dev, u32 *p)
2724{
2725 const struct port_info *pi = netdev_priv(dev);
2726 unsigned int n = pi->rss_size;
2727
671b0060 2728 while (n--)
7850f63f 2729 p[n] = pi->rss[n];
671b0060
DM		 2730 return 0;
2731}
2732
7850f63f 2733static int set_rss_table(struct net_device *dev, const u32 *p)
671b0060
DM		 2734{
2735 unsigned int i;
2736 struct port_info *pi = netdev_priv(dev);
2737
7850f63f
BH		 2738 for (i = 0; i < pi->rss_size; i++)
2739 pi->rss[i] = p[i];
671b0060
DM		 2740 if (pi->adapter->flags & FULL_INIT_DONE)
2741 return write_rss(pi, pi->rss);
2742 return 0;
2743}
2744
2745static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2746 u32 *rules)
671b0060 2747{
f796564a
DM		 2748 const struct port_info *pi = netdev_priv(dev);
2749
671b0060 2750 switch (info->cmd) {
f796564a
DM		 2751 case ETHTOOL_GRXFH: {
2752 unsigned int v = pi->rss_mode;
2753
2754 info->data = 0;
2755 switch (info->flow_type) {
2756 case TCP_V4_FLOW:
2757 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2758 info->data = RXH_IP_SRC | RXH_IP_DST |
2759 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2760 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2761 info->data = RXH_IP_SRC | RXH_IP_DST;
2762 break;
2763 case UDP_V4_FLOW:
2764 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2765 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2766 info->data = RXH_IP_SRC | RXH_IP_DST |
2767 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2768 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2769 info->data = RXH_IP_SRC | RXH_IP_DST;
2770 break;
2771 case SCTP_V4_FLOW:
2772 case AH_ESP_V4_FLOW:
2773 case IPV4_FLOW:
2774 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2775 info->data = RXH_IP_SRC | RXH_IP_DST;
2776 break;
2777 case TCP_V6_FLOW:
2778 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2779 info->data = RXH_IP_SRC | RXH_IP_DST |
2780 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2781 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2782 info->data = RXH_IP_SRC | RXH_IP_DST;
2783 break;
2784 case UDP_V6_FLOW:
2785 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2786 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2787 info->data = RXH_IP_SRC | RXH_IP_DST |
2788 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2789 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2790 info->data = RXH_IP_SRC | RXH_IP_DST;
2791 break;
2792 case SCTP_V6_FLOW:
2793 case AH_ESP_V6_FLOW:
2794 case IPV6_FLOW:
2795 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2796 info->data = RXH_IP_SRC | RXH_IP_DST;
2797 break;
2798 }
2799 return 0;
2800 }
671b0060 2801 case ETHTOOL_GRXRINGS:
f796564a 2802 info->data = pi->nqsets;
671b0060
DM		 2803 return 0;
2804 }
2805 return -EOPNOTSUPP;
2806}
2807
9b07be4b 2808static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 2809 .get_settings = get_settings,
2810 .set_settings = set_settings,
2811 .get_drvinfo = get_drvinfo,
2812 .get_msglevel = get_msglevel,
2813 .set_msglevel = set_msglevel,
2814 .get_ringparam = get_sge_param,
2815 .set_ringparam = set_sge_param,
2816 .get_coalesce = get_coalesce,
2817 .set_coalesce = set_coalesce,
2818 .get_eeprom_len = get_eeprom_len,
2819 .get_eeprom = get_eeprom,
2820 .set_eeprom = set_eeprom,
2821 .get_pauseparam = get_pauseparam,
2822 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2823 .get_link = ethtool_op_get_link,
2824 .get_strings = get_strings,
c5e06360 2825 .set_phys_id = identify_port,
b8ff05a9
DM		 2826 .nway_reset = restart_autoneg,
2827 .get_sset_count = get_sset_count,
2828 .get_ethtool_stats = get_stats,
2829 .get_regs_len = get_regs_len,
2830 .get_regs = get_regs,
2831 .get_wol = get_wol,
2832 .set_wol = set_wol,
671b0060 2833 .get_rxnfc = get_rxnfc,
7850f63f 2834 .get_rxfh_indir_size = get_rss_table_size,
671b0060
DM		 2835 .get_rxfh_indir = get_rss_table,
2836 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 2837 .flash_device = set_flash,
2838};
2839
2840/*
2841 * debugfs support
2842 */
b8ff05a9
DM		 2843static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2844 loff_t *ppos)
2845{
2846 loff_t pos = *ppos;
496ad9aa 2847 loff_t avail = file_inode(file)->i_size;
b8ff05a9
DM		 2848 unsigned int mem = (uintptr_t)file->private_data & 3;
2849 struct adapter *adap = file->private_data - mem;
2850
2851 if (pos < 0)
2852 return -EINVAL;
2853 if (pos >= avail)
2854 return 0;
2855 if (count > avail - pos)
2856 count = avail - pos;
2857
2858 while (count) {
2859 size_t len;
2860 int ret, ofst;
2861 __be32 data[16];
2862
19dd37ba
SR		 2863 if ((mem == MEM_MC) || (mem == MEM_MC1))
2864 ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);
b8ff05a9
DM		 2865 else
2866 ret = t4_edc_read(adap, mem, pos, data, NULL);
2867 if (ret)
2868 return ret;
2869
2870 ofst = pos % sizeof(data);
2871 len = min(count, sizeof(data) - ofst);
2872 if (copy_to_user(buf, (u8 *)data + ofst, len))
2873 return -EFAULT;
2874
2875 buf += len;
2876 pos += len;
2877 count -= len;
2878 }
2879 count = pos - *ppos;
2880 *ppos = pos;
2881 return count;
2882}
2883
2884static const struct file_operations mem_debugfs_fops = {
2885 .owner = THIS_MODULE,
234e3405 2886 .open = simple_open,
b8ff05a9 2887 .read = mem_read,
6038f373 2888 .llseek = default_llseek,
b8ff05a9
DM		 2889};
2890
91744948 2891static void add_debugfs_mem(struct adapter *adap, const char *name,
1dd06ae8 2892 unsigned int idx, unsigned int size_mb)
b8ff05a9
DM		 2893{
2894 struct dentry *de;
2895
2896 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2897 (void *)adap + idx, &mem_debugfs_fops);
2898 if (de && de->d_inode)
2899 de->d_inode->i_size = size_mb << 20;
2900}
2901
91744948 2902static int setup_debugfs(struct adapter *adap)
b8ff05a9
DM		 2903{
2904 int i;
19dd37ba 2905 u32 size;
b8ff05a9
DM		 2906
2907 if (IS_ERR_OR_NULL(adap->debugfs_root))
2908 return -1;
2909
2910 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
19dd37ba
SR		 2911 if (i & EDRAM0_ENABLE) {
2912 size = t4_read_reg(adap, MA_EDRAM0_BAR);
2913 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));
2914 }
2915 if (i & EDRAM1_ENABLE) {
2916 size = t4_read_reg(adap, MA_EDRAM1_BAR);
2917 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));
2918 }
d14807dd 2919 if (is_t4(adap->params.chip)) {
19dd37ba
SR		 2920 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2921 if (i & EXT_MEM_ENABLE)
2922 add_debugfs_mem(adap, "mc", MEM_MC,
2923 EXT_MEM_SIZE_GET(size));
2924 } else {
2925 if (i & EXT_MEM_ENABLE) {
2926 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2927 add_debugfs_mem(adap, "mc0", MEM_MC0,
2928 EXT_MEM_SIZE_GET(size));
2929 }
2930 if (i & EXT_MEM1_ENABLE) {
2931 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);
2932 add_debugfs_mem(adap, "mc1", MEM_MC1,
2933 EXT_MEM_SIZE_GET(size));
2934 }
2935 }
b8ff05a9
DM		 2936 if (adap->l2t)
2937 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2938 &t4_l2t_fops);
2939 return 0;
2940}
2941
2942/*
2943 * upper-layer driver support
2944 */
2945
2946/*
2947 * Allocate an active-open TID and set it to the supplied value.
2948 */
2949int cxgb4_alloc_atid(struct tid_info *t, void *data)
2950{
2951 int atid = -1;
2952
2953 spin_lock_bh(&t->atid_lock);
2954 if (t->afree) {
2955 union aopen_entry *p = t->afree;
2956
f2b7e78d 2957 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 2958 t->afree = p->next;
2959 p->data = data;
2960 t->atids_in_use++;
2961 }
2962 spin_unlock_bh(&t->atid_lock);
2963 return atid;
2964}
2965EXPORT_SYMBOL(cxgb4_alloc_atid);
2966
2967/*
2968 * Release an active-open TID.
2969 */
2970void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2971{
f2b7e78d 2972 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 2973
2974 spin_lock_bh(&t->atid_lock);
2975 p->next = t->afree;
2976 t->afree = p;
2977 t->atids_in_use--;
2978 spin_unlock_bh(&t->atid_lock);
2979}
2980EXPORT_SYMBOL(cxgb4_free_atid);
2981
2982/*
2983 * Allocate a server TID and set it to the supplied value.
2984 */
2985int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2986{
2987 int stid;
2988
2989 spin_lock_bh(&t->stid_lock);
2990 if (family == PF_INET) {
2991 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2992 if (stid < t->nstids)
2993 __set_bit(stid, t->stid_bmap);
2994 else
2995 stid = -1;
2996 } else {
2997 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2998 if (stid < 0)
2999 stid = -1;
3000 }
3001 if (stid >= 0) {
3002 t->stid_tab[stid].data = data;
3003 stid += t->stid_base;
15f63b74
KS		 3004 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3005 * This is equivalent to 4 TIDs. With CLIP enabled it
3006 * needs 2 TIDs.
3007 */
3008 if (family == PF_INET)
3009 t->stids_in_use++;
3010 else
3011 t->stids_in_use += 4;
b8ff05a9
DM		 3012 }
3013 spin_unlock_bh(&t->stid_lock);
3014 return stid;
3015}
3016EXPORT_SYMBOL(cxgb4_alloc_stid);
3017
dca4faeb
VP		 3018/* Allocate a server filter TID and set it to the supplied value.
3019 */
3020int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3021{
3022 int stid;
3023
3024 spin_lock_bh(&t->stid_lock);
3025 if (family == PF_INET) {
3026 stid = find_next_zero_bit(t->stid_bmap,
3027 t->nstids + t->nsftids, t->nstids);
3028 if (stid < (t->nstids + t->nsftids))
3029 __set_bit(stid, t->stid_bmap);
3030 else
3031 stid = -1;
3032 } else {
3033 stid = -1;
3034 }
3035 if (stid >= 0) {
3036 t->stid_tab[stid].data = data;
470c60c4
KS		 3037 stid -= t->nstids;
3038 stid += t->sftid_base;
dca4faeb
VP		 3039 t->stids_in_use++;
3040 }
3041 spin_unlock_bh(&t->stid_lock);
3042 return stid;
3043}
3044EXPORT_SYMBOL(cxgb4_alloc_sftid);
3045
3046/* Release a server TID.
b8ff05a9
DM		 3047 */
3048void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3049{
470c60c4
KS		 3050 /* Is it a server filter TID? */
3051 if (t->nsftids && (stid >= t->sftid_base)) {
3052 stid -= t->sftid_base;
3053 stid += t->nstids;
3054 } else {
3055 stid -= t->stid_base;
3056 }
3057
b8ff05a9
DM		 3058 spin_lock_bh(&t->stid_lock);
3059 if (family == PF_INET)
3060 __clear_bit(stid, t->stid_bmap);
3061 else
3062 bitmap_release_region(t->stid_bmap, stid, 2);
3063 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3064 if (family == PF_INET)
3065 t->stids_in_use--;
3066 else
3067 t->stids_in_use -= 4;
b8ff05a9
DM		 3068 spin_unlock_bh(&t->stid_lock);
3069}
3070EXPORT_SYMBOL(cxgb4_free_stid);
3071
3072/*
3073 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3074 */
3075static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3076 unsigned int tid)
3077{
3078 struct cpl_tid_release *req;
3079
3080 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3081 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3082 INIT_TP_WR(req, tid);
3083 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3084}
3085
3086/*
3087 * Queue a TID release request and if necessary schedule a work queue to
3088 * process it.
3089 */
31b9c19b 3090static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3091 unsigned int tid)
b8ff05a9
DM		 3092{
3093 void **p = &t->tid_tab[tid];
3094 struct adapter *adap = container_of(t, struct adapter, tids);
3095
3096 spin_lock_bh(&adap->tid_release_lock);
3097 *p = adap->tid_release_head;
3098 /* Low 2 bits encode the Tx channel number */
3099 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3100 if (!adap->tid_release_task_busy) {
3101 adap->tid_release_task_busy = true;
3069ee9b 3102 queue_work(workq, &adap->tid_release_task);
b8ff05a9
DM		 3103 }
3104 spin_unlock_bh(&adap->tid_release_lock);
3105}
b8ff05a9
DM		 3106
3107/*
3108 * Process the list of pending TID release requests.
3109 */
3110static void process_tid_release_list(struct work_struct *work)
3111{
3112 struct sk_buff *skb;
3113 struct adapter *adap;
3114
3115 adap = container_of(work, struct adapter, tid_release_task);
3116
3117 spin_lock_bh(&adap->tid_release_lock);
3118 while (adap->tid_release_head) {
3119 void **p = adap->tid_release_head;
3120 unsigned int chan = (uintptr_t)p & 3;
3121 p = (void *)p - chan;
3122
3123 adap->tid_release_head = *p;
3124 *p = NULL;
3125 spin_unlock_bh(&adap->tid_release_lock);
3126
3127 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3128 GFP_KERNEL)))
3129 schedule_timeout_uninterruptible(1);
3130
3131 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3132 t4_ofld_send(adap, skb);
3133 spin_lock_bh(&adap->tid_release_lock);
3134 }
3135 adap->tid_release_task_busy = false;
3136 spin_unlock_bh(&adap->tid_release_lock);
3137}
3138
3139/*
3140 * Release a TID and inform HW. If we are unable to allocate the release
3141 * message we defer to a work queue.
3142 */
3143void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3144{
3145 void *old;
3146 struct sk_buff *skb;
3147 struct adapter *adap = container_of(t, struct adapter, tids);
3148
3149 old = t->tid_tab[tid];
3150 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3151 if (likely(skb)) {
3152 t->tid_tab[tid] = NULL;
3153 mk_tid_release(skb, chan, tid);
3154 t4_ofld_send(adap, skb);
3155 } else
3156 cxgb4_queue_tid_release(t, chan, tid);
3157 if (old)
3158 atomic_dec(&t->tids_in_use);
3159}
3160EXPORT_SYMBOL(cxgb4_remove_tid);
3161
3162/*
3163 * Allocate and initialize the TID tables. Returns 0 on success.
3164 */
3165static int tid_init(struct tid_info *t)
3166{
3167 size_t size;
f2b7e78d 3168 unsigned int stid_bmap_size;
b8ff05a9 3169 unsigned int natids = t->natids;
b6f8eaec 3170 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3171
dca4faeb 3172 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3173 size = t->ntids * sizeof(*t->tid_tab) +
3174 natids * sizeof(*t->atid_tab) +
b8ff05a9 3175 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3176 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3177 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3178 t->nftids * sizeof(*t->ftid_tab) +
3179 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3180
b8ff05a9
DM		 3181 t->tid_tab = t4_alloc_mem(size);
3182 if (!t->tid_tab)
3183 return -ENOMEM;
3184
3185 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3186 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3187 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3188 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3189 spin_lock_init(&t->stid_lock);
3190 spin_lock_init(&t->atid_lock);
3191
3192 t->stids_in_use = 0;
3193 t->afree = NULL;
3194 t->atids_in_use = 0;
3195 atomic_set(&t->tids_in_use, 0);
3196
3197 /* Setup the free list for atid_tab and clear the stid bitmap. */
3198 if (natids) {
3199 while (--natids)
3200 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3201 t->afree = t->atid_tab;
3202 }
dca4faeb 3203 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3204 /* Reserve stid 0 for T4/T5 adapters */
3205 if (!t->stid_base &&
3206 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3207 __set_bit(0, t->stid_bmap);
3208
b8ff05a9
DM		 3209 return 0;
3210}
3211
01bcca68
VP		 3212static int cxgb4_clip_get(const struct net_device *dev,
3213 const struct in6_addr *lip)
3214{
3215 struct adapter *adap;
3216 struct fw_clip_cmd c;
3217
3218 adap = netdev2adap(dev);
3219 memset(&c, 0, sizeof(c));
3220 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3221 FW_CMD_REQUEST | FW_CMD_WRITE);
3222 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
3223 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);
3224 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3225 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3226}
3227
3228static int cxgb4_clip_release(const struct net_device *dev,
3229 const struct in6_addr *lip)
3230{
3231 struct adapter *adap;
3232 struct fw_clip_cmd c;
3233
3234 adap = netdev2adap(dev);
3235 memset(&c, 0, sizeof(c));
3236 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3237 FW_CMD_REQUEST | FW_CMD_READ);
3238 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
3239 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);
3240 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3241 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3242}
3243
b8ff05a9
DM		 3244/**
3245 * cxgb4_create_server - create an IP server
3246 * @dev: the device
3247 * @stid: the server TID
3248 * @sip: local IP address to bind server to
3249 * @sport: the server's TCP port
3250 * @queue: queue to direct messages from this server to
3251 *
3252 * Create an IP server for the given port and address.
3253 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3254 */
3255int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3256 __be32 sip, __be16 sport, __be16 vlan,
3257 unsigned int queue)
b8ff05a9
DM		 3258{
3259 unsigned int chan;
3260 struct sk_buff *skb;
3261 struct adapter *adap;
3262 struct cpl_pass_open_req *req;
80f40c1f 3263 int ret;
b8ff05a9
DM		 3264
3265 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3266 if (!skb)
3267 return -ENOMEM;
3268
3269 adap = netdev2adap(dev);
3270 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3271 INIT_TP_WR(req, 0);
3272 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3273 req->local_port = sport;
3274 req->peer_port = htons(0);
3275 req->local_ip = sip;
3276 req->peer_ip = htonl(0);
e46dab4d 3277 chan = rxq_to_chan(&adap->sge, queue);
b8ff05a9
DM		 3278 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3279 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3280 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
80f40c1f
VP		 3281 ret = t4_mgmt_tx(adap, skb);
3282 return net_xmit_eval(ret);
b8ff05a9
DM		 3283}
3284EXPORT_SYMBOL(cxgb4_create_server);
3285
80f40c1f
VP		 3286/* cxgb4_create_server6 - create an IPv6 server
3287 * @dev: the device
3288 * @stid: the server TID
3289 * @sip: local IPv6 address to bind server to
3290 * @sport: the server's TCP port
3291 * @queue: queue to direct messages from this server to
3292 *
3293 * Create an IPv6 server for the given port and address.
3294 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3295 */
3296int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3297 const struct in6_addr *sip, __be16 sport,
3298 unsigned int queue)
3299{
3300 unsigned int chan;
3301 struct sk_buff *skb;
3302 struct adapter *adap;
3303 struct cpl_pass_open_req6 *req;
3304 int ret;
3305
3306 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3307 if (!skb)
3308 return -ENOMEM;
3309
3310 adap = netdev2adap(dev);
3311 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3312 INIT_TP_WR(req, 0);
3313 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3314 req->local_port = sport;
3315 req->peer_port = htons(0);
3316 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3317 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3318 req->peer_ip_hi = cpu_to_be64(0);
3319 req->peer_ip_lo = cpu_to_be64(0);
3320 chan = rxq_to_chan(&adap->sge, queue);
3321 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3322 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3323 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3324 ret = t4_mgmt_tx(adap, skb);
3325 return net_xmit_eval(ret);
3326}
3327EXPORT_SYMBOL(cxgb4_create_server6);
3328
3329int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3330 unsigned int queue, bool ipv6)
3331{
3332 struct sk_buff *skb;
3333 struct adapter *adap;
3334 struct cpl_close_listsvr_req *req;
3335 int ret;
3336
3337 adap = netdev2adap(dev);
3338
3339 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3340 if (!skb)
3341 return -ENOMEM;
3342
3343 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3344 INIT_TP_WR(req, 0);
3345 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3346 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3347 LISTSVR_IPV6(0)) | QUEUENO(queue));
3348 ret = t4_mgmt_tx(adap, skb);
3349 return net_xmit_eval(ret);
3350}
3351EXPORT_SYMBOL(cxgb4_remove_server);
3352
b8ff05a9
DM		 3353/**
3354 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3355 * @mtus: the HW MTU table
3356 * @mtu: the target MTU
3357 * @idx: index of selected entry in the MTU table
3358 *
3359 * Returns the index and the value in the HW MTU table that is closest to
3360 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3361 * table, in which case that smallest available value is selected.
3362 */
3363unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3364 unsigned int *idx)
3365{
3366 unsigned int i = 0;
3367
3368 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3369 ++i;
3370 if (idx)
3371 *idx = i;
3372 return mtus[i];
3373}
3374EXPORT_SYMBOL(cxgb4_best_mtu);
3375
3376/**
3377 * cxgb4_port_chan - get the HW channel of a port
3378 * @dev: the net device for the port
3379 *
3380 * Return the HW Tx channel of the given port.
3381 */
3382unsigned int cxgb4_port_chan(const struct net_device *dev)
3383{
3384 return netdev2pinfo(dev)->tx_chan;
3385}
3386EXPORT_SYMBOL(cxgb4_port_chan);
3387
881806bc
VP		 3388unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3389{
3390 struct adapter *adap = netdev2adap(dev);
2cc301d2 3391 u32 v1, v2, lp_count, hp_count;
881806bc 3392
2cc301d2
SR		 3393 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3394 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
d14807dd 3395 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3396 lp_count = G_LP_COUNT(v1);
3397 hp_count = G_HP_COUNT(v1);
3398 } else {
3399 lp_count = G_LP_COUNT_T5(v1);
3400 hp_count = G_HP_COUNT_T5(v2);
3401 }
3402 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3403}
3404EXPORT_SYMBOL(cxgb4_dbfifo_count);
3405
b8ff05a9
DM		 3406/**
3407 * cxgb4_port_viid - get the VI id of a port
3408 * @dev: the net device for the port
3409 *
3410 * Return the VI id of the given port.
3411 */
3412unsigned int cxgb4_port_viid(const struct net_device *dev)
3413{
3414 return netdev2pinfo(dev)->viid;
3415}
3416EXPORT_SYMBOL(cxgb4_port_viid);
3417
3418/**
3419 * cxgb4_port_idx - get the index of a port
3420 * @dev: the net device for the port
3421 *
3422 * Return the index of the given port.
3423 */
3424unsigned int cxgb4_port_idx(const struct net_device *dev)
3425{
3426 return netdev2pinfo(dev)->port_id;
3427}
3428EXPORT_SYMBOL(cxgb4_port_idx);
3429
b8ff05a9
DM		 3430void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3431 struct tp_tcp_stats *v6)
3432{
3433 struct adapter *adap = pci_get_drvdata(pdev);
3434
3435 spin_lock(&adap->stats_lock);
3436 t4_tp_get_tcp_stats(adap, v4, v6);
3437 spin_unlock(&adap->stats_lock);
3438}
3439EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3440
3441void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3442 const unsigned int *pgsz_order)
3443{
3444 struct adapter *adap = netdev2adap(dev);
3445
3446 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3447 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3448 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3449 HPZ3(pgsz_order[3]));
3450}
3451EXPORT_SYMBOL(cxgb4_iscsi_init);
3452
3069ee9b
VP		 3453int cxgb4_flush_eq_cache(struct net_device *dev)
3454{
3455 struct adapter *adap = netdev2adap(dev);
3456 int ret;
3457
3458 ret = t4_fwaddrspace_write(adap, adap->mbox,
3459 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3460 return ret;
3461}
3462EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3463
3464static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3465{
3466 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3467 __be64 indices;
3468 int ret;
3469
3470 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
3471 if (!ret) {
404d9e3f
VP		 3472 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3473 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3474 }
3475 return ret;
3476}
3477
3478int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3479 u16 size)
3480{
3481 struct adapter *adap = netdev2adap(dev);
3482 u16 hw_pidx, hw_cidx;
3483 int ret;
3484
3485 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3486 if (ret)
3487 goto out;
3488
3489 if (pidx != hw_pidx) {
3490 u16 delta;
3491
3492 if (pidx >= hw_pidx)
3493 delta = pidx - hw_pidx;
3494 else
3495 delta = size - hw_pidx + pidx;
3496 wmb();
840f3000
VP		 3497 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3498 QID(qid) | PIDX(delta));
3069ee9b
VP		 3499 }
3500out:
3501 return ret;
3502}
3503EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3504
3cbdb928
VP		 3505void cxgb4_disable_db_coalescing(struct net_device *dev)
3506{
3507 struct adapter *adap;
3508
3509 adap = netdev2adap(dev);
3510 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3511 F_NOCOALESCE);
3512}
3513EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3514
3515void cxgb4_enable_db_coalescing(struct net_device *dev)
3516{
3517 struct adapter *adap;
3518
3519 adap = netdev2adap(dev);
3520 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3521}
3522EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3523
b8ff05a9
DM		 3524static struct pci_driver cxgb4_driver;
3525
3526static void check_neigh_update(struct neighbour *neigh)
3527{
3528 const struct device *parent;
3529 const struct net_device *netdev = neigh->dev;
3530
3531 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3532 netdev = vlan_dev_real_dev(netdev);
3533 parent = netdev->dev.parent;
3534 if (parent && parent->driver == &cxgb4_driver.driver)
3535 t4_l2t_update(dev_get_drvdata(parent), neigh);
3536}
3537
3538static int netevent_cb(struct notifier_block *nb, unsigned long event,
3539 void *data)
3540{
3541 switch (event) {
3542 case NETEVENT_NEIGH_UPDATE:
3543 check_neigh_update(data);
3544 break;
b8ff05a9
DM		 3545 case NETEVENT_REDIRECT:
3546 default:
3547 break;
3548 }
3549 return 0;
3550}
3551
3552static bool netevent_registered;
3553static struct notifier_block cxgb4_netevent_nb = {
3554 .notifier_call = netevent_cb
3555};
3556
3069ee9b
VP		 3557static void drain_db_fifo(struct adapter *adap, int usecs)
3558{
2cc301d2 3559 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3560
3561 do {
2cc301d2
SR		 3562 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3563 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
d14807dd 3564 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3565 lp_count = G_LP_COUNT(v1);
3566 hp_count = G_HP_COUNT(v1);
3567 } else {
3568 lp_count = G_LP_COUNT_T5(v1);
3569 hp_count = G_HP_COUNT_T5(v2);
3570 }
3571
3572 if (lp_count == 0 && hp_count == 0)
3573 break;
3069ee9b
VP		 3574 set_current_state(TASK_UNINTERRUPTIBLE);
3575 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3576 } while (1);
3577}
3578
3579static void disable_txq_db(struct sge_txq *q)
3580{
05eb2389
SW		 3581 unsigned long flags;
3582
3583 spin_lock_irqsave(&q->db_lock, flags);
3069ee9b 3584 q->db_disabled = 1;
05eb2389 3585 spin_unlock_irqrestore(&q->db_lock, flags);
3069ee9b
VP		 3586}
3587
05eb2389 3588static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3069ee9b
VP		 3589{
3590 spin_lock_irq(&q->db_lock);
05eb2389
SW		 3591 if (q->db_pidx_inc) {
3592 /* Make sure that all writes to the TX descriptors
3593 * are committed before we tell HW about them.
3594 */
3595 wmb();
3596 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3597 QID(q->cntxt_id) | PIDX(q->db_pidx_inc));
3598 q->db_pidx_inc = 0;
3599 }
3069ee9b
VP		 3600 q->db_disabled = 0;
3601 spin_unlock_irq(&q->db_lock);
3602}
3603
3604static void disable_dbs(struct adapter *adap)
3605{
3606 int i;
3607
3608 for_each_ethrxq(&adap->sge, i)
3609 disable_txq_db(&adap->sge.ethtxq[i].q);
3610 for_each_ofldrxq(&adap->sge, i)
3611 disable_txq_db(&adap->sge.ofldtxq[i].q);
3612 for_each_port(adap, i)
3613 disable_txq_db(&adap->sge.ctrlq[i].q);
3614}
3615
3616static void enable_dbs(struct adapter *adap)
3617{
3618 int i;
3619
3620 for_each_ethrxq(&adap->sge, i)
05eb2389 3621 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3069ee9b 3622 for_each_ofldrxq(&adap->sge, i)
05eb2389 3623 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3069ee9b 3624 for_each_port(adap, i)
05eb2389
SW		 3625 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3626}
3627
3628static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3629{
3630 if (adap->uld_handle[CXGB4_ULD_RDMA])
3631 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3632 cmd);
3633}
3634
3635static void process_db_full(struct work_struct *work)
3636{
3637 struct adapter *adap;
3638
3639 adap = container_of(work, struct adapter, db_full_task);
3640
3641 drain_db_fifo(adap, dbfifo_drain_delay);
3642 enable_dbs(adap);
3643 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3644 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3645 DBFIFO_HP_INT | DBFIFO_LP_INT,
3646 DBFIFO_HP_INT | DBFIFO_LP_INT);
3069ee9b
VP		 3647}
3648
3649static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3650{
3651 u16 hw_pidx, hw_cidx;
3652 int ret;
3653
05eb2389 3654 spin_lock_irq(&q->db_lock);
3069ee9b
VP		 3655 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3656 if (ret)
3657 goto out;
3658 if (q->db_pidx != hw_pidx) {
3659 u16 delta;
3660
3661 if (q->db_pidx >= hw_pidx)
3662 delta = q->db_pidx - hw_pidx;
3663 else
3664 delta = q->size - hw_pidx + q->db_pidx;
3665 wmb();
840f3000
VP		 3666 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3667 QID(q->cntxt_id) | PIDX(delta));
3069ee9b
VP		 3668 }
3669out:
3670 q->db_disabled = 0;
05eb2389
SW		 3671 q->db_pidx_inc = 0;
3672 spin_unlock_irq(&q->db_lock);
3069ee9b
VP		 3673 if (ret)
3674 CH_WARN(adap, "DB drop recovery failed.\n");
3675}
3676static void recover_all_queues(struct adapter *adap)
3677{
3678 int i;
3679
3680 for_each_ethrxq(&adap->sge, i)
3681 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3682 for_each_ofldrxq(&adap->sge, i)
3683 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3684 for_each_port(adap, i)
3685 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3686}
3687
881806bc
VP		 3688static void process_db_drop(struct work_struct *work)
3689{
3690 struct adapter *adap;
881806bc 3691
3069ee9b 3692 adap = container_of(work, struct adapter, db_drop_task);
881806bc 3693
d14807dd 3694 if (is_t4(adap->params.chip)) {
05eb2389 3695 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3696 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
05eb2389 3697 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3698 recover_all_queues(adap);
05eb2389 3699 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3700 enable_dbs(adap);
05eb2389 3701 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2cc301d2
SR		 3702 } else {
3703 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3704 u16 qid = (dropped_db >> 15) & 0x1ffff;
3705 u16 pidx_inc = dropped_db & 0x1fff;
3706 unsigned int s_qpp;
3707 unsigned short udb_density;
3708 unsigned long qpshift;
3709 int page;
3710 u32 udb;
3711
3712 dev_warn(adap->pdev_dev,
3713 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3714 dropped_db, qid,
3715 (dropped_db >> 14) & 1,
3716 (dropped_db >> 13) & 1,
3717 pidx_inc);
3718
3719 drain_db_fifo(adap, 1);
3720
3721 s_qpp = QUEUESPERPAGEPF1 * adap->fn;
3722 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,
3723 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
3724 qpshift = PAGE_SHIFT - ilog2(udb_density);
3725 udb = qid << qpshift;
3726 udb &= PAGE_MASK;
3727 page = udb / PAGE_SIZE;
3728 udb += (qid - (page * udb_density)) * 128;
3729
3730 writel(PIDX(pidx_inc), adap->bar2 + udb + 8);
3731
3732 /* Re-enable BAR2 WC */
3733 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3734 }
3735
3069ee9b 3736 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
881806bc
VP		 3737}
3738
3739void t4_db_full(struct adapter *adap)
3740{
d14807dd 3741 if (is_t4(adap->params.chip)) {
05eb2389
SW		 3742 disable_dbs(adap);
3743 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
2cc301d2
SR		 3744 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3745 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3746 queue_work(workq, &adap->db_full_task);
3747 }
881806bc
VP		 3748}
3749
3750void t4_db_dropped(struct adapter *adap)
3751{
05eb2389
SW		 3752 if (is_t4(adap->params.chip)) {
3753 disable_dbs(adap);
3754 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3755 }
3756 queue_work(workq, &adap->db_drop_task);
881806bc
VP		 3757}
3758
b8ff05a9
DM		 3759static void uld_attach(struct adapter *adap, unsigned int uld)
3760{
3761 void *handle;
3762 struct cxgb4_lld_info lli;
dca4faeb 3763 unsigned short i;
b8ff05a9
DM		 3764
3765 lli.pdev = adap->pdev;
3766 lli.l2t = adap->l2t;
3767 lli.tids = &adap->tids;
3768 lli.ports = adap->port;
3769 lli.vr = &adap->vres;
3770 lli.mtus = adap->params.mtus;
3771 if (uld == CXGB4_ULD_RDMA) {
3772 lli.rxq_ids = adap->sge.rdma_rxq;
3773 lli.nrxq = adap->sge.rdmaqs;
3774 } else if (uld == CXGB4_ULD_ISCSI) {
3775 lli.rxq_ids = adap->sge.ofld_rxq;
3776 lli.nrxq = adap->sge.ofldqsets;
3777 }
3778 lli.ntxq = adap->sge.ofldqsets;
3779 lli.nchan = adap->params.nports;
3780 lli.nports = adap->params.nports;
3781 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 3782 lli.adapter_type = adap->params.chip;
b8ff05a9
DM		 3783 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3784 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 3785 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3786 (adap->fn * 4));
b8ff05a9 3787 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 3788 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3789 (adap->fn * 4));
dcf7b6f5 3790 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 3791 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3792 for (i = 0; i < NCHAN; i++)
3793 lli.tx_modq[i] = i;
b8ff05a9
DM		 3794 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3795 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3796 lli.fw_vers = adap->params.fw_vers;
3069ee9b 3797 lli.dbfifo_int_thresh = dbfifo_int_thresh;
dca4faeb
VP		 3798 lli.sge_pktshift = adap->sge.pktshift;
3799 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
1ac0f095 3800 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 3801
3802 handle = ulds[uld].add(&lli);
3803 if (IS_ERR(handle)) {
3804 dev_warn(adap->pdev_dev,
3805 "could not attach to the %s driver, error %ld\n",
3806 uld_str[uld], PTR_ERR(handle));
3807 return;
3808 }
3809
3810 adap->uld_handle[uld] = handle;
3811
3812 if (!netevent_registered) {
3813 register_netevent_notifier(&cxgb4_netevent_nb);
3814 netevent_registered = true;
3815 }
e29f5dbc
DM		 3816
3817 if (adap->flags & FULL_INIT_DONE)
3818 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 3819}
3820
3821static void attach_ulds(struct adapter *adap)
3822{
3823 unsigned int i;
3824
01bcca68
VP		 3825 spin_lock(&adap_rcu_lock);
3826 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
3827 spin_unlock(&adap_rcu_lock);
3828
b8ff05a9
DM		 3829 mutex_lock(&uld_mutex);
3830 list_add_tail(&adap->list_node, &adapter_list);
3831 for (i = 0; i < CXGB4_ULD_MAX; i++)
3832 if (ulds[i].add)
3833 uld_attach(adap, i);
3834 mutex_unlock(&uld_mutex);
3835}
3836
3837static void detach_ulds(struct adapter *adap)
3838{
3839 unsigned int i;
3840
3841 mutex_lock(&uld_mutex);
3842 list_del(&adap->list_node);
3843 for (i = 0; i < CXGB4_ULD_MAX; i++)
3844 if (adap->uld_handle[i]) {
3845 ulds[i].state_change(adap->uld_handle[i],
3846 CXGB4_STATE_DETACH);
3847 adap->uld_handle[i] = NULL;
3848 }
3849 if (netevent_registered && list_empty(&adapter_list)) {
3850 unregister_netevent_notifier(&cxgb4_netevent_nb);
3851 netevent_registered = false;
3852 }
3853 mutex_unlock(&uld_mutex);
01bcca68
VP		 3854
3855 spin_lock(&adap_rcu_lock);
3856 list_del_rcu(&adap->rcu_node);
3857 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 3858}
3859
3860static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3861{
3862 unsigned int i;
3863
3864 mutex_lock(&uld_mutex);
3865 for (i = 0; i < CXGB4_ULD_MAX; i++)
3866 if (adap->uld_handle[i])
3867 ulds[i].state_change(adap->uld_handle[i], new_state);
3868 mutex_unlock(&uld_mutex);
3869}
3870
3871/**
3872 * cxgb4_register_uld - register an upper-layer driver
3873 * @type: the ULD type
3874 * @p: the ULD methods
3875 *
3876 * Registers an upper-layer driver with this driver and notifies the ULD
3877 * about any presently available devices that support its type. Returns
3878 * %-EBUSY if a ULD of the same type is already registered.
3879 */
3880int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
3881{
3882 int ret = 0;
3883 struct adapter *adap;
3884
3885 if (type >= CXGB4_ULD_MAX)
3886 return -EINVAL;
3887 mutex_lock(&uld_mutex);
3888 if (ulds[type].add) {
3889 ret = -EBUSY;
3890 goto out;
3891 }
3892 ulds[type] = *p;
3893 list_for_each_entry(adap, &adapter_list, list_node)
3894 uld_attach(adap, type);
3895out: mutex_unlock(&uld_mutex);
3896 return ret;
3897}
3898EXPORT_SYMBOL(cxgb4_register_uld);
3899
3900/**
3901 * cxgb4_unregister_uld - unregister an upper-layer driver
3902 * @type: the ULD type
3903 *
3904 * Unregisters an existing upper-layer driver.
3905 */
3906int cxgb4_unregister_uld(enum cxgb4_uld type)
3907{
3908 struct adapter *adap;
3909
3910 if (type >= CXGB4_ULD_MAX)
3911 return -EINVAL;
3912 mutex_lock(&uld_mutex);
3913 list_for_each_entry(adap, &adapter_list, list_node)
3914 adap->uld_handle[type] = NULL;
3915 ulds[type].add = NULL;
3916 mutex_unlock(&uld_mutex);
3917 return 0;
3918}
3919EXPORT_SYMBOL(cxgb4_unregister_uld);
3920
01bcca68
VP		 3921/* Check if netdev on which event is occured belongs to us or not. Return
3922 * suceess (1) if it belongs otherwise failure (0).
3923 */
3924static int cxgb4_netdev(struct net_device *netdev)
3925{
3926 struct adapter *adap;
3927 int i;
3928
3929 spin_lock(&adap_rcu_lock);
3930 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
3931 for (i = 0; i < MAX_NPORTS; i++)
3932 if (adap->port[i] == netdev) {
3933 spin_unlock(&adap_rcu_lock);
3934 return 1;
3935 }
3936 spin_unlock(&adap_rcu_lock);
3937 return 0;
3938}
3939
3940static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
3941 unsigned long event)
3942{
3943 int ret = NOTIFY_DONE;
3944
3945 rcu_read_lock();
3946 if (cxgb4_netdev(event_dev)) {
3947 switch (event) {
3948 case NETDEV_UP:
3949 ret = cxgb4_clip_get(event_dev,
3950 (const struct in6_addr *)ifa->addr.s6_addr);
3951 if (ret < 0) {
3952 rcu_read_unlock();
3953 return ret;
3954 }
3955 ret = NOTIFY_OK;
3956 break;
3957 case NETDEV_DOWN:
3958 cxgb4_clip_release(event_dev,
3959 (const struct in6_addr *)ifa->addr.s6_addr);
3960 ret = NOTIFY_OK;
3961 break;
3962 default:
3963 break;
3964 }
3965 }
3966 rcu_read_unlock();
3967 return ret;
3968}
3969
3970static int cxgb4_inet6addr_handler(struct notifier_block *this,
3971 unsigned long event, void *data)
3972{
3973 struct inet6_ifaddr *ifa = data;
3974 struct net_device *event_dev;
3975 int ret = NOTIFY_DONE;
01bcca68 3976 struct bonding *bond = netdev_priv(ifa->idev->dev);
9caff1e7 3977 struct list_head *iter;
01bcca68
VP		 3978 struct slave *slave;
3979 struct pci_dev *first_pdev = NULL;
3980
3981 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
3982 event_dev = vlan_dev_real_dev(ifa->idev->dev);
3983 ret = clip_add(event_dev, ifa, event);
3984 } else if (ifa->idev->dev->flags & IFF_MASTER) {
3985 /* It is possible that two different adapters are bonded in one
3986 * bond. We need to find such different adapters and add clip
3987 * in all of them only once.
3988 */
3989 read_lock(&bond->lock);
9caff1e7 3990 bond_for_each_slave(bond, slave, iter) {
01bcca68
VP		 3991 if (!first_pdev) {
3992 ret = clip_add(slave->dev, ifa, event);
3993 /* If clip_add is success then only initialize
3994 * first_pdev since it means it is our device
3995 */
3996 if (ret == NOTIFY_OK)
3997 first_pdev = to_pci_dev(
3998 slave->dev->dev.parent);
3999 } else if (first_pdev !=
4000 to_pci_dev(slave->dev->dev.parent))
4001 ret = clip_add(slave->dev, ifa, event);
4002 }
4003 read_unlock(&bond->lock);
4004 } else
4005 ret = clip_add(ifa->idev->dev, ifa, event);
4006
4007 return ret;
4008}
4009
4010static struct notifier_block cxgb4_inet6addr_notifier = {
4011 .notifier_call = cxgb4_inet6addr_handler
4012};
4013
4014/* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4015 * a physical device.
4016 * The physical device reference is needed to send the actul CLIP command.
4017 */
4018static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
4019{
4020 struct inet6_dev *idev = NULL;
4021 struct inet6_ifaddr *ifa;
4022 int ret = 0;
4023
4024 idev = __in6_dev_get(root_dev);
4025 if (!idev)
4026 return ret;
4027
4028 read_lock_bh(&idev->lock);
4029 list_for_each_entry(ifa, &idev->addr_list, if_list) {
4030 ret = cxgb4_clip_get(dev,
4031 (const struct in6_addr *)ifa->addr.s6_addr);
4032 if (ret < 0)
4033 break;
4034 }
4035 read_unlock_bh(&idev->lock);
4036
4037 return ret;
4038}
4039
4040static int update_root_dev_clip(struct net_device *dev)
4041{
4042 struct net_device *root_dev = NULL;
4043 int i, ret = 0;
4044
4045 /* First populate the real net device's IPv6 addresses */
4046 ret = update_dev_clip(dev, dev);
4047 if (ret)
4048 return ret;
4049
4050 /* Parse all bond and vlan devices layered on top of the physical dev */
4051 for (i = 0; i < VLAN_N_VID; i++) {
4052 root_dev = __vlan_find_dev_deep(dev, htons(ETH_P_8021Q), i);
4053 if (!root_dev)
4054 continue;
4055
4056 ret = update_dev_clip(root_dev, dev);
4057 if (ret)
4058 break;
4059 }
4060 return ret;
4061}
4062
4063static void update_clip(const struct adapter *adap)
4064{
4065 int i;
4066 struct net_device *dev;
4067 int ret;
4068
4069 rcu_read_lock();
4070
4071 for (i = 0; i < MAX_NPORTS; i++) {
4072 dev = adap->port[i];
4073 ret = 0;
4074
4075 if (dev)
4076 ret = update_root_dev_clip(dev);
4077
4078 if (ret < 0)
4079 break;
4080 }
4081 rcu_read_unlock();
4082}
4083
b8ff05a9
DM		 4084/**
4085 * cxgb_up - enable the adapter
4086 * @adap: adapter being enabled
4087 *
4088 * Called when the first port is enabled, this function performs the
4089 * actions necessary to make an adapter operational, such as completing
4090 * the initialization of HW modules, and enabling interrupts.
4091 *
4092 * Must be called with the rtnl lock held.
4093 */
4094static int cxgb_up(struct adapter *adap)
4095{
aaefae9b 4096 int err;
b8ff05a9 4097
aaefae9b
DM		 4098 err = setup_sge_queues(adap);
4099 if (err)
4100 goto out;
4101 err = setup_rss(adap);
4102 if (err)
4103 goto freeq;
b8ff05a9
DM		 4104
4105 if (adap->flags & USING_MSIX) {
aaefae9b 4106 name_msix_vecs(adap);
b8ff05a9
DM		 4107 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4108 adap->msix_info[0].desc, adap);
4109 if (err)
4110 goto irq_err;
4111
4112 err = request_msix_queue_irqs(adap);
4113 if (err) {
4114 free_irq(adap->msix_info[0].vec, adap);
4115 goto irq_err;
4116 }
4117 } else {
4118 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4119 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4120 adap->port[0]->name, adap);
b8ff05a9
DM		 4121 if (err)
4122 goto irq_err;
4123 }
4124 enable_rx(adap);
4125 t4_sge_start(adap);
4126 t4_intr_enable(adap);
aaefae9b 4127 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4128 notify_ulds(adap, CXGB4_STATE_UP);
01bcca68 4129 update_clip(adap);
b8ff05a9
DM		 4130 out:
4131 return err;
4132 irq_err:
4133 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4134 freeq:
4135 t4_free_sge_resources(adap);
b8ff05a9
DM		 4136 goto out;
4137}
4138
4139static void cxgb_down(struct adapter *adapter)
4140{
4141 t4_intr_disable(adapter);
4142 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4143 cancel_work_sync(&adapter->db_full_task);
4144 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4145 adapter->tid_release_task_busy = false;
204dc3c0 4146 adapter->tid_release_head = NULL;
b8ff05a9
DM		 4147
4148 if (adapter->flags & USING_MSIX) {
4149 free_msix_queue_irqs(adapter);
4150 free_irq(adapter->msix_info[0].vec, adapter);
4151 } else
4152 free_irq(adapter->pdev->irq, adapter);
4153 quiesce_rx(adapter);
aaefae9b
DM		 4154 t4_sge_stop(adapter);
4155 t4_free_sge_resources(adapter);
4156 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4157}
4158
4159/*
4160 * net_device operations
4161 */
4162static int cxgb_open(struct net_device *dev)
4163{
4164 int err;
4165 struct port_info *pi = netdev_priv(dev);
4166 struct adapter *adapter = pi->adapter;
4167
6a3c869a
DM		 4168 netif_carrier_off(dev);
4169
aaefae9b
DM		 4170 if (!(adapter->flags & FULL_INIT_DONE)) {
4171 err = cxgb_up(adapter);
4172 if (err < 0)
4173 return err;
4174 }
b8ff05a9 4175
f68707b8
DM		 4176 err = link_start(dev);
4177 if (!err)
4178 netif_tx_start_all_queues(dev);
4179 return err;
b8ff05a9
DM		 4180}
4181
4182static int cxgb_close(struct net_device *dev)
4183{
b8ff05a9
DM		 4184 struct port_info *pi = netdev_priv(dev);
4185 struct adapter *adapter = pi->adapter;
4186
4187 netif_tx_stop_all_queues(dev);
4188 netif_carrier_off(dev);
060e0c75 4189 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4190}
4191
f2b7e78d
VP		 4192/* Return an error number if the indicated filter isn't writable ...
4193 */
4194static int writable_filter(struct filter_entry *f)
4195{
4196 if (f->locked)
4197 return -EPERM;
4198 if (f->pending)
4199 return -EBUSY;
4200
4201 return 0;
4202}
4203
4204/* Delete the filter at the specified index (if valid). The checks for all
4205 * the common problems with doing this like the filter being locked, currently
4206 * pending in another operation, etc.
4207 */
4208static int delete_filter(struct adapter *adapter, unsigned int fidx)
4209{
4210 struct filter_entry *f;
4211 int ret;
4212
dca4faeb 4213 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4214 return -EINVAL;
4215
4216 f = &adapter->tids.ftid_tab[fidx];
4217 ret = writable_filter(f);
4218 if (ret)
4219 return ret;
4220 if (f->valid)
4221 return del_filter_wr(adapter, fidx);
4222
4223 return 0;
4224}
4225
dca4faeb 4226int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4227 __be32 sip, __be16 sport, __be16 vlan,
4228 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4229{
4230 int ret;
4231 struct filter_entry *f;
4232 struct adapter *adap;
4233 int i;
4234 u8 *val;
4235
4236 adap = netdev2adap(dev);
4237
1cab775c 4238 /* Adjust stid to correct filter index */
470c60c4 4239 stid -= adap->tids.sftid_base;
1cab775c
VP		 4240 stid += adap->tids.nftids;
4241
dca4faeb
VP		 4242 /* Check to make sure the filter requested is writable ...
4243 */
4244 f = &adap->tids.ftid_tab[stid];
4245 ret = writable_filter(f);
4246 if (ret)
4247 return ret;
4248
4249 /* Clear out any old resources being used by the filter before
4250 * we start constructing the new filter.
4251 */
4252 if (f->valid)
4253 clear_filter(adap, f);
4254
4255 /* Clear out filter specifications */
4256 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4257 f->fs.val.lport = cpu_to_be16(sport);
4258 f->fs.mask.lport = ~0;
4259 val = (u8 *)&sip;
793dad94 4260 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4261 for (i = 0; i < 4; i++) {
4262 f->fs.val.lip[i] = val[i];
4263 f->fs.mask.lip[i] = ~0;
4264 }
dcf7b6f5 4265 if (adap->params.tp.vlan_pri_map & F_PORT) {
793dad94
VP		 4266 f->fs.val.iport = port;
4267 f->fs.mask.iport = mask;
4268 }
4269 }
dca4faeb 4270
dcf7b6f5 4271 if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {
7c89e555
KS		 4272 f->fs.val.proto = IPPROTO_TCP;
4273 f->fs.mask.proto = ~0;
4274 }
4275
dca4faeb
VP		 4276 f->fs.dirsteer = 1;
4277 f->fs.iq = queue;
4278 /* Mark filter as locked */
4279 f->locked = 1;
4280 f->fs.rpttid = 1;
4281
4282 ret = set_filter_wr(adap, stid);
4283 if (ret) {
4284 clear_filter(adap, f);
4285 return ret;
4286 }
4287
4288 return 0;
4289}
4290EXPORT_SYMBOL(cxgb4_create_server_filter);
4291
4292int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4293 unsigned int queue, bool ipv6)
4294{
4295 int ret;
4296 struct filter_entry *f;
4297 struct adapter *adap;
4298
4299 adap = netdev2adap(dev);
1cab775c
VP		 4300
4301 /* Adjust stid to correct filter index */
470c60c4 4302 stid -= adap->tids.sftid_base;
1cab775c
VP		 4303 stid += adap->tids.nftids;
4304
dca4faeb
VP		 4305 f = &adap->tids.ftid_tab[stid];
4306 /* Unlock the filter */
4307 f->locked = 0;
4308
4309 ret = delete_filter(adap, stid);
4310 if (ret)
4311 return ret;
4312
4313 return 0;
4314}
4315EXPORT_SYMBOL(cxgb4_remove_server_filter);
4316
f5152c90
DM		 4317static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4318 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4319{
4320 struct port_stats stats;
4321 struct port_info *p = netdev_priv(dev);
4322 struct adapter *adapter = p->adapter;
b8ff05a9 4323
9fe6cb58
GS		 4324 /* Block retrieving statistics during EEH error
4325 * recovery. Otherwise, the recovery might fail
4326 * and the PCI device will be removed permanently
4327 */
b8ff05a9 4328 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4329 if (!netif_device_present(dev)) {
4330 spin_unlock(&adapter->stats_lock);
4331 return ns;
4332 }
b8ff05a9
DM		 4333 t4_get_port_stats(adapter, p->tx_chan, &stats);
4334 spin_unlock(&adapter->stats_lock);
4335
4336 ns->tx_bytes = stats.tx_octets;
4337 ns->tx_packets = stats.tx_frames;
4338 ns->rx_bytes = stats.rx_octets;
4339 ns->rx_packets = stats.rx_frames;
4340 ns->multicast = stats.rx_mcast_frames;
4341
4342 /* detailed rx_errors */
4343 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4344 stats.rx_runt;
4345 ns->rx_over_errors = 0;
4346 ns->rx_crc_errors = stats.rx_fcs_err;
4347 ns->rx_frame_errors = stats.rx_symbol_err;
4348 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4349 stats.rx_ovflow2 + stats.rx_ovflow3 +
4350 stats.rx_trunc0 + stats.rx_trunc1 +
4351 stats.rx_trunc2 + stats.rx_trunc3;
4352 ns->rx_missed_errors = 0;
4353
4354 /* detailed tx_errors */
4355 ns->tx_aborted_errors = 0;
4356 ns->tx_carrier_errors = 0;
4357 ns->tx_fifo_errors = 0;
4358 ns->tx_heartbeat_errors = 0;
4359 ns->tx_window_errors = 0;
4360
4361 ns->tx_errors = stats.tx_error_frames;
4362 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4363 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4364 return ns;
4365}
4366
4367static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4368{
060e0c75 4369 unsigned int mbox;
b8ff05a9
DM		 4370 int ret = 0, prtad, devad;
4371 struct port_info *pi = netdev_priv(dev);
4372 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4373
4374 switch (cmd) {
4375 case SIOCGMIIPHY:
4376 if (pi->mdio_addr < 0)
4377 return -EOPNOTSUPP;
4378 data->phy_id = pi->mdio_addr;
4379 break;
4380 case SIOCGMIIREG:
4381 case SIOCSMIIREG:
4382 if (mdio_phy_id_is_c45(data->phy_id)) {
4383 prtad = mdio_phy_id_prtad(data->phy_id);
4384 devad = mdio_phy_id_devad(data->phy_id);
4385 } else if (data->phy_id < 32) {
4386 prtad = data->phy_id;
4387 devad = 0;
4388 data->reg_num &= 0x1f;
4389 } else
4390 return -EINVAL;
4391
060e0c75 4392 mbox = pi->adapter->fn;
b8ff05a9 4393 if (cmd == SIOCGMIIREG)
060e0c75 4394 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4395 data->reg_num, &data->val_out);
4396 else
060e0c75 4397 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4398 data->reg_num, data->val_in);
4399 break;
4400 default:
4401 return -EOPNOTSUPP;
4402 }
4403 return ret;
4404}
4405
4406static void cxgb_set_rxmode(struct net_device *dev)
4407{
4408 /* unfortunately we can't return errors to the stack */
4409 set_rxmode(dev, -1, false);
4410}
4411
4412static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4413{
4414 int ret;
4415 struct port_info *pi = netdev_priv(dev);
4416
4417 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4418 return -EINVAL;
060e0c75
DM		 4419 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4420 -1, -1, -1, true);
b8ff05a9
DM		 4421 if (!ret)
4422 dev->mtu = new_mtu;
4423 return ret;
4424}
4425
4426static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4427{
4428 int ret;
4429 struct sockaddr *addr = p;
4430 struct port_info *pi = netdev_priv(dev);
4431
4432 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4433 return -EADDRNOTAVAIL;
b8ff05a9 4434
060e0c75
DM		 4435 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4436 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4437 if (ret < 0)
4438 return ret;
4439
4440 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4441 pi->xact_addr_filt = ret;
4442 return 0;
4443}
4444
b8ff05a9
DM		 4445#ifdef CONFIG_NET_POLL_CONTROLLER
4446static void cxgb_netpoll(struct net_device *dev)
4447{
4448 struct port_info *pi = netdev_priv(dev);
4449 struct adapter *adap = pi->adapter;
4450
4451 if (adap->flags & USING_MSIX) {
4452 int i;
4453 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4454
4455 for (i = pi->nqsets; i; i--, rx++)
4456 t4_sge_intr_msix(0, &rx->rspq);
4457 } else
4458 t4_intr_handler(adap)(0, adap);
4459}
4460#endif
4461
4462static const struct net_device_ops cxgb4_netdev_ops = {
4463 .ndo_open = cxgb_open,
4464 .ndo_stop = cxgb_close,
4465 .ndo_start_xmit = t4_eth_xmit,
9be793bf 4466 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4467 .ndo_set_rx_mode = cxgb_set_rxmode,
4468 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4469 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4470 .ndo_validate_addr = eth_validate_addr,
4471 .ndo_do_ioctl = cxgb_ioctl,
4472 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4473#ifdef CONFIG_NET_POLL_CONTROLLER
4474 .ndo_poll_controller = cxgb_netpoll,
4475#endif
4476};
4477
4478void t4_fatal_err(struct adapter *adap)
4479{
4480 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4481 t4_intr_disable(adap);
4482 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4483}
4484
4485static void setup_memwin(struct adapter *adap)
4486{
19dd37ba 4487 u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;
b8ff05a9
DM		 4488
4489 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
d14807dd 4490 if (is_t4(adap->params.chip)) {
19dd37ba
SR		 4491 mem_win0_base = bar0 + MEMWIN0_BASE;
4492 mem_win1_base = bar0 + MEMWIN1_BASE;
4493 mem_win2_base = bar0 + MEMWIN2_BASE;
4494 } else {
4495 /* For T5, only relative offset inside the PCIe BAR is passed */
4496 mem_win0_base = MEMWIN0_BASE;
4497 mem_win1_base = MEMWIN1_BASE_T5;
4498 mem_win2_base = MEMWIN2_BASE_T5;
4499 }
b8ff05a9 4500 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
19dd37ba 4501 mem_win0_base | BIR(0) |
b8ff05a9
DM		 4502 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4503 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
19dd37ba 4504 mem_win1_base | BIR(0) |
b8ff05a9
DM		 4505 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4506 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
19dd37ba 4507 mem_win2_base | BIR(0) |
b8ff05a9 4508 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
636f9d37
VP		 4509}
4510
4511static void setup_memwin_rdma(struct adapter *adap)
4512{
1ae970e0
DM		 4513 if (adap->vres.ocq.size) {
4514 unsigned int start, sz_kb;
4515
4516 start = pci_resource_start(adap->pdev, 2) +
4517 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4518 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4519 t4_write_reg(adap,
4520 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4521 start | BIR(1) | WINDOW(ilog2(sz_kb)));
4522 t4_write_reg(adap,
4523 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4524 adap->vres.ocq.start);
4525 t4_read_reg(adap,
4526 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4527 }
b8ff05a9
DM		 4528}
4529
02b5fb8e
DM		 4530static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4531{
4532 u32 v;
4533 int ret;
4534
4535 /* get device capabilities */
4536 memset(c, 0, sizeof(*c));
4537 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4538 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 4539 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4540 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4541 if (ret < 0)
4542 return ret;
4543
4544 /* select capabilities we'll be using */
4545 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4546 if (!vf_acls)
4547 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4548 else
4549 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4550 } else if (vf_acls) {
4551 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4552 return ret;
4553 }
4554 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4555 FW_CMD_REQUEST | FW_CMD_WRITE);
060e0c75 4556 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4557 if (ret < 0)
4558 return ret;
4559
060e0c75 4560 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e
DM		 4561 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4562 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4563 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
4564 if (ret < 0)
4565 return ret;
4566
060e0c75
DM		 4567 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4568 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 4569 if (ret < 0)
4570 return ret;
4571
4572 t4_sge_init(adap);
4573
02b5fb8e
DM		 4574 /* tweak some settings */
4575 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4576 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4577 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4578 v = t4_read_reg(adap, TP_PIO_DATA);
4579 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
060e0c75 4580
dca4faeb
VP		 4581 /* first 4 Tx modulation queues point to consecutive Tx channels */
4582 adap->params.tp.tx_modq_map = 0xE4;
4583 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4584 V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4585
4586 /* associate each Tx modulation queue with consecutive Tx channels */
4587 v = 0x84218421;
4588 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4589 &v, 1, A_TP_TX_SCHED_HDR);
4590 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4591 &v, 1, A_TP_TX_SCHED_FIFO);
4592 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4593 &v, 1, A_TP_TX_SCHED_PCMD);
4594
4595#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4596 if (is_offload(adap)) {
4597 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4598 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4599 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4600 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4601 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4602 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4603 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4604 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4605 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4606 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4607 }
4608
060e0c75
DM		 4609 /* get basic stuff going */
4610 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4611}
4612
b8ff05a9
DM		 4613/*
4614 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4615 */
4616#define MAX_ATIDS 8192U
4617
636f9d37
VP		 4618/*
4619 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4620 *
4621 * If the firmware we're dealing with has Configuration File support, then
4622 * we use that to perform all configuration
4623 */
4624
4625/*
4626 * Tweak configuration based on module parameters, etc. Most of these have
4627 * defaults assigned to them by Firmware Configuration Files (if we're using
4628 * them) but need to be explicitly set if we're using hard-coded
4629 * initialization. But even in the case of using Firmware Configuration
4630 * Files, we'd like to expose the ability to change these via module
4631 * parameters so these are essentially common tweaks/settings for
4632 * Configuration Files and hard-coded initialization ...
4633 */
4634static int adap_init0_tweaks(struct adapter *adapter)
4635{
4636 /*
4637 * Fix up various Host-Dependent Parameters like Page Size, Cache
4638 * Line Size, etc. The firmware default is for a 4KB Page Size and
4639 * 64B Cache Line Size ...
4640 */
4641 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4642
4643 /*
4644 * Process module parameters which affect early initialization.
4645 */
4646 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4647 dev_err(&adapter->pdev->dev,
4648 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4649 rx_dma_offset);
4650 rx_dma_offset = 2;
4651 }
4652 t4_set_reg_field(adapter, SGE_CONTROL,
4653 PKTSHIFT_MASK,
4654 PKTSHIFT(rx_dma_offset));
4655
4656 /*
4657 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4658 * adds the pseudo header itself.
4659 */
4660 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
4661 CSUM_HAS_PSEUDO_HDR, 0);
4662
4663 return 0;
4664}
4665
4666/*
4667 * Attempt to initialize the adapter via a Firmware Configuration File.
4668 */
4669static int adap_init0_config(struct adapter *adapter, int reset)
4670{
4671 struct fw_caps_config_cmd caps_cmd;
4672 const struct firmware *cf;
4673 unsigned long mtype = 0, maddr = 0;
4674 u32 finiver, finicsum, cfcsum;
16e47624
HS		 4675 int ret;
4676 int config_issued = 0;
0a57a536 4677 char *fw_config_file, fw_config_file_path[256];
16e47624 4678 char *config_name = NULL;
636f9d37
VP		 4679
4680 /*
4681 * Reset device if necessary.
4682 */
4683 if (reset) {
4684 ret = t4_fw_reset(adapter, adapter->mbox,
4685 PIORSTMODE | PIORST);
4686 if (ret < 0)
4687 goto bye;
4688 }
4689
4690 /*
4691 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4692 * then use that. Otherwise, use the configuration file stored
4693 * in the adapter flash ...
4694 */
d14807dd 4695 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 4696 case CHELSIO_T4:
16e47624 4697 fw_config_file = FW4_CFNAME;
0a57a536
SR		 4698 break;
4699 case CHELSIO_T5:
4700 fw_config_file = FW5_CFNAME;
4701 break;
4702 default:
4703 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4704 adapter->pdev->device);
4705 ret = -EINVAL;
4706 goto bye;
4707 }
4708
4709 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 4710 if (ret < 0) {
16e47624 4711 config_name = "On FLASH";
636f9d37
VP		 4712 mtype = FW_MEMTYPE_CF_FLASH;
4713 maddr = t4_flash_cfg_addr(adapter);
4714 } else {
4715 u32 params[7], val[7];
4716
16e47624
HS		 4717 sprintf(fw_config_file_path,
4718 "/lib/firmware/%s", fw_config_file);
4719 config_name = fw_config_file_path;
4720
636f9d37
VP		 4721 if (cf->size >= FLASH_CFG_MAX_SIZE)
4722 ret = -ENOMEM;
4723 else {
4724 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4725 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4726 ret = t4_query_params(adapter, adapter->mbox,
4727 adapter->fn, 0, 1, params, val);
4728 if (ret == 0) {
4729 /*
4730 * For t4_memory_write() below addresses and
4731 * sizes have to be in terms of multiples of 4
4732 * bytes. So, if the Configuration File isn't
4733 * a multiple of 4 bytes in length we'll have
4734 * to write that out separately since we can't
4735 * guarantee that the bytes following the
4736 * residual byte in the buffer returned by
4737 * request_firmware() are zeroed out ...
4738 */
4739 size_t resid = cf->size & 0x3;
4740 size_t size = cf->size & ~0x3;
4741 __be32 *data = (__be32 *)cf->data;
4742
4743 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
4744 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
4745
4746 ret = t4_memory_write(adapter, mtype, maddr,
4747 size, data);
4748 if (ret == 0 && resid != 0) {
4749 union {
4750 __be32 word;
4751 char buf[4];
4752 } last;
4753 int i;
4754
4755 last.word = data[size >> 2];
4756 for (i = resid; i < 4; i++)
4757 last.buf[i] = 0;
4758 ret = t4_memory_write(adapter, mtype,
4759 maddr + size,
4760 4, &last.word);
4761 }
4762 }
4763 }
4764
4765 release_firmware(cf);
4766 if (ret)
4767 goto bye;
4768 }
4769
4770 /*
4771 * Issue a Capability Configuration command to the firmware to get it
4772 * to parse the Configuration File. We don't use t4_fw_config_file()
4773 * because we want the ability to modify various features after we've
4774 * processed the configuration file ...
4775 */
4776 memset(&caps_cmd, 0, sizeof(caps_cmd));
4777 caps_cmd.op_to_write =
4778 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4779 FW_CMD_REQUEST |
4780 FW_CMD_READ);
ce91a923 4781 caps_cmd.cfvalid_to_len16 =
636f9d37
VP		 4782 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
4783 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4784 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
4785 FW_LEN16(caps_cmd));
4786 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4787 &caps_cmd);
16e47624
HS		 4788
4789 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4790 * Configuration File in FLASH), our last gasp effort is to use the
4791 * Firmware Configuration File which is embedded in the firmware. A
4792 * very few early versions of the firmware didn't have one embedded
4793 * but we can ignore those.
4794 */
4795 if (ret == -ENOENT) {
4796 memset(&caps_cmd, 0, sizeof(caps_cmd));
4797 caps_cmd.op_to_write =
4798 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4799 FW_CMD_REQUEST |
4800 FW_CMD_READ);
4801 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4802 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4803 sizeof(caps_cmd), &caps_cmd);
4804 config_name = "Firmware Default";
4805 }
4806
4807 config_issued = 1;
636f9d37
VP		 4808 if (ret < 0)
4809 goto bye;
4810
4811 finiver = ntohl(caps_cmd.finiver);
4812 finicsum = ntohl(caps_cmd.finicsum);
4813 cfcsum = ntohl(caps_cmd.cfcsum);
4814 if (finicsum != cfcsum)
4815 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4816 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4817 finicsum, cfcsum);
4818
636f9d37
VP		 4819 /*
4820 * And now tell the firmware to use the configuration we just loaded.
4821 */
4822 caps_cmd.op_to_write =
4823 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4824 FW_CMD_REQUEST |
4825 FW_CMD_WRITE);
ce91a923 4826 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 4827 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4828 NULL);
4829 if (ret < 0)
4830 goto bye;
4831
4832 /*
4833 * Tweak configuration based on system architecture, module
4834 * parameters, etc.
4835 */
4836 ret = adap_init0_tweaks(adapter);
4837 if (ret < 0)
4838 goto bye;
4839
4840 /*
4841 * And finally tell the firmware to initialize itself using the
4842 * parameters from the Configuration File.
4843 */
4844 ret = t4_fw_initialize(adapter, adapter->mbox);
4845 if (ret < 0)
4846 goto bye;
4847
4848 /*
4849 * Return successfully and note that we're operating with parameters
4850 * not supplied by the driver, rather than from hard-wired
4851 * initialization constants burried in the driver.
4852 */
4853 adapter->flags |= USING_SOFT_PARAMS;
4854 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 4855 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
4856 config_name, finiver, cfcsum);
636f9d37
VP		 4857 return 0;
4858
4859 /*
4860 * Something bad happened. Return the error ... (If the "error"
4861 * is that there's no Configuration File on the adapter we don't
4862 * want to issue a warning since this is fairly common.)
4863 */
4864bye:
16e47624
HS		 4865 if (config_issued && ret != -ENOENT)
4866 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
4867 config_name, -ret);
636f9d37
VP		 4868 return ret;
4869}
4870
13ee15d3
VP		 4871/*
4872 * Attempt to initialize the adapter via hard-coded, driver supplied
4873 * parameters ...
4874 */
4875static int adap_init0_no_config(struct adapter *adapter, int reset)
4876{
4877 struct sge *s = &adapter->sge;
4878 struct fw_caps_config_cmd caps_cmd;
4879 u32 v;
4880 int i, ret;
4881
4882 /*
4883 * Reset device if necessary
4884 */
4885 if (reset) {
4886 ret = t4_fw_reset(adapter, adapter->mbox,
4887 PIORSTMODE | PIORST);
4888 if (ret < 0)
4889 goto bye;
4890 }
4891
4892 /*
4893 * Get device capabilities and select which we'll be using.
4894 */
4895 memset(&caps_cmd, 0, sizeof(caps_cmd));
4896 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4897 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 4898 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
13ee15d3
VP		 4899 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4900 &caps_cmd);
4901 if (ret < 0)
4902 goto bye;
4903
13ee15d3
VP		 4904 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4905 if (!vf_acls)
4906 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4907 else
4908 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4909 } else if (vf_acls) {
4910 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
4911 goto bye;
4912 }
4913 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4914 FW_CMD_REQUEST | FW_CMD_WRITE);
4915 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4916 NULL);
4917 if (ret < 0)
4918 goto bye;
4919
4920 /*
4921 * Tweak configuration based on system architecture, module
4922 * parameters, etc.
4923 */
4924 ret = adap_init0_tweaks(adapter);
4925 if (ret < 0)
4926 goto bye;
4927
4928 /*
4929 * Select RSS Global Mode we want to use. We use "Basic Virtual"
4930 * mode which maps each Virtual Interface to its own section of
4931 * the RSS Table and we turn on all map and hash enables ...
4932 */
4933 adapter->flags |= RSS_TNLALLLOOKUP;
4934 ret = t4_config_glbl_rss(adapter, adapter->mbox,
4935 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4936 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4937 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
4938 ((adapter->flags & RSS_TNLALLLOOKUP) ?
4939 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
4940 if (ret < 0)
4941 goto bye;
4942
4943 /*
4944 * Set up our own fundamental resource provisioning ...
4945 */
4946 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
4947 PFRES_NEQ, PFRES_NETHCTRL,
4948 PFRES_NIQFLINT, PFRES_NIQ,
4949 PFRES_TC, PFRES_NVI,
4950 FW_PFVF_CMD_CMASK_MASK,
4951 pfvfres_pmask(adapter, adapter->fn, 0),
4952 PFRES_NEXACTF,
4953 PFRES_R_CAPS, PFRES_WX_CAPS);
4954 if (ret < 0)
4955 goto bye;
4956
4957 /*
4958 * Perform low level SGE initialization. We need to do this before we
4959 * send the firmware the INITIALIZE command because that will cause
4960 * any other PF Drivers which are waiting for the Master
4961 * Initialization to proceed forward.
4962 */
4963 for (i = 0; i < SGE_NTIMERS - 1; i++)
4964 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
4965 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
4966 s->counter_val[0] = 1;
4967 for (i = 1; i < SGE_NCOUNTERS; i++)
4968 s->counter_val[i] = min(intr_cnt[i - 1],
4969 THRESHOLD_0_GET(THRESHOLD_0_MASK));
4970 t4_sge_init(adapter);
4971
4972#ifdef CONFIG_PCI_IOV
4973 /*
4974 * Provision resource limits for Virtual Functions. We currently
4975 * grant them all the same static resource limits except for the Port
4976 * Access Rights Mask which we're assigning based on the PF. All of
4977 * the static provisioning stuff for both the PF and VF really needs
4978 * to be managed in a persistent manner for each device which the
4979 * firmware controls.
4980 */
4981 {
4982 int pf, vf;
4983
7d6727cf 4984 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
13ee15d3
VP		 4985 if (num_vf[pf] <= 0)
4986 continue;
4987
4988 /* VF numbering starts at 1! */
4989 for (vf = 1; vf <= num_vf[pf]; vf++) {
4990 ret = t4_cfg_pfvf(adapter, adapter->mbox,
4991 pf, vf,
4992 VFRES_NEQ, VFRES_NETHCTRL,
4993 VFRES_NIQFLINT, VFRES_NIQ,
4994 VFRES_TC, VFRES_NVI,
1f1e4958 4995 FW_PFVF_CMD_CMASK_MASK,
13ee15d3
VP		 4996 pfvfres_pmask(
4997 adapter, pf, vf),
4998 VFRES_NEXACTF,
4999 VFRES_R_CAPS, VFRES_WX_CAPS);
5000 if (ret < 0)
5001 dev_warn(adapter->pdev_dev,
5002 "failed to "\
5003 "provision pf/vf=%d/%d; "
5004 "err=%d\n", pf, vf, ret);
5005 }
5006 }
5007 }
5008#endif
5009
5010 /*
5011 * Set up the default filter mode. Later we'll want to implement this
5012 * via a firmware command, etc. ... This needs to be done before the
5013 * firmare initialization command ... If the selected set of fields
5014 * isn't equal to the default value, we'll need to make sure that the
5015 * field selections will fit in the 36-bit budget.
5016 */
5017 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
404d9e3f 5018 int j, bits = 0;
13ee15d3 5019
404d9e3f
VP		 5020 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
5021 switch (tp_vlan_pri_map & (1 << j)) {
13ee15d3
VP		 5022 case 0:
5023 /* compressed filter field not enabled */
5024 break;
5025 case FCOE_MASK:
5026 bits += 1;
5027 break;
5028 case PORT_MASK:
5029 bits += 3;
5030 break;
5031 case VNIC_ID_MASK:
5032 bits += 17;
5033 break;
5034 case VLAN_MASK:
5035 bits += 17;
5036 break;
5037 case TOS_MASK:
5038 bits += 8;
5039 break;
5040 case PROTOCOL_MASK:
5041 bits += 8;
5042 break;
5043 case ETHERTYPE_MASK:
5044 bits += 16;
5045 break;
5046 case MACMATCH_MASK:
5047 bits += 9;
5048 break;
5049 case MPSHITTYPE_MASK:
5050 bits += 3;
5051 break;
5052 case FRAGMENTATION_MASK:
5053 bits += 1;
5054 break;
5055 }
5056
5057 if (bits > 36) {
5058 dev_err(adapter->pdev_dev,
5059 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5060 " using %#x\n", tp_vlan_pri_map, bits,
5061 TP_VLAN_PRI_MAP_DEFAULT);
5062 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5063 }
5064 }
5065 v = tp_vlan_pri_map;
5066 t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
5067 &v, 1, TP_VLAN_PRI_MAP);
5068
5069 /*
5070 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5071 * to support any of the compressed filter fields above. Newer
5072 * versions of the firmware do this automatically but it doesn't hurt
5073 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5074 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5075 * since the firmware automatically turns this on and off when we have
5076 * a non-zero number of filters active (since it does have a
5077 * performance impact).
5078 */
5079 if (tp_vlan_pri_map)
5080 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
5081 FIVETUPLELOOKUP_MASK,
5082 FIVETUPLELOOKUP_MASK);
5083
5084 /*
5085 * Tweak some settings.
5086 */
5087 t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
5088 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5089 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5090 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5091
5092 /*
5093 * Get basic stuff going by issuing the Firmware Initialize command.
5094 * Note that this _must_ be after all PFVF commands ...
5095 */
5096 ret = t4_fw_initialize(adapter, adapter->mbox);
5097 if (ret < 0)
5098 goto bye;
5099
5100 /*
5101 * Return successfully!
5102 */
5103 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5104 "driver parameters\n");
5105 return 0;
5106
5107 /*
5108 * Something bad happened. Return the error ...
5109 */
5110bye:
5111 return ret;
5112}
5113
16e47624
HS		 5114static struct fw_info fw_info_array[] = {
5115 {
5116 .chip = CHELSIO_T4,
5117 .fs_name = FW4_CFNAME,
5118 .fw_mod_name = FW4_FNAME,
5119 .fw_hdr = {
5120 .chip = FW_HDR_CHIP_T4,
5121 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5122 .intfver_nic = FW_INTFVER(T4, NIC),
5123 .intfver_vnic = FW_INTFVER(T4, VNIC),
5124 .intfver_ri = FW_INTFVER(T4, RI),
5125 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5126 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5127 },
5128 }, {
5129 .chip = CHELSIO_T5,
5130 .fs_name = FW5_CFNAME,
5131 .fw_mod_name = FW5_FNAME,
5132 .fw_hdr = {
5133 .chip = FW_HDR_CHIP_T5,
5134 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5135 .intfver_nic = FW_INTFVER(T5, NIC),
5136 .intfver_vnic = FW_INTFVER(T5, VNIC),
5137 .intfver_ri = FW_INTFVER(T5, RI),
5138 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5139 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5140 },
5141 }
5142};
5143
5144static struct fw_info *find_fw_info(int chip)
5145{
5146 int i;
5147
5148 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5149 if (fw_info_array[i].chip == chip)
5150 return &fw_info_array[i];
5151 }
5152 return NULL;
5153}
5154
b8ff05a9
DM		 5155/*
5156 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5157 */
5158static int adap_init0(struct adapter *adap)
5159{
5160 int ret;
5161 u32 v, port_vec;
5162 enum dev_state state;
5163 u32 params[7], val[7];
9a4da2cd 5164 struct fw_caps_config_cmd caps_cmd;
dcf7b6f5 5165 int reset = 1;
b8ff05a9 5166
636f9d37
VP		 5167 /*
5168 * Contact FW, advertising Master capability (and potentially forcing
5169 * ourselves as the Master PF if our module parameter force_init is
5170 * set).
5171 */
5172 ret = t4_fw_hello(adap, adap->mbox, adap->fn,
5173 force_init ? MASTER_MUST : MASTER_MAY,
5174 &state);
b8ff05a9
DM		 5175 if (ret < 0) {
5176 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5177 ret);
5178 return ret;
5179 }
636f9d37
VP		 5180 if (ret == adap->mbox)
5181 adap->flags |= MASTER_PF;
5182 if (force_init && state == DEV_STATE_INIT)
5183 state = DEV_STATE_UNINIT;
b8ff05a9 5184
636f9d37
VP		 5185 /*
5186 * If we're the Master PF Driver and the device is uninitialized,
5187 * then let's consider upgrading the firmware ... (We always want
5188 * to check the firmware version number in order to A. get it for
5189 * later reporting and B. to warn if the currently loaded firmware
5190 * is excessively mismatched relative to the driver.)
5191 */
16e47624
HS		 5192 t4_get_fw_version(adap, &adap->params.fw_vers);
5193 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5194 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5195 struct fw_info *fw_info;
5196 struct fw_hdr *card_fw;
5197 const struct firmware *fw;
5198 const u8 *fw_data = NULL;
5199 unsigned int fw_size = 0;
5200
5201 /* This is the firmware whose headers the driver was compiled
5202 * against
5203 */
5204 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5205 if (fw_info == NULL) {
5206 dev_err(adap->pdev_dev,
5207 "unable to get firmware info for chip %d.\n",
5208 CHELSIO_CHIP_VERSION(adap->params.chip));
5209 return -EINVAL;
636f9d37 5210 }
16e47624
HS		 5211
5212 /* allocate memory to read the header of the firmware on the
5213 * card
5214 */
5215 card_fw = t4_alloc_mem(sizeof(*card_fw));
5216
5217 /* Get FW from from /lib/firmware/ */
5218 ret = request_firmware(&fw, fw_info->fw_mod_name,
5219 adap->pdev_dev);
5220 if (ret < 0) {
5221 dev_err(adap->pdev_dev,
5222 "unable to load firmware image %s, error %d\n",
5223 fw_info->fw_mod_name, ret);
5224 } else {
5225 fw_data = fw->data;
5226 fw_size = fw->size;
5227 }
5228
5229 /* upgrade FW logic */
5230 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5231 state, &reset);
5232
5233 /* Cleaning up */
5234 if (fw != NULL)
5235 release_firmware(fw);
5236 t4_free_mem(card_fw);
5237
636f9d37 5238 if (ret < 0)
16e47624 5239 goto bye;
636f9d37 5240 }
b8ff05a9 5241
636f9d37
VP		 5242 /*
5243 * Grab VPD parameters. This should be done after we establish a
5244 * connection to the firmware since some of the VPD parameters
5245 * (notably the Core Clock frequency) are retrieved via requests to
5246 * the firmware. On the other hand, we need these fairly early on
5247 * so we do this right after getting ahold of the firmware.
5248 */
5249 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5250 if (ret < 0)
5251 goto bye;
a0881cab 5252
636f9d37 5253 /*
13ee15d3
VP		 5254 * Find out what ports are available to us. Note that we need to do
5255 * this before calling adap_init0_no_config() since it needs nports
5256 * and portvec ...
636f9d37
VP		 5257 */
5258 v =
5259 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5260 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
5261 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5262 if (ret < 0)
5263 goto bye;
5264
636f9d37
VP		 5265 adap->params.nports = hweight32(port_vec);
5266 adap->params.portvec = port_vec;
5267
5268 /*
5269 * If the firmware is initialized already (and we're not forcing a
5270 * master initialization), note that we're living with existing
5271 * adapter parameters. Otherwise, it's time to try initializing the
5272 * adapter ...
5273 */
5274 if (state == DEV_STATE_INIT) {
5275 dev_info(adap->pdev_dev, "Coming up as %s: "\
5276 "Adapter already initialized\n",
5277 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5278 adap->flags |= USING_SOFT_PARAMS;
5279 } else {
5280 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5281 "Initializing adapter\n");
636f9d37
VP		 5282
5283 /*
5284 * If the firmware doesn't support Configuration
5285 * Files warn user and exit,
5286 */
5287 if (ret < 0)
13ee15d3 5288 dev_warn(adap->pdev_dev, "Firmware doesn't support "
636f9d37 5289 "configuration file.\n");
13ee15d3
VP		 5290 if (force_old_init)
5291 ret = adap_init0_no_config(adap, reset);
636f9d37
VP		 5292 else {
5293 /*
13ee15d3
VP		 5294 * Find out whether we're dealing with a version of
5295 * the firmware which has configuration file support.
636f9d37 5296 */
13ee15d3
VP		 5297 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5298 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
5299 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5300 params, val);
636f9d37 5301
13ee15d3
VP		 5302 /*
5303 * If the firmware doesn't support Configuration
5304 * Files, use the old Driver-based, hard-wired
5305 * initialization. Otherwise, try using the
5306 * Configuration File support and fall back to the
5307 * Driver-based initialization if there's no
5308 * Configuration File found.
5309 */
5310 if (ret < 0)
5311 ret = adap_init0_no_config(adap, reset);
5312 else {
5313 /*
5314 * The firmware provides us with a memory
5315 * buffer where we can load a Configuration
5316 * File from the host if we want to override
5317 * the Configuration File in flash.
5318 */
5319
5320 ret = adap_init0_config(adap, reset);
5321 if (ret == -ENOENT) {
5322 dev_info(adap->pdev_dev,
5323 "No Configuration File present "
16e47624 5324 "on adapter. Using hard-wired "
13ee15d3
VP		 5325 "configuration parameters.\n");
5326 ret = adap_init0_no_config(adap, reset);
5327 }
636f9d37
VP		 5328 }
5329 }
5330 if (ret < 0) {
5331 dev_err(adap->pdev_dev,
5332 "could not initialize adapter, error %d\n",
5333 -ret);
5334 goto bye;
5335 }
5336 }
5337
5338 /*
5339 * If we're living with non-hard-coded parameters (either from a
5340 * Firmware Configuration File or values programmed by a different PF
5341 * Driver), give the SGE code a chance to pull in anything that it
5342 * needs ... Note that this must be called after we retrieve our VPD
5343 * parameters in order to know how to convert core ticks to seconds.
5344 */
5345 if (adap->flags & USING_SOFT_PARAMS) {
5346 ret = t4_sge_init(adap);
5347 if (ret < 0)
5348 goto bye;
5349 }
5350
9a4da2cd
VP		 5351 if (is_bypass_device(adap->pdev->device))
5352 adap->params.bypass = 1;
5353
636f9d37
VP		 5354 /*
5355 * Grab some of our basic fundamental operating parameters.
5356 */
5357#define FW_PARAM_DEV(param) \
5358 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5359 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5360
b8ff05a9 5361#define FW_PARAM_PFVF(param) \
636f9d37
VP		 5362 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5363 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
5364 FW_PARAMS_PARAM_Y(0) | \
5365 FW_PARAMS_PARAM_Z(0)
b8ff05a9 5366
636f9d37 5367 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5368 params[1] = FW_PARAM_PFVF(L2T_START);
5369 params[2] = FW_PARAM_PFVF(L2T_END);
5370 params[3] = FW_PARAM_PFVF(FILTER_START);
5371 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5372 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5373 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5374 if (ret < 0)
5375 goto bye;
636f9d37
VP		 5376 adap->sge.egr_start = val[0];
5377 adap->l2t_start = val[1];
5378 adap->l2t_end = val[2];
b8ff05a9
DM		 5379 adap->tids.ftid_base = val[3];
5380 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5381 adap->sge.ingr_start = val[5];
b8ff05a9 5382
636f9d37
VP		 5383 /* query params related to active filter region */
5384 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5385 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5386 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5387 /* If Active filter size is set we enable establishing
5388 * offload connection through firmware work request
5389 */
5390 if ((val[0] != val[1]) && (ret >= 0)) {
5391 adap->flags |= FW_OFLD_CONN;
5392 adap->tids.aftid_base = val[0];
5393 adap->tids.aftid_end = val[1];
5394 }
5395
b407a4a9
VP		 5396 /* If we're running on newer firmware, let it know that we're
5397 * prepared to deal with encapsulated CPL messages. Older
5398 * firmware won't understand this and we'll just get
5399 * unencapsulated messages ...
5400 */
5401 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5402 val[0] = 1;
5403 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5404
1ac0f095
KS		 5405 /*
5406 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5407 * capability. Earlier versions of the firmware didn't have the
5408 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5409 * permission to use ULPTX MEMWRITE DSGL.
5410 */
5411 if (is_t4(adap->params.chip)) {
5412 adap->params.ulptx_memwrite_dsgl = false;
5413 } else {
5414 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5415 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5416 1, params, val);
5417 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5418 }
5419
636f9d37
VP		 5420 /*
5421 * Get device capabilities so we can determine what resources we need
5422 * to manage.
5423 */
5424 memset(&caps_cmd, 0, sizeof(caps_cmd));
9a4da2cd 5425 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
13ee15d3 5426 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 5427 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5428 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5429 &caps_cmd);
5430 if (ret < 0)
5431 goto bye;
5432
13ee15d3 5433 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5434 /* query offload-related parameters */
5435 params[0] = FW_PARAM_DEV(NTID);
5436 params[1] = FW_PARAM_PFVF(SERVER_START);
5437 params[2] = FW_PARAM_PFVF(SERVER_END);
5438 params[3] = FW_PARAM_PFVF(TDDP_START);
5439 params[4] = FW_PARAM_PFVF(TDDP_END);
5440 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5441 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5442 params, val);
b8ff05a9
DM		 5443 if (ret < 0)
5444 goto bye;
5445 adap->tids.ntids = val[0];
5446 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5447 adap->tids.stid_base = val[1];
5448 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37
VP		 5449 /*
5450 * Setup server filter region. Divide the availble filter
5451 * region into two parts. Regular filters get 1/3rd and server
5452 * filters get 2/3rd part. This is only enabled if workarond
5453 * path is enabled.
5454 * 1. For regular filters.
5455 * 2. Server filter: This are special filters which are used
5456 * to redirect SYN packets to offload queue.
5457 */
5458 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5459 adap->tids.sftid_base = adap->tids.ftid_base +
5460 DIV_ROUND_UP(adap->tids.nftids, 3);
5461 adap->tids.nsftids = adap->tids.nftids -
5462 DIV_ROUND_UP(adap->tids.nftids, 3);
5463 adap->tids.nftids = adap->tids.sftid_base -
5464 adap->tids.ftid_base;
5465 }
b8ff05a9
DM		 5466 adap->vres.ddp.start = val[3];
5467 adap->vres.ddp.size = val[4] - val[3] + 1;
5468 adap->params.ofldq_wr_cred = val[5];
636f9d37 5469
b8ff05a9
DM		 5470 adap->params.offload = 1;
5471 }
636f9d37 5472 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5473 params[0] = FW_PARAM_PFVF(STAG_START);
5474 params[1] = FW_PARAM_PFVF(STAG_END);
5475 params[2] = FW_PARAM_PFVF(RQ_START);
5476 params[3] = FW_PARAM_PFVF(RQ_END);
5477 params[4] = FW_PARAM_PFVF(PBL_START);
5478 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5479 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5480 params, val);
b8ff05a9
DM		 5481 if (ret < 0)
5482 goto bye;
5483 adap->vres.stag.start = val[0];
5484 adap->vres.stag.size = val[1] - val[0] + 1;
5485 adap->vres.rq.start = val[2];
5486 adap->vres.rq.size = val[3] - val[2] + 1;
5487 adap->vres.pbl.start = val[4];
5488 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5489
5490 params[0] = FW_PARAM_PFVF(SQRQ_START);
5491 params[1] = FW_PARAM_PFVF(SQRQ_END);
5492 params[2] = FW_PARAM_PFVF(CQ_START);
5493 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5494 params[4] = FW_PARAM_PFVF(OCQ_START);
5495 params[5] = FW_PARAM_PFVF(OCQ_END);
636f9d37 5496 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
a0881cab
DM		 5497 if (ret < 0)
5498 goto bye;
5499 adap->vres.qp.start = val[0];
5500 adap->vres.qp.size = val[1] - val[0] + 1;
5501 adap->vres.cq.start = val[2];
5502 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5503 adap->vres.ocq.start = val[4];
5504 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9 5505 }
636f9d37 5506 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5507 params[0] = FW_PARAM_PFVF(ISCSI_START);
5508 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5509 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5510 params, val);
b8ff05a9
DM		 5511 if (ret < 0)
5512 goto bye;
5513 adap->vres.iscsi.start = val[0];
5514 adap->vres.iscsi.size = val[1] - val[0] + 1;
5515 }
5516#undef FW_PARAM_PFVF
5517#undef FW_PARAM_DEV
5518
636f9d37
VP		 5519 /*
5520 * These are finalized by FW initialization, load their values now.
5521 */
b8ff05a9
DM		 5522 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5523 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5524 adap->params.b_wnd);
7ee9ff94 5525
dcf7b6f5 5526 t4_init_tp_params(adap);
636f9d37 5527 adap->flags |= FW_OK;
b8ff05a9
DM		 5528 return 0;
5529
5530 /*
636f9d37
VP		 5531 * Something bad happened. If a command timed out or failed with EIO
5532 * FW does not operate within its spec or something catastrophic
5533 * happened to HW/FW, stop issuing commands.
b8ff05a9 5534 */
636f9d37
VP		 5535bye:
5536 if (ret != -ETIMEDOUT && ret != -EIO)
5537 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5538 return ret;
5539}
5540
204dc3c0
DM		 5541/* EEH callbacks */
5542
5543static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5544 pci_channel_state_t state)
5545{
5546 int i;
5547 struct adapter *adap = pci_get_drvdata(pdev);
5548
5549 if (!adap)
5550 goto out;
5551
5552 rtnl_lock();
5553 adap->flags &= ~FW_OK;
5554 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5555 spin_lock(&adap->stats_lock);
204dc3c0
DM		 5556 for_each_port(adap, i) {
5557 struct net_device *dev = adap->port[i];
5558
5559 netif_device_detach(dev);
5560 netif_carrier_off(dev);
5561 }
9fe6cb58 5562 spin_unlock(&adap->stats_lock);
204dc3c0
DM		 5563 if (adap->flags & FULL_INIT_DONE)
5564 cxgb_down(adap);
5565 rtnl_unlock();
144be3d9
GS		 5566 if ((adap->flags & DEV_ENABLED)) {
5567 pci_disable_device(pdev);
5568 adap->flags &= ~DEV_ENABLED;
5569 }
204dc3c0
DM		 5570out: return state == pci_channel_io_perm_failure ?
5571 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5572}
5573
5574static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5575{
5576 int i, ret;
5577 struct fw_caps_config_cmd c;
5578 struct adapter *adap = pci_get_drvdata(pdev);
5579
5580 if (!adap) {
5581 pci_restore_state(pdev);
5582 pci_save_state(pdev);
5583 return PCI_ERS_RESULT_RECOVERED;
5584 }
5585
144be3d9
GS		 5586 if (!(adap->flags & DEV_ENABLED)) {
5587 if (pci_enable_device(pdev)) {
5588 dev_err(&pdev->dev, "Cannot reenable PCI "
5589 "device after reset\n");
5590 return PCI_ERS_RESULT_DISCONNECT;
5591 }
5592 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 5593 }
5594
5595 pci_set_master(pdev);
5596 pci_restore_state(pdev);
5597 pci_save_state(pdev);
5598 pci_cleanup_aer_uncorrect_error_status(pdev);
5599
5600 if (t4_wait_dev_ready(adap) < 0)
5601 return PCI_ERS_RESULT_DISCONNECT;
777c2300 5602 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 5603 return PCI_ERS_RESULT_DISCONNECT;
5604 adap->flags |= FW_OK;
5605 if (adap_init1(adap, &c))
5606 return PCI_ERS_RESULT_DISCONNECT;
5607
5608 for_each_port(adap, i) {
5609 struct port_info *p = adap2pinfo(adap, i);
5610
060e0c75
DM		 5611 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5612 NULL, NULL);
204dc3c0
DM		 5613 if (ret < 0)
5614 return PCI_ERS_RESULT_DISCONNECT;
5615 p->viid = ret;
5616 p->xact_addr_filt = -1;
5617 }
5618
5619 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5620 adap->params.b_wnd);
1ae970e0 5621 setup_memwin(adap);
204dc3c0
DM		 5622 if (cxgb_up(adap))
5623 return PCI_ERS_RESULT_DISCONNECT;
5624 return PCI_ERS_RESULT_RECOVERED;
5625}
5626
5627static void eeh_resume(struct pci_dev *pdev)
5628{
5629 int i;
5630 struct adapter *adap = pci_get_drvdata(pdev);
5631
5632 if (!adap)
5633 return;
5634
5635 rtnl_lock();
5636 for_each_port(adap, i) {
5637 struct net_device *dev = adap->port[i];
5638
5639 if (netif_running(dev)) {
5640 link_start(dev);
5641 cxgb_set_rxmode(dev);
5642 }
5643 netif_device_attach(dev);
5644 }
5645 rtnl_unlock();
5646}
5647
3646f0e5 5648static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 5649 .error_detected = eeh_err_detected,
5650 .slot_reset = eeh_slot_reset,
5651 .resume = eeh_resume,
5652};
5653
57d8b764 5654static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 5655{
57d8b764
KS		 5656 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5657 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 5658}
5659
5660static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
5661 unsigned int size, unsigned int iqe_size)
5662{
5663 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
5664 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
5665 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
5666 q->iqe_len = iqe_size;
5667 q->size = size;
5668}
5669
5670/*
5671 * Perform default configuration of DMA queues depending on the number and type
5672 * of ports we found and the number of available CPUs. Most settings can be
5673 * modified by the admin prior to actual use.
5674 */
91744948 5675static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 5676{
5677 struct sge *s = &adap->sge;
5678 int i, q10g = 0, n10g = 0, qidx = 0;
5679
5680 for_each_port(adap, i)
57d8b764 5681 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
b8ff05a9
DM		 5682
5683 /*
5684 * We default to 1 queue per non-10G port and up to # of cores queues
5685 * per 10G port.
5686 */
5687 if (n10g)
5688 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 5689 if (q10g > netif_get_num_default_rss_queues())
5690 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 5691
5692 for_each_port(adap, i) {
5693 struct port_info *pi = adap2pinfo(adap, i);
5694
5695 pi->first_qset = qidx;
57d8b764 5696 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 5697 qidx += pi->nqsets;
5698 }
5699
5700 s->ethqsets = qidx;
5701 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5702
5703 if (is_offload(adap)) {
5704 /*
5705 * For offload we use 1 queue/channel if all ports are up to 1G,
5706 * otherwise we divide all available queues amongst the channels
5707 * capped by the number of available cores.
5708 */
5709 if (n10g) {
5710 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5711 num_online_cpus());
5712 s->ofldqsets = roundup(i, adap->params.nports);
5713 } else
5714 s->ofldqsets = adap->params.nports;
5715 /* For RDMA one Rx queue per channel suffices */
5716 s->rdmaqs = adap->params.nports;
5717 }
5718
5719 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5720 struct sge_eth_rxq *r = &s->ethrxq[i];
5721
5722 init_rspq(&r->rspq, 0, 0, 1024, 64);
5723 r->fl.size = 72;
5724 }
5725
5726 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5727 s->ethtxq[i].q.size = 1024;
5728
5729 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5730 s->ctrlq[i].q.size = 512;
5731
5732 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5733 s->ofldtxq[i].q.size = 1024;
5734
5735 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5736 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5737
5738 init_rspq(&r->rspq, 0, 0, 1024, 64);
5739 r->rspq.uld = CXGB4_ULD_ISCSI;
5740 r->fl.size = 72;
5741 }
5742
5743 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5744 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5745
5746 init_rspq(&r->rspq, 0, 0, 511, 64);
5747 r->rspq.uld = CXGB4_ULD_RDMA;
5748 r->fl.size = 72;
5749 }
5750
5751 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
5752 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
5753}
5754
5755/*
5756 * Reduce the number of Ethernet queues across all ports to at most n.
5757 * n provides at least one queue per port.
5758 */
91744948 5759static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 5760{
5761 int i;
5762 struct port_info *pi;
5763
5764 while (n < adap->sge.ethqsets)
5765 for_each_port(adap, i) {
5766 pi = adap2pinfo(adap, i);
5767 if (pi->nqsets > 1) {
5768 pi->nqsets--;
5769 adap->sge.ethqsets--;
5770 if (adap->sge.ethqsets <= n)
5771 break;
5772 }
5773 }
5774
5775 n = 0;
5776 for_each_port(adap, i) {
5777 pi = adap2pinfo(adap, i);
5778 pi->first_qset = n;
5779 n += pi->nqsets;
5780 }
5781}
5782
5783/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5784#define EXTRA_VECS 2
5785
91744948 5786static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 5787{
5788 int ofld_need = 0;
c32ad224 5789 int i, want, need;
b8ff05a9
DM		 5790 struct sge *s = &adap->sge;
5791 unsigned int nchan = adap->params.nports;
5792 struct msix_entry entries[MAX_INGQ + 1];
5793
5794 for (i = 0; i < ARRAY_SIZE(entries); ++i)
5795 entries[i].entry = i;
5796
5797 want = s->max_ethqsets + EXTRA_VECS;
5798 if (is_offload(adap)) {
5799 want += s->rdmaqs + s->ofldqsets;
5800 /* need nchan for each possible ULD */
5801 ofld_need = 2 * nchan;
5802 }
5803 need = adap->params.nports + EXTRA_VECS + ofld_need;
5804
c32ad224
AG		 5805 want = pci_enable_msix_range(adap->pdev, entries, need, want);
5806 if (want < 0)
5807 return want;
b8ff05a9 5808
c32ad224
AG		 5809 /*
5810 * Distribute available vectors to the various queue groups.
5811 * Every group gets its minimum requirement and NIC gets top
5812 * priority for leftovers.
5813 */
5814 i = want - EXTRA_VECS - ofld_need;
5815 if (i < s->max_ethqsets) {
5816 s->max_ethqsets = i;
5817 if (i < s->ethqsets)
5818 reduce_ethqs(adap, i);
5819 }
5820 if (is_offload(adap)) {
5821 i = want - EXTRA_VECS - s->max_ethqsets;
5822 i -= ofld_need - nchan;
5823 s->ofldqsets = (i / nchan) * nchan; /* round down */
5824 }
5825 for (i = 0; i < want; ++i)
5826 adap->msix_info[i].vec = entries[i].vector;
5827
5828 return 0;
b8ff05a9
DM		 5829}
5830
5831#undef EXTRA_VECS
5832
91744948 5833static int init_rss(struct adapter *adap)
671b0060
DM		 5834{
5835 unsigned int i, j;
5836
5837 for_each_port(adap, i) {
5838 struct port_info *pi = adap2pinfo(adap, i);
5839
5840 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5841 if (!pi->rss)
5842 return -ENOMEM;
5843 for (j = 0; j < pi->rss_size; j++)
278bc429 5844 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 5845 }
5846 return 0;
5847}
5848
91744948 5849static void print_port_info(const struct net_device *dev)
b8ff05a9 5850{
b8ff05a9 5851 char buf[80];
118969ed 5852 char *bufp = buf;
f1a051b9 5853 const char *spd = "";
118969ed
DM		 5854 const struct port_info *pi = netdev_priv(dev);
5855 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 5856
5857 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5858 spd = " 2.5 GT/s";
5859 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5860 spd = " 5 GT/s";
b8ff05a9 5861
118969ed
DM		 5862 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5863 bufp += sprintf(bufp, "100/");
5864 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5865 bufp += sprintf(bufp, "1000/");
5866 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5867 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 5868 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5869 bufp += sprintf(bufp, "40G/");
118969ed
DM		 5870 if (bufp != buf)
5871 --bufp;
72aca4bf 5872 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 5873
5874 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 5875 adap->params.vpd.id,
d14807dd 5876 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 5877 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5878 (adap->flags & USING_MSIX) ? " MSI-X" :
5879 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 5880 netdev_info(dev, "S/N: %s, P/N: %s\n",
5881 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 5882}
5883
91744948 5884static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 5885{
e5c8ae5f 5886 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 5887}
5888
06546391
DM		 5889/*
5890 * Free the following resources:
5891 * - memory used for tables
5892 * - MSI/MSI-X
5893 * - net devices
5894 * - resources FW is holding for us
5895 */
5896static void free_some_resources(struct adapter *adapter)
5897{
5898 unsigned int i;
5899
5900 t4_free_mem(adapter->l2t);
5901 t4_free_mem(adapter->tids.tid_tab);
5902 disable_msi(adapter);
5903
5904 for_each_port(adapter, i)
671b0060
DM		 5905 if (adapter->port[i]) {
5906 kfree(adap2pinfo(adapter, i)->rss);
06546391 5907 free_netdev(adapter->port[i]);
671b0060 5908 }
06546391 5909 if (adapter->flags & FW_OK)
060e0c75 5910 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 5911}
5912
2ed28baa 5913#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 5914#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 5915 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 5916#define SEGMENT_SIZE 128
b8ff05a9 5917
1dd06ae8 5918static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 5919{
22adfe0a 5920 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 5921 struct port_info *pi;
c8f44aff 5922 bool highdma = false;
b8ff05a9
DM		 5923 struct adapter *adapter = NULL;
5924
5925 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5926
5927 err = pci_request_regions(pdev, KBUILD_MODNAME);
5928 if (err) {
5929 /* Just info, some other driver may have claimed the device. */
5930 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5931 return err;
5932 }
5933
060e0c75 5934 /* We control everything through one PF */
b8ff05a9 5935 func = PCI_FUNC(pdev->devfn);
060e0c75 5936 if (func != ent->driver_data) {
204dc3c0 5937 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 5938 goto sriov;
204dc3c0 5939 }
b8ff05a9
DM		 5940
5941 err = pci_enable_device(pdev);
5942 if (err) {
5943 dev_err(&pdev->dev, "cannot enable PCI device\n");
5944 goto out_release_regions;
5945 }
5946
5947 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 5948 highdma = true;
b8ff05a9
DM		 5949 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5950 if (err) {
5951 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5952 "coherent allocations\n");
5953 goto out_disable_device;
5954 }
5955 } else {
5956 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5957 if (err) {
5958 dev_err(&pdev->dev, "no usable DMA configuration\n");
5959 goto out_disable_device;
5960 }
5961 }
5962
5963 pci_enable_pcie_error_reporting(pdev);
ef306b50 5964 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 5965 pci_set_master(pdev);
5966 pci_save_state(pdev);
5967
5968 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5969 if (!adapter) {
5970 err = -ENOMEM;
5971 goto out_disable_device;
5972 }
5973
144be3d9
GS		 5974 /* PCI device has been enabled */
5975 adapter->flags |= DEV_ENABLED;
5976
b8ff05a9
DM		 5977 adapter->regs = pci_ioremap_bar(pdev, 0);
5978 if (!adapter->regs) {
5979 dev_err(&pdev->dev, "cannot map device registers\n");
5980 err = -ENOMEM;
5981 goto out_free_adapter;
5982 }
5983
5984 adapter->pdev = pdev;
5985 adapter->pdev_dev = &pdev->dev;
3069ee9b 5986 adapter->mbox = func;
060e0c75 5987 adapter->fn = func;
b8ff05a9
DM		 5988 adapter->msg_enable = dflt_msg_enable;
5989 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
5990
5991 spin_lock_init(&adapter->stats_lock);
5992 spin_lock_init(&adapter->tid_release_lock);
5993
5994 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 5995 INIT_WORK(&adapter->db_full_task, process_db_full);
5996 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 5997
5998 err = t4_prep_adapter(adapter);
5999 if (err)
22adfe0a
SR		 6000 goto out_unmap_bar0;
6001
d14807dd 6002 if (!is_t4(adapter->params.chip)) {
22adfe0a
SR		 6003 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
6004 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
6005 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
6006 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6007
6008 /* Each segment size is 128B. Write coalescing is enabled only
6009 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6010 * queue is less no of segments that can be accommodated in
6011 * a page size.
6012 */
6013 if (qpp > num_seg) {
6014 dev_err(&pdev->dev,
6015 "Incorrect number of egress queues per page\n");
6016 err = -EINVAL;
6017 goto out_unmap_bar0;
6018 }
6019 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6020 pci_resource_len(pdev, 2));
6021 if (!adapter->bar2) {
6022 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6023 err = -ENOMEM;
6024 goto out_unmap_bar0;
6025 }
6026 }
6027
636f9d37 6028 setup_memwin(adapter);
b8ff05a9 6029 err = adap_init0(adapter);
636f9d37 6030 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6031 if (err)
6032 goto out_unmap_bar;
6033
6034 for_each_port(adapter, i) {
6035 struct net_device *netdev;
6036
6037 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6038 MAX_ETH_QSETS);
6039 if (!netdev) {
6040 err = -ENOMEM;
6041 goto out_free_dev;
6042 }
6043
6044 SET_NETDEV_DEV(netdev, &pdev->dev);
6045
6046 adapter->port[i] = netdev;
6047 pi = netdev_priv(netdev);
6048 pi->adapter = adapter;
6049 pi->xact_addr_filt = -1;
b8ff05a9 6050 pi->port_id = i;
b8ff05a9
DM		 6051 netdev->irq = pdev->irq;
6052
2ed28baa
MM		 6053 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6054 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6055 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6056 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6057 if (highdma)
6058 netdev->hw_features |= NETIF_F_HIGHDMA;
6059 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6060 netdev->vlan_features = netdev->features & VLAN_FEAT;
6061
01789349
JP		 6062 netdev->priv_flags |= IFF_UNICAST_FLT;
6063
b8ff05a9
DM		 6064 netdev->netdev_ops = &cxgb4_netdev_ops;
6065 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
6066 }
6067
6068 pci_set_drvdata(pdev, adapter);
6069
6070 if (adapter->flags & FW_OK) {
060e0c75 6071 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6072 if (err)
6073 goto out_free_dev;
6074 }
6075
6076 /*
6077 * Configure queues and allocate tables now, they can be needed as
6078 * soon as the first register_netdev completes.
6079 */
6080 cfg_queues(adapter);
6081
6082 adapter->l2t = t4_init_l2t();
6083 if (!adapter->l2t) {
6084 /* We tolerate a lack of L2T, giving up some functionality */
6085 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6086 adapter->params.offload = 0;
6087 }
6088
6089 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6090 dev_warn(&pdev->dev, "could not allocate TID table, "
6091 "continuing\n");
6092 adapter->params.offload = 0;
6093 }
6094
f7cabcdd
DM		 6095 /* See what interrupts we'll be using */
6096 if (msi > 1 && enable_msix(adapter) == 0)
6097 adapter->flags |= USING_MSIX;
6098 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6099 adapter->flags |= USING_MSI;
6100
671b0060
DM		 6101 err = init_rss(adapter);
6102 if (err)
6103 goto out_free_dev;
6104
b8ff05a9
DM		 6105 /*
6106 * The card is now ready to go. If any errors occur during device
6107 * registration we do not fail the whole card but rather proceed only
6108 * with the ports we manage to register successfully. However we must
6109 * register at least one net device.
6110 */
6111 for_each_port(adapter, i) {
a57cabe0
DM		 6112 pi = adap2pinfo(adapter, i);
6113 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6114 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6115
b8ff05a9
DM		 6116 err = register_netdev(adapter->port[i]);
6117 if (err)
b1a3c2b6 6118 break;
b1a3c2b6
DM		 6119 adapter->chan_map[pi->tx_chan] = i;
6120 print_port_info(adapter->port[i]);
b8ff05a9 6121 }
b1a3c2b6 6122 if (i == 0) {
b8ff05a9
DM		 6123 dev_err(&pdev->dev, "could not register any net devices\n");
6124 goto out_free_dev;
6125 }
b1a3c2b6
DM		 6126 if (err) {
6127 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6128 err = 0;
6403eab1 6129 }
b8ff05a9
DM		 6130
6131 if (cxgb4_debugfs_root) {
6132 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6133 cxgb4_debugfs_root);
6134 setup_debugfs(adapter);
6135 }
6136
6482aa7c
DLR		 6137 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6138 pdev->needs_freset = 1;
6139
b8ff05a9
DM		 6140 if (is_offload(adapter))
6141 attach_ulds(adapter);
6142
b8ff05a9
DM		 6143sriov:
6144#ifdef CONFIG_PCI_IOV
7d6727cf 6145 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6146 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6147 dev_info(&pdev->dev,
6148 "instantiated %u virtual functions\n",
6149 num_vf[func]);
6150#endif
6151 return 0;
6152
6153 out_free_dev:
06546391 6154 free_some_resources(adapter);
b8ff05a9 6155 out_unmap_bar:
d14807dd 6156 if (!is_t4(adapter->params.chip))
22adfe0a
SR		 6157 iounmap(adapter->bar2);
6158 out_unmap_bar0:
b8ff05a9
DM		 6159 iounmap(adapter->regs);
6160 out_free_adapter:
6161 kfree(adapter);
6162 out_disable_device:
6163 pci_disable_pcie_error_reporting(pdev);
6164 pci_disable_device(pdev);
6165 out_release_regions:
6166 pci_release_regions(pdev);
b8ff05a9
DM		 6167 return err;
6168}
6169
91744948 6170static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6171{
6172 struct adapter *adapter = pci_get_drvdata(pdev);
6173
636f9d37 6174#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6175 pci_disable_sriov(pdev);
6176
636f9d37
VP		 6177#endif
6178
b8ff05a9
DM		 6179 if (adapter) {
6180 int i;
6181
6182 if (is_offload(adapter))
6183 detach_ulds(adapter);
6184
6185 for_each_port(adapter, i)
8f3a7676 6186 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6187 unregister_netdev(adapter->port[i]);
6188
6189 if (adapter->debugfs_root)
6190 debugfs_remove_recursive(adapter->debugfs_root);
6191
f2b7e78d
VP		 6192 /* If we allocated filters, free up state associated with any
6193 * valid filters ...
6194 */
6195 if (adapter->tids.ftid_tab) {
6196 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6197 for (i = 0; i < (adapter->tids.nftids +
6198 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6199 if (f->valid)
6200 clear_filter(adapter, f);
6201 }
6202
aaefae9b
DM		 6203 if (adapter->flags & FULL_INIT_DONE)
6204 cxgb_down(adapter);
b8ff05a9 6205
06546391 6206 free_some_resources(adapter);
b8ff05a9 6207 iounmap(adapter->regs);
d14807dd 6208 if (!is_t4(adapter->params.chip))
22adfe0a 6209 iounmap(adapter->bar2);
b8ff05a9 6210 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6211 if ((adapter->flags & DEV_ENABLED)) {
6212 pci_disable_device(pdev);
6213 adapter->flags &= ~DEV_ENABLED;
6214 }
b8ff05a9 6215 pci_release_regions(pdev);
8b662fe7 6216 kfree(adapter);
a069ec91 6217 } else
b8ff05a9
DM		 6218 pci_release_regions(pdev);
6219}
6220
6221static struct pci_driver cxgb4_driver = {
6222 .name = KBUILD_MODNAME,
6223 .id_table = cxgb4_pci_tbl,
6224 .probe = init_one,
91744948 6225 .remove = remove_one,
687d705c 6226 .shutdown = remove_one,
204dc3c0 6227 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6228};
6229
6230static int __init cxgb4_init_module(void)
6231{
6232 int ret;
6233
3069ee9b
VP		 6234 workq = create_singlethread_workqueue("cxgb4");
6235 if (!workq)
6236 return -ENOMEM;
6237
b8ff05a9
DM		 6238 /* Debugfs support is optional, just warn if this fails */
6239 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6240 if (!cxgb4_debugfs_root)
428ac43f 6241 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6242
6243 ret = pci_register_driver(&cxgb4_driver);
73a695f8 6244 if (ret < 0) {
b8ff05a9 6245 debugfs_remove(cxgb4_debugfs_root);
73a695f8
WY		 6246 destroy_workqueue(workq);
6247 }
01bcca68
VP		 6248
6249 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6250
b8ff05a9
DM		 6251 return ret;
6252}
6253
6254static void __exit cxgb4_cleanup_module(void)
6255{
01bcca68 6256 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
b8ff05a9
DM		 6257 pci_unregister_driver(&cxgb4_driver);
6258 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3069ee9b
VP		 6259 flush_workqueue(workq);
6260 destroy_workqueue(workq);
b8ff05a9
DM		 6261}
6262
6263module_init(cxgb4_init_module);
6264module_exit(cxgb4_cleanup_module);
Linux 6.x block layer and io_uring tree(s)