projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/smscx5xx: use the device tree for mac address
[linux-2.6-block.git] / drivers / net / ethernet / chelsio / cxgb3 / cxgb3_main.c
CommitLineData
4d22de3e 1/*
a02d44a0 2 * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
4d22de3e 3 *
1d68e93d
DLR		 4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
4d22de3e 9 *
1d68e93d
DLR		 10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
4d22de3e 31 */
428ac43f
JP		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
4d22de3e
DLR		 35#include <linux/module.h>
36#include <linux/moduleparam.h>
37#include <linux/init.h>
38#include <linux/pci.h>
39#include <linux/dma-mapping.h>
40#include <linux/netdevice.h>
41#include <linux/etherdevice.h>
42#include <linux/if_vlan.h>
0f07c4ee 43#include <linux/mdio.h>
4d22de3e
DLR		 44#include <linux/sockios.h>
45#include <linux/workqueue.h>
46#include <linux/proc_fs.h>
47#include <linux/rtnetlink.h>
2e283962 48#include <linux/firmware.h>
d9da466a 49#include <linux/log2.h>
34336ec0 50#include <linux/stringify.h>
e998f245 51#include <linux/sched.h>
5a0e3ad6 52#include <linux/slab.h>
4d22de3e
DLR		 53#include <asm/uaccess.h>
54
55#include "common.h"
56#include "cxgb3_ioctl.h"
57#include "regs.h"
58#include "cxgb3_offload.h"
59#include "version.h"
60
61#include "cxgb3_ctl_defs.h"
62#include "t3_cpl.h"
63#include "firmware_exports.h"
64
65enum {
66 MAX_TXQ_ENTRIES = 16384,
67 MAX_CTRL_TXQ_ENTRIES = 1024,
68 MAX_RSPQ_ENTRIES = 16384,
69 MAX_RX_BUFFERS = 16384,
70 MAX_RX_JUMBO_BUFFERS = 16384,
71 MIN_TXQ_ENTRIES = 4,
72 MIN_CTRL_TXQ_ENTRIES = 4,
73 MIN_RSPQ_ENTRIES = 32,
74 MIN_FL_ENTRIES = 32
75};
76
77#define PORT_MASK ((1 << MAX_NPORTS) - 1)
78
79#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
80 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
81 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
82
83#define EEPROM_MAGIC 0x38E2F10C
84
678771d6
DLR		 85#define CH_DEVICE(devid, idx) \
86 { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
4d22de3e 87
9baa3c34 88static const struct pci_device_id cxgb3_pci_tbl[] = {
678771d6
DLR		 89 CH_DEVICE(0x20, 0), /* PE9000 */
90 CH_DEVICE(0x21, 1), /* T302E */
91 CH_DEVICE(0x22, 2), /* T310E */
92 CH_DEVICE(0x23, 3), /* T320X */
93 CH_DEVICE(0x24, 1), /* T302X */
94 CH_DEVICE(0x25, 3), /* T320E */
95 CH_DEVICE(0x26, 2), /* T310X */
96 CH_DEVICE(0x30, 2), /* T3B10 */
97 CH_DEVICE(0x31, 3), /* T3B20 */
98 CH_DEVICE(0x32, 1), /* T3B02 */
ce03aadd 99 CH_DEVICE(0x35, 6), /* T3C20-derived T3C10 */
74451424
DLR		 100 CH_DEVICE(0x36, 3), /* S320E-CR */
101 CH_DEVICE(0x37, 7), /* N320E-G2 */
4d22de3e
DLR		 102 {0,}
103};
104
105MODULE_DESCRIPTION(DRV_DESC);
106MODULE_AUTHOR("Chelsio Communications");
1d68e93d 107MODULE_LICENSE("Dual BSD/GPL");
4d22de3e
DLR		 108MODULE_VERSION(DRV_VERSION);
109MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
110
111static int dflt_msg_enable = DFLT_MSG_ENABLE;
112
113module_param(dflt_msg_enable, int, 0644);
114MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
115
116/*
117 * The driver uses the best interrupt scheme available on a platform in the
118 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
119 * of these schemes the driver may consider as follows:
120 *
121 * msi = 2: choose from among all three options
122 * msi = 1: only consider MSI and pin interrupts
123 * msi = 0: force pin interrupts
124 */
125static int msi = 2;
126
127module_param(msi, int, 0644);
128MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
129
130/*
131 * The driver enables offload as a default.
132 * To disable it, use ofld_disable = 1.
133 */
134
135static int ofld_disable = 0;
136
137module_param(ofld_disable, int, 0644);
138MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
139
140/*
141 * We have work elements that we need to cancel when an interface is taken
142 * down. Normally the work elements would be executed by keventd but that
143 * can deadlock because of linkwatch. If our close method takes the rtnl
144 * lock and linkwatch is ahead of our work elements in keventd, linkwatch
145 * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
146 * for our work to complete. Get our own work queue to solve this.
147 */
e998f245 148struct workqueue_struct *cxgb3_wq;
4d22de3e
DLR		 149
150/**
151 * link_report - show link status and link speed/duplex
152 * @p: the port whose settings are to be reported
153 *
154 * Shows the link status, speed, and duplex of a port.
155 */
156static void link_report(struct net_device *dev)
157{
158 if (!netif_carrier_ok(dev))
428ac43f 159 netdev_info(dev, "link down\n");
4d22de3e
DLR		 160 else {
161 const char *s = "10Mbps";
162 const struct port_info *p = netdev_priv(dev);
163
164 switch (p->link_config.speed) {
165 case SPEED_10000:
166 s = "10Gbps";
167 break;
168 case SPEED_1000:
169 s = "1000Mbps";
170 break;
171 case SPEED_100:
172 s = "100Mbps";
173 break;
174 }
175
428ac43f
JP		 176 netdev_info(dev, "link up, %s, %s-duplex\n",
177 s, p->link_config.duplex == DUPLEX_FULL
178 ? "full" : "half");
4d22de3e
DLR		 179 }
180}
181
34701fde
DLR		 182static void enable_tx_fifo_drain(struct adapter *adapter,
183 struct port_info *pi)
184{
185 t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
186 F_ENDROPPKT);
187 t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
188 t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
189 t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
190}
191
192static void disable_tx_fifo_drain(struct adapter *adapter,
193 struct port_info *pi)
194{
195 t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
196 F_ENDROPPKT, 0);
197}
198
bf792094
DLR		 199void t3_os_link_fault(struct adapter *adap, int port_id, int state)
200{
201 struct net_device *dev = adap->port[port_id];
202 struct port_info *pi = netdev_priv(dev);
203
204 if (state == netif_carrier_ok(dev))
205 return;
206
207 if (state) {
208 struct cmac *mac = &pi->mac;
209
210 netif_carrier_on(dev);
211
34701fde
DLR		 212 disable_tx_fifo_drain(adap, pi);
213
bf792094
DLR		 214 /* Clear local faults */
215 t3_xgm_intr_disable(adap, pi->port_id);
216 t3_read_reg(adap, A_XGM_INT_STATUS +
217 pi->mac.offset);
218 t3_write_reg(adap,
219 A_XGM_INT_CAUSE + pi->mac.offset,
220 F_XGM_INT);
221
222 t3_set_reg_field(adap,
223 A_XGM_INT_ENABLE +
224 pi->mac.offset,
225 F_XGM_INT, F_XGM_INT);
226 t3_xgm_intr_enable(adap, pi->port_id);
227
228 t3_mac_enable(mac, MAC_DIRECTION_TX);
34701fde 229 } else {
bf792094
DLR		 230 netif_carrier_off(dev);
231
34701fde
DLR		 232 /* Flush TX FIFO */
233 enable_tx_fifo_drain(adap, pi);
234 }
bf792094
DLR		 235 link_report(dev);
236}
237
4d22de3e
DLR		 238/**
239 * t3_os_link_changed - handle link status changes
240 * @adapter: the adapter associated with the link change
241 * @port_id: the port index whose limk status has changed
242 * @link_stat: the new status of the link
243 * @speed: the new speed setting
244 * @duplex: the new duplex setting
245 * @pause: the new flow-control setting
246 *
247 * This is the OS-dependent handler for link status changes. The OS
248 * neutral handler takes care of most of the processing for these events,
249 * then calls this handler for any OS-specific processing.
250 */
251void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
252 int speed, int duplex, int pause)
253{
254 struct net_device *dev = adapter->port[port_id];
6d6dabac
DLR		 255 struct port_info *pi = netdev_priv(dev);
256 struct cmac *mac = &pi->mac;
4d22de3e
DLR		 257
258 /* Skip changes from disabled ports. */
259 if (!netif_running(dev))
260 return;
261
262 if (link_stat != netif_carrier_ok(dev)) {
6d6dabac 263 if (link_stat) {
34701fde
DLR		 264 disable_tx_fifo_drain(adapter, pi);
265
59cf8107 266 t3_mac_enable(mac, MAC_DIRECTION_RX);
bf792094
DLR		 267
268 /* Clear local faults */
269 t3_xgm_intr_disable(adapter, pi->port_id);
270 t3_read_reg(adapter, A_XGM_INT_STATUS +
271 pi->mac.offset);
272 t3_write_reg(adapter,
273 A_XGM_INT_CAUSE + pi->mac.offset,
274 F_XGM_INT);
275
276 t3_set_reg_field(adapter,
277 A_XGM_INT_ENABLE + pi->mac.offset,
278 F_XGM_INT, F_XGM_INT);
279 t3_xgm_intr_enable(adapter, pi->port_id);
280
4d22de3e 281 netif_carrier_on(dev);
6d6dabac 282 } else {
4d22de3e 283 netif_carrier_off(dev);
bf792094
DLR		 284
285 t3_xgm_intr_disable(adapter, pi->port_id);
286 t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
287 t3_set_reg_field(adapter,
288 A_XGM_INT_ENABLE + pi->mac.offset,
289 F_XGM_INT, 0);
290
291 if (is_10G(adapter))
292 pi->phy.ops->power_down(&pi->phy, 1);
293
294 t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
59cf8107
DLR		 295 t3_mac_disable(mac, MAC_DIRECTION_RX);
296 t3_link_start(&pi->phy, mac, &pi->link_config);
34701fde
DLR		 297
298 /* Flush TX FIFO */
299 enable_tx_fifo_drain(adapter, pi);
6d6dabac
DLR		 300 }
301
4d22de3e
DLR		 302 link_report(dev);
303 }
304}
305
1e882025
DLR		 306/**
307 * t3_os_phymod_changed - handle PHY module changes
308 * @phy: the PHY reporting the module change
309 * @mod_type: new module type
310 *
311 * This is the OS-dependent handler for PHY module changes. It is
312 * invoked when a PHY module is removed or inserted for any OS-specific
313 * processing.
314 */
315void t3_os_phymod_changed(struct adapter *adap, int port_id)
316{
317 static const char *mod_str[] = {
318 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
319 };
320
321 const struct net_device *dev = adap->port[port_id];
322 const struct port_info *pi = netdev_priv(dev);
323
324 if (pi->phy.modtype == phy_modtype_none)
428ac43f 325 netdev_info(dev, "PHY module unplugged\n");
1e882025 326 else
428ac43f
JP		 327 netdev_info(dev, "%s PHY module inserted\n",
328 mod_str[pi->phy.modtype]);
1e882025
DLR		 329}
330
4d22de3e
DLR		 331static void cxgb_set_rxmode(struct net_device *dev)
332{
4d22de3e
DLR		 333 struct port_info *pi = netdev_priv(dev);
334
0988d269 335 t3_mac_set_rx_mode(&pi->mac, dev);
4d22de3e
DLR		 336}
337
338/**
339 * link_start - enable a port
340 * @dev: the device to enable
341 *
342 * Performs the MAC and PHY actions needed to enable a port.
343 */
344static void link_start(struct net_device *dev)
345{
4d22de3e
DLR		 346 struct port_info *pi = netdev_priv(dev);
347 struct cmac *mac = &pi->mac;
348
4d22de3e 349 t3_mac_reset(mac);
f14d42f3 350 t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
4d22de3e 351 t3_mac_set_mtu(mac, dev->mtu);
f14d42f3
KX		 352 t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
353 t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
0988d269 354 t3_mac_set_rx_mode(mac, dev);
4d22de3e
DLR		 355 t3_link_start(&pi->phy, mac, &pi->link_config);
356 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
357}
358
359static inline void cxgb_disable_msi(struct adapter *adapter)
360{
361 if (adapter->flags & USING_MSIX) {
362 pci_disable_msix(adapter->pdev);
363 adapter->flags &= ~USING_MSIX;
364 } else if (adapter->flags & USING_MSI) {
365 pci_disable_msi(adapter->pdev);
366 adapter->flags &= ~USING_MSI;
367 }
368}
369
370/*
371 * Interrupt handler for asynchronous events used with MSI-X.
372 */
373static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
374{
375 t3_slow_intr_handler(cookie);
376 return IRQ_HANDLED;
377}
378
379/*
380 * Name the MSI-X interrupts.
381 */
382static void name_msix_vecs(struct adapter *adap)
383{
384 int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
385
386 snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
387 adap->msix_info[0].desc[n] = 0;
388
389 for_each_port(adap, j) {
390 struct net_device *d = adap->port[j];
391 const struct port_info *pi = netdev_priv(d);
392
393 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
394 snprintf(adap->msix_info[msi_idx].desc, n,
8c263761 395 "%s-%d", d->name, pi->first_qset + i);
4d22de3e
DLR		 396 adap->msix_info[msi_idx].desc[n] = 0;
397 }
8c263761 398 }
4d22de3e
DLR		 399}
400
401static int request_msix_data_irqs(struct adapter *adap)
402{
403 int i, j, err, qidx = 0;
404
405 for_each_port(adap, i) {
406 int nqsets = adap2pinfo(adap, i)->nqsets;
407
408 for (j = 0; j < nqsets; ++j) {
409 err = request_irq(adap->msix_info[qidx + 1].vec,
410 t3_intr_handler(adap,
411 adap->sge.qs[qidx].
412 rspq.polling), 0,
413 adap->msix_info[qidx + 1].desc,
414 &adap->sge.qs[qidx]);
415 if (err) {
416 while (--qidx >= 0)
417 free_irq(adap->msix_info[qidx + 1].vec,
418 &adap->sge.qs[qidx]);
419 return err;
420 }
421 qidx++;
422 }
423 }
424 return 0;
425}
426
8c263761
DLR		 427static void free_irq_resources(struct adapter *adapter)
428{
429 if (adapter->flags & USING_MSIX) {
430 int i, n = 0;
431
432 free_irq(adapter->msix_info[0].vec, adapter);
433 for_each_port(adapter, i)
5cda9364 434 n += adap2pinfo(adapter, i)->nqsets;
8c263761
DLR		 435
436 for (i = 0; i < n; ++i)
437 free_irq(adapter->msix_info[i + 1].vec,
438 &adapter->sge.qs[i]);
439 } else
440 free_irq(adapter->pdev->irq, adapter);
441}
442
b881955b
DLR		 443static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
444 unsigned long n)
445{
e95ef5d3 446 int attempts = 10;
b881955b
DLR		 447
448 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
449 if (!--attempts)
450 return -ETIMEDOUT;
451 msleep(10);
452 }
453 return 0;
454}
455
456static int init_tp_parity(struct adapter *adap)
457{
458 int i;
459 struct sk_buff *skb;
460 struct cpl_set_tcb_field *greq;
461 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
462
463 t3_tp_set_offload_mode(adap, 1);
464
465 for (i = 0; i < 16; i++) {
466 struct cpl_smt_write_req *req;
467
74b793e1
DLR		 468 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
469 if (!skb)
470 skb = adap->nofail_skb;
471 if (!skb)
472 goto alloc_skb_fail;
473
b881955b
DLR		 474 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
475 memset(req, 0, sizeof(*req));
476 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
477 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
dce7d1d0 478 req->mtu_idx = NMTUS - 1;
b881955b
DLR		 479 req->iff = i;
480 t3_mgmt_tx(adap, skb);
74b793e1
DLR		 481 if (skb == adap->nofail_skb) {
482 await_mgmt_replies(adap, cnt, i + 1);
483 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
484 if (!adap->nofail_skb)
485 goto alloc_skb_fail;
486 }
b881955b
DLR		 487 }
488
489 for (i = 0; i < 2048; i++) {
490 struct cpl_l2t_write_req *req;
491
74b793e1
DLR		 492 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
493 if (!skb)
494 skb = adap->nofail_skb;
495 if (!skb)
496 goto alloc_skb_fail;
497
b881955b
DLR		 498 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
499 memset(req, 0, sizeof(*req));
500 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
501 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
502 req->params = htonl(V_L2T_W_IDX(i));
503 t3_mgmt_tx(adap, skb);
74b793e1
DLR		 504 if (skb == adap->nofail_skb) {
505 await_mgmt_replies(adap, cnt, 16 + i + 1);
506 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
507 if (!adap->nofail_skb)
508 goto alloc_skb_fail;
509 }
b881955b
DLR		 510 }
511
512 for (i = 0; i < 2048; i++) {
513 struct cpl_rte_write_req *req;
514
74b793e1
DLR		 515 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
516 if (!skb)
517 skb = adap->nofail_skb;
518 if (!skb)
519 goto alloc_skb_fail;
520
b881955b
DLR		 521 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
522 memset(req, 0, sizeof(*req));
523 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
524 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
525 req->l2t_idx = htonl(V_L2T_W_IDX(i));
526 t3_mgmt_tx(adap, skb);
74b793e1
DLR		 527 if (skb == adap->nofail_skb) {
528 await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
529 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
530 if (!adap->nofail_skb)
531 goto alloc_skb_fail;
532 }
b881955b
DLR		 533 }
534
74b793e1
DLR		 535 skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
536 if (!skb)
537 skb = adap->nofail_skb;
538 if (!skb)
539 goto alloc_skb_fail;
540
b881955b
DLR		 541 greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
542 memset(greq, 0, sizeof(*greq));
543 greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
544 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
545 greq->mask = cpu_to_be64(1);
546 t3_mgmt_tx(adap, skb);
547
548 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
74b793e1
DLR		 549 if (skb == adap->nofail_skb) {
550 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
551 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
552 }
553
b881955b
DLR		 554 t3_tp_set_offload_mode(adap, 0);
555 return i;
74b793e1
DLR		 556
557alloc_skb_fail:
558 t3_tp_set_offload_mode(adap, 0);
559 return -ENOMEM;
b881955b
DLR		 560}
561
4d22de3e
DLR		 562/**
563 * setup_rss - configure RSS
564 * @adap: the adapter
565 *
566 * Sets up RSS to distribute packets to multiple receive queues. We
567 * configure the RSS CPU lookup table to distribute to the number of HW
568 * receive queues, and the response queue lookup table to narrow that
569 * down to the response queues actually configured for each port.
570 * We always configure the RSS mapping for two ports since the mapping
571 * table has plenty of entries.
572 */
573static void setup_rss(struct adapter *adap)
574{
575 int i;
576 unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
577 unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
578 u8 cpus[SGE_QSETS + 1];
579 u16 rspq_map[RSS_TABLE_SIZE];
580
581 for (i = 0; i < SGE_QSETS; ++i)
582 cpus[i] = i;
583 cpus[SGE_QSETS] = 0xff; /* terminator */
584
585 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
586 rspq_map[i] = i % nq0;
587 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
588 }
589
590 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
591 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
a2604be5 592 V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
4d22de3e
DLR		 593}
594
e998f245
SW		 595static void ring_dbs(struct adapter *adap)
596{
597 int i, j;
598
599 for (i = 0; i < SGE_QSETS; i++) {
600 struct sge_qset *qs = &adap->sge.qs[i];
601
602 if (qs->adap)
603 for (j = 0; j < SGE_TXQ_PER_SET; j++)
604 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
605 }
606}
607
bea3348e 608static void init_napi(struct adapter *adap)
4d22de3e 609{
bea3348e 610 int i;
4d22de3e 611
bea3348e
SH		 612 for (i = 0; i < SGE_QSETS; i++) {
613 struct sge_qset *qs = &adap->sge.qs[i];
4d22de3e 614
bea3348e
SH		 615 if (qs->adap)
616 netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
617 64);
4d22de3e 618 }
48c4b6db
DLR		 619
620 /*
621 * netif_napi_add() can be called only once per napi_struct because it
622 * adds each new napi_struct to a list. Be careful not to call it a
623 * second time, e.g., during EEH recovery, by making a note of it.
624 */
625 adap->flags |= NAPI_INIT;
4d22de3e
DLR		 626}
627
628/*
629 * Wait until all NAPI handlers are descheduled. This includes the handlers of
630 * both netdevices representing interfaces and the dummy ones for the extra
631 * queues.
632 */
633static void quiesce_rx(struct adapter *adap)
634{
635 int i;
4d22de3e 636
bea3348e
SH		 637 for (i = 0; i < SGE_QSETS; i++)
638 if (adap->sge.qs[i].adap)
639 napi_disable(&adap->sge.qs[i].napi);
640}
4d22de3e 641
bea3348e
SH		 642static void enable_all_napi(struct adapter *adap)
643{
644 int i;
645 for (i = 0; i < SGE_QSETS; i++)
646 if (adap->sge.qs[i].adap)
647 napi_enable(&adap->sge.qs[i].napi);
4d22de3e
DLR		 648}
649
650/**
651 * setup_sge_qsets - configure SGE Tx/Rx/response queues
652 * @adap: the adapter
653 *
654 * Determines how many sets of SGE queues to use and initializes them.
655 * We support multiple queue sets per port if we have MSI-X, otherwise
656 * just one queue set per port.
657 */
658static int setup_sge_qsets(struct adapter *adap)
659{
bea3348e 660 int i, j, err, irq_idx = 0, qset_idx = 0;
8ac3ba68 661 unsigned int ntxq = SGE_TXQ_PER_SET;
4d22de3e
DLR		 662
663 if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
664 irq_idx = -1;
665
666 for_each_port(adap, i) {
667 struct net_device *dev = adap->port[i];
bea3348e 668 struct port_info *pi = netdev_priv(dev);
4d22de3e 669
bea3348e 670 pi->qs = &adap->sge.qs[pi->first_qset];
e594e96e 671 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
4d22de3e
DLR		 672 err = t3_sge_alloc_qset(adap, qset_idx, 1,
673 (adap->flags & USING_MSIX) ? qset_idx + 1 :
674 irq_idx,
82ad3329
DLR		 675 &adap->params.sge.qset[qset_idx], ntxq, dev,
676 netdev_get_tx_queue(dev, j));
4d22de3e
DLR		 677 if (err) {
678 t3_free_sge_resources(adap);
679 return err;
680 }
681 }
682 }
683
684 return 0;
685}
686
3e5192ee 687static ssize_t attr_show(struct device *d, char *buf,
896392ef 688 ssize_t(*format) (struct net_device *, char *))
4d22de3e
DLR		 689{
690 ssize_t len;
4d22de3e
DLR		 691
692 /* Synchronize with ioctls that may shut down the device */
693 rtnl_lock();
896392ef 694 len = (*format) (to_net_dev(d), buf);
4d22de3e
DLR		 695 rtnl_unlock();
696 return len;
697}
698
3e5192ee 699static ssize_t attr_store(struct device *d,
0ee8d33c 700 const char *buf, size_t len,
896392ef 701 ssize_t(*set) (struct net_device *, unsigned int),
4d22de3e
DLR		 702 unsigned int min_val, unsigned int max_val)
703{
4d22de3e
DLR		 704 ssize_t ret;
705 unsigned int val;
4d22de3e
DLR		 706
707 if (!capable(CAP_NET_ADMIN))
708 return -EPERM;
709
e72c932d
LC		 710 ret = kstrtouint(buf, 0, &val);
711 if (ret)
712 return ret;
713 if (val < min_val || val > max_val)
4d22de3e
DLR		 714 return -EINVAL;
715
716 rtnl_lock();
896392ef 717 ret = (*set) (to_net_dev(d), val);
4d22de3e
DLR		 718 if (!ret)
719 ret = len;
720 rtnl_unlock();
721 return ret;
722}
723
724#define CXGB3_SHOW(name, val_expr) \
896392ef 725static ssize_t format_##name(struct net_device *dev, char *buf) \
4d22de3e 726{ \
5fbf816f
DLR		 727 struct port_info *pi = netdev_priv(dev); \
728 struct adapter *adap = pi->adapter; \
4d22de3e
DLR		 729 return sprintf(buf, "%u\n", val_expr); \
730} \
0ee8d33c
DLR		 731static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
732 char *buf) \
4d22de3e 733{ \
3e5192ee 734 return attr_show(d, buf, format_##name); \
4d22de3e
DLR		 735}
736
896392ef 737static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
4d22de3e 738{
5fbf816f
DLR		 739 struct port_info *pi = netdev_priv(dev);
740 struct adapter *adap = pi->adapter;
9f238486 741 int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
896392ef 742
4d22de3e
DLR		 743 if (adap->flags & FULL_INIT_DONE)
744 return -EBUSY;
745 if (val && adap->params.rev == 0)
746 return -EINVAL;
9f238486
DLR		 747 if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
748 min_tids)
4d22de3e
DLR		 749 return -EINVAL;
750 adap->params.mc5.nfilters = val;
751 return 0;
752}
753
0ee8d33c
DLR		 754static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
755 const char *buf, size_t len)
4d22de3e 756{
3e5192ee 757 return attr_store(d, buf, len, set_nfilters, 0, ~0);
4d22de3e
DLR		 758}
759
896392ef 760static ssize_t set_nservers(struct net_device *dev, unsigned int val)
4d22de3e 761{
5fbf816f
DLR		 762 struct port_info *pi = netdev_priv(dev);
763 struct adapter *adap = pi->adapter;
896392ef 764
4d22de3e
DLR		 765 if (adap->flags & FULL_INIT_DONE)
766 return -EBUSY;
9f238486
DLR		 767 if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
768 MC5_MIN_TIDS)
4d22de3e
DLR		 769 return -EINVAL;
770 adap->params.mc5.nservers = val;
771 return 0;
772}
773
0ee8d33c
DLR		 774static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
775 const char *buf, size_t len)
4d22de3e 776{
3e5192ee 777 return attr_store(d, buf, len, set_nservers, 0, ~0);
4d22de3e
DLR		 778}
779
780#define CXGB3_ATTR_R(name, val_expr) \
781CXGB3_SHOW(name, val_expr) \
0ee8d33c 782static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
4d22de3e
DLR		 783
784#define CXGB3_ATTR_RW(name, val_expr, store_method) \
785CXGB3_SHOW(name, val_expr) \
0ee8d33c 786static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
4d22de3e
DLR		 787
788CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
789CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
790CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
791
792static struct attribute *cxgb3_attrs[] = {
0ee8d33c
DLR		 793 &dev_attr_cam_size.attr,
794 &dev_attr_nfilters.attr,
795 &dev_attr_nservers.attr,
4d22de3e
DLR		 796 NULL
797};
798
799static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
800
3e5192ee 801static ssize_t tm_attr_show(struct device *d,
0ee8d33c 802 char *buf, int sched)
4d22de3e 803{
5fbf816f
DLR		 804 struct port_info *pi = netdev_priv(to_net_dev(d));
805 struct adapter *adap = pi->adapter;
4d22de3e 806 unsigned int v, addr, bpt, cpt;
5fbf816f 807 ssize_t len;
4d22de3e
DLR		 808
809 addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
810 rtnl_lock();
811 t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
812 v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
813 if (sched & 1)
814 v >>= 16;
815 bpt = (v >> 8) & 0xff;
816 cpt = v & 0xff;
817 if (!cpt)
818 len = sprintf(buf, "disabled\n");
819 else {
820 v = (adap->params.vpd.cclk * 1000) / cpt;
821 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
822 }
823 rtnl_unlock();
824 return len;
825}
826
3e5192ee 827static ssize_t tm_attr_store(struct device *d,
0ee8d33c 828 const char *buf, size_t len, int sched)
4d22de3e 829{
5fbf816f
DLR		 830 struct port_info *pi = netdev_priv(to_net_dev(d));
831 struct adapter *adap = pi->adapter;
832 unsigned int val;
4d22de3e 833 ssize_t ret;
4d22de3e
DLR		 834
835 if (!capable(CAP_NET_ADMIN))
836 return -EPERM;
837
e72c932d
LC		 838 ret = kstrtouint(buf, 0, &val);
839 if (ret)
840 return ret;
841 if (val > 10000000)
4d22de3e
DLR		 842 return -EINVAL;
843
844 rtnl_lock();
845 ret = t3_config_sched(adap, val, sched);
846 if (!ret)
847 ret = len;
848 rtnl_unlock();
849 return ret;
850}
851
852#define TM_ATTR(name, sched) \
0ee8d33c
DLR		 853static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
854 char *buf) \
4d22de3e 855{ \
3e5192ee 856 return tm_attr_show(d, buf, sched); \
4d22de3e 857} \
0ee8d33c
DLR		 858static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
859 const char *buf, size_t len) \
4d22de3e 860{ \
3e5192ee 861 return tm_attr_store(d, buf, len, sched); \
4d22de3e 862} \
0ee8d33c 863static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
4d22de3e
DLR		 864
865TM_ATTR(sched0, 0);
866TM_ATTR(sched1, 1);
867TM_ATTR(sched2, 2);
868TM_ATTR(sched3, 3);
869TM_ATTR(sched4, 4);
870TM_ATTR(sched5, 5);
871TM_ATTR(sched6, 6);
872TM_ATTR(sched7, 7);
873
874static struct attribute *offload_attrs[] = {
0ee8d33c
DLR		 875 &dev_attr_sched0.attr,
876 &dev_attr_sched1.attr,
877 &dev_attr_sched2.attr,
878 &dev_attr_sched3.attr,
879 &dev_attr_sched4.attr,
880 &dev_attr_sched5.attr,
881 &dev_attr_sched6.attr,
882 &dev_attr_sched7.attr,
4d22de3e
DLR		 883 NULL
884};
885
886static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
887
888/*
889 * Sends an sk_buff to an offload queue driver
890 * after dealing with any active network taps.
891 */
892static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
893{
894 int ret;
895
896 local_bh_disable();
897 ret = t3_offload_tx(tdev, skb);
898 local_bh_enable();
899 return ret;
900}
901
902static int write_smt_entry(struct adapter *adapter, int idx)
903{
904 struct cpl_smt_write_req *req;
f14d42f3 905 struct port_info *pi = netdev_priv(adapter->port[idx]);
4d22de3e
DLR		 906 struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
907
908 if (!skb)
909 return -ENOMEM;
910
911 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
912 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
913 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
914 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */
915 req->iff = idx;
4d22de3e 916 memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
f14d42f3 917 memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
4d22de3e
DLR		 918 skb->priority = 1;
919 offload_tx(&adapter->tdev, skb);
920 return 0;
921}
922
923static int init_smt(struct adapter *adapter)
924{
925 int i;
926
927 for_each_port(adapter, i)
928 write_smt_entry(adapter, i);
929 return 0;
930}
931
932static void init_port_mtus(struct adapter *adapter)
933{
934 unsigned int mtus = adapter->port[0]->mtu;
935
936 if (adapter->port[1])
937 mtus |= adapter->port[1]->mtu << 16;
938 t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
939}
940
8c263761 941static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
14ab9892
DLR		 942 int hi, int port)
943{
944 struct sk_buff *skb;
945 struct mngt_pktsched_wr *req;
8c263761 946 int ret;
14ab9892 947
74b793e1
DLR		 948 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
949 if (!skb)
950 skb = adap->nofail_skb;
951 if (!skb)
952 return -ENOMEM;
953
14ab9892
DLR		 954 req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
955 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
956 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
957 req->sched = sched;
958 req->idx = qidx;
959 req->min = lo;
960 req->max = hi;
961 req->binding = port;
8c263761 962 ret = t3_mgmt_tx(adap, skb);
74b793e1
DLR		 963 if (skb == adap->nofail_skb) {
964 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
965 GFP_KERNEL);
966 if (!adap->nofail_skb)
967 ret = -ENOMEM;
968 }
8c263761
DLR		 969
970 return ret;
14ab9892
DLR		 971}
972
8c263761 973static int bind_qsets(struct adapter *adap)
14ab9892 974{
8c263761 975 int i, j, err = 0;
14ab9892
DLR		 976
977 for_each_port(adap, i) {
978 const struct port_info *pi = adap2pinfo(adap, i);
979
8c263761
DLR		 980 for (j = 0; j < pi->nqsets; ++j) {
981 int ret = send_pktsched_cmd(adap, 1,
982 pi->first_qset + j, -1,
983 -1, i);
984 if (ret)
985 err = ret;
986 }
14ab9892 987 }
8c263761
DLR		 988
989 return err;
14ab9892
DLR		 990}
991
34336ec0
BH		 992#define FW_VERSION __stringify(FW_VERSION_MAJOR) "." \
993 __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
994#define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
995#define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "." \
996 __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
997#define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
2e8c07c3
DLR		 998#define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
999#define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
9450526a 1000#define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
34336ec0
BH		 1001MODULE_FIRMWARE(FW_FNAME);
1002MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1003MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1004MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1005MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1006MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
2e8c07c3
DLR		 1007
1008static inline const char *get_edc_fw_name(int edc_idx)
1009{
1010 const char *fw_name = NULL;
1011
1012 switch (edc_idx) {
1013 case EDC_OPT_AEL2005:
1014 fw_name = AEL2005_OPT_EDC_NAME;
1015 break;
1016 case EDC_TWX_AEL2005:
1017 fw_name = AEL2005_TWX_EDC_NAME;
1018 break;
1019 case EDC_TWX_AEL2020:
1020 fw_name = AEL2020_TWX_EDC_NAME;
1021 break;
1022 }
1023 return fw_name;
1024}
1025
1026int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1027{
1028 struct adapter *adapter = phy->adapter;
1029 const struct firmware *fw;
92a486ca 1030 const char *fw_name;
2e8c07c3
DLR		 1031 u32 csum;
1032 const __be32 *p;
1033 u16 *cache = phy->phy_cache;
92a486ca 1034 int i, ret = -EINVAL;
2e8c07c3 1035
92a486ca
KC		 1036 fw_name = get_edc_fw_name(edc_idx);
1037 if (fw_name)
1038 ret = request_firmware(&fw, fw_name, &adapter->pdev->dev);
2e8c07c3
DLR		 1039 if (ret < 0) {
1040 dev_err(&adapter->pdev->dev,
1041 "could not upgrade firmware: unable to load %s\n",
92a486ca 1042 fw_name);
2e8c07c3
DLR		 1043 return ret;
1044 }
1045
1046 /* check size, take checksum in account */
1047 if (fw->size > size + 4) {
1048 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1049 (unsigned int)fw->size, size + 4);
1050 ret = -EINVAL;
1051 }
1052
1053 /* compute checksum */
1054 p = (const __be32 *)fw->data;
1055 for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1056 csum += ntohl(p[i]);
1057
1058 if (csum != 0xffffffff) {
1059 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1060 csum);
1061 ret = -EINVAL;
1062 }
1063
1064 for (i = 0; i < size / 4 ; i++) {
1065 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1066 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1067 }
1068
1069 release_firmware(fw);
1070
1071 return ret;
1072}
2e283962
DLR		 1073
1074static int upgrade_fw(struct adapter *adap)
1075{
1076 int ret;
2e283962
DLR		 1077 const struct firmware *fw;
1078 struct device *dev = &adap->pdev->dev;
1079
34336ec0 1080 ret = request_firmware(&fw, FW_FNAME, dev);
2e283962
DLR		 1081 if (ret < 0) {
1082 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
34336ec0 1083 FW_FNAME);
2e283962
DLR		 1084 return ret;
1085 }
1086 ret = t3_load_fw(adap, fw->data, fw->size);
1087 release_firmware(fw);
47330077
DLR		 1088
1089 if (ret == 0)
1090 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1091 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1092 else
1093 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1094 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
2eab17ab 1095
47330077
DLR		 1096 return ret;
1097}
1098
1099static inline char t3rev2char(struct adapter *adapter)
1100{
1101 char rev = 0;
1102
1103 switch(adapter->params.rev) {
1104 case T3_REV_B:
1105 case T3_REV_B2:
1106 rev = 'b';
1107 break;
1aafee26
DLR		 1108 case T3_REV_C:
1109 rev = 'c';
1110 break;
47330077
DLR		 1111 }
1112 return rev;
1113}
1114
9265fabf 1115static int update_tpsram(struct adapter *adap)
47330077
DLR		 1116{
1117 const struct firmware *tpsram;
1118 char buf[64];
1119 struct device *dev = &adap->pdev->dev;
1120 int ret;
1121 char rev;
2eab17ab 1122
47330077
DLR		 1123 rev = t3rev2char(adap);
1124 if (!rev)
1125 return 0;
1126
34336ec0 1127 snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
47330077
DLR		 1128
1129 ret = request_firmware(&tpsram, buf, dev);
1130 if (ret < 0) {
1131 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1132 buf);
1133 return ret;
1134 }
2eab17ab 1135
47330077
DLR		 1136 ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1137 if (ret)
2eab17ab 1138 goto release_tpsram;
47330077
DLR		 1139
1140 ret = t3_set_proto_sram(adap, tpsram->data);
1141 if (ret == 0)
1142 dev_info(dev,
1143 "successful update of protocol engine "
1144 "to %d.%d.%d\n",
1145 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1146 else
1147 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1148 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1149 if (ret)
1150 dev_err(dev, "loading protocol SRAM failed\n");
1151
1152release_tpsram:
1153 release_firmware(tpsram);
2eab17ab 1154
2e283962
DLR		 1155 return ret;
1156}
1157
60158e64
RD		 1158/**
1159 * t3_synchronize_rx - wait for current Rx processing on a port to complete
1160 * @adap: the adapter
1161 * @p: the port
1162 *
1163 * Ensures that current Rx processing on any of the queues associated with
1164 * the given port completes before returning. We do this by acquiring and
1165 * releasing the locks of the response queues associated with the port.
1166 */
1167static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1168{
1169 int i;
1170
1171 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1172 struct sge_rspq *q = &adap->sge.qs[i].rspq;
1173
1174 spin_lock_irq(&q->lock);
1175 spin_unlock_irq(&q->lock);
1176 }
1177}
1178
1179static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1180{
1181 struct port_info *pi = netdev_priv(dev);
1182 struct adapter *adapter = pi->adapter;
1183
1184 if (adapter->params.rev > 0) {
1185 t3_set_vlan_accel(adapter, 1 << pi->port_id,
f646968f 1186 features & NETIF_F_HW_VLAN_CTAG_RX);
60158e64
RD		 1187 } else {
1188 /* single control for all ports */
f646968f 1189 unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX;
60158e64
RD		 1190
1191 for_each_port(adapter, i)
1192 have_vlans |=
f646968f
PM		 1193 adapter->port[i]->features &
1194 NETIF_F_HW_VLAN_CTAG_RX;
60158e64
RD		 1195
1196 t3_set_vlan_accel(adapter, 1, have_vlans);
1197 }
1198 t3_synchronize_rx(adapter, pi);
1199}
1200
4d22de3e
DLR		 1201/**
1202 * cxgb_up - enable the adapter
1203 * @adapter: adapter being enabled
1204 *
1205 * Called when the first port is enabled, this function performs the
1206 * actions necessary to make an adapter operational, such as completing
1207 * the initialization of HW modules, and enabling interrupts.
1208 *
1209 * Must be called with the rtnl lock held.
1210 */
1211static int cxgb_up(struct adapter *adap)
1212{
60158e64 1213 int i, err;
4d22de3e
DLR		 1214
1215 if (!(adap->flags & FULL_INIT_DONE)) {
8207befa 1216 err = t3_check_fw_version(adap);
a5a3b460 1217 if (err == -EINVAL) {
2e283962 1218 err = upgrade_fw(adap);
8207befa
DLR		 1219 CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1220 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1221 FW_VERSION_MICRO, err ? "failed" : "succeeded");
a5a3b460 1222 }
4d22de3e 1223
8207befa 1224 err = t3_check_tpsram_version(adap);
47330077
DLR		 1225 if (err == -EINVAL) {
1226 err = update_tpsram(adap);
8207befa
DLR		 1227 CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1228 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1229 TP_VERSION_MICRO, err ? "failed" : "succeeded");
47330077
DLR		 1230 }
1231
20d3fc11
DLR		 1232 /*
1233 * Clear interrupts now to catch errors if t3_init_hw fails.
1234 * We clear them again later as initialization may trigger
1235 * conditions that can interrupt.
1236 */
1237 t3_intr_clear(adap);
1238
4d22de3e
DLR		 1239 err = t3_init_hw(adap, 0);
1240 if (err)
1241 goto out;
1242
b881955b 1243 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
6cdbd77e 1244 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
bea3348e 1245
4d22de3e
DLR		 1246 err = setup_sge_qsets(adap);
1247 if (err)
1248 goto out;
1249
60158e64
RD		 1250 for_each_port(adap, i)
1251 cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1252
4d22de3e 1253 setup_rss(adap);
48c4b6db
DLR		 1254 if (!(adap->flags & NAPI_INIT))
1255 init_napi(adap);
31563789
DLR		 1256
1257 t3_start_sge_timers(adap);
4d22de3e
DLR		 1258 adap->flags |= FULL_INIT_DONE;
1259 }
1260
1261 t3_intr_clear(adap);
1262
1263 if (adap->flags & USING_MSIX) {
1264 name_msix_vecs(adap);
1265 err = request_irq(adap->msix_info[0].vec,
1266 t3_async_intr_handler, 0,
1267 adap->msix_info[0].desc, adap);
1268 if (err)
1269 goto irq_err;
1270
42256f57
DLR		 1271 err = request_msix_data_irqs(adap);
1272 if (err) {
4d22de3e
DLR		 1273 free_irq(adap->msix_info[0].vec, adap);
1274 goto irq_err;
1275 }
1276 } else if ((err = request_irq(adap->pdev->irq,
1277 t3_intr_handler(adap,
1278 adap->sge.qs[0].rspq.
1279 polling),
2db6346f
TG		 1280 (adap->flags & USING_MSI) ?
1281 0 : IRQF_SHARED,
4d22de3e
DLR		 1282 adap->name, adap)))
1283 goto irq_err;
1284
bea3348e 1285 enable_all_napi(adap);
4d22de3e
DLR		 1286 t3_sge_start(adap);
1287 t3_intr_enable(adap);
14ab9892 1288
b881955b
DLR		 1289 if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1290 is_offload(adap) && init_tp_parity(adap) == 0)
1291 adap->flags |= TP_PARITY_INIT;
1292
1293 if (adap->flags & TP_PARITY_INIT) {
1294 t3_write_reg(adap, A_TP_INT_CAUSE,
1295 F_CMCACHEPERR | F_ARPLUTPERR);
1296 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1297 }
1298
8c263761 1299 if (!(adap->flags & QUEUES_BOUND)) {
18edc84c
DLR		 1300 int ret = bind_qsets(adap);
1301
1302 if (ret < 0) {
1303 CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
8c263761
DLR		 1304 t3_intr_disable(adap);
1305 free_irq_resources(adap);
18edc84c 1306 err = ret;
8c263761
DLR		 1307 goto out;
1308 }
1309 adap->flags |= QUEUES_BOUND;
1310 }
14ab9892 1311
4d22de3e
DLR		 1312out:
1313 return err;
1314irq_err:
1315 CH_ERR(adap, "request_irq failed, err %d\n", err);
1316 goto out;
1317}
1318
1319/*
1320 * Release resources when all the ports and offloading have been stopped.
1321 */
55bc3228 1322static void cxgb_down(struct adapter *adapter, int on_wq)
4d22de3e
DLR		 1323{
1324 t3_sge_stop(adapter);
1325 spin_lock_irq(&adapter->work_lock); /* sync with PHY intr task */
1326 t3_intr_disable(adapter);
1327 spin_unlock_irq(&adapter->work_lock);
1328
8c263761 1329 free_irq_resources(adapter);
4d22de3e 1330 quiesce_rx(adapter);
a6f018e3 1331 t3_sge_stop(adapter);
55bc3228
CL		 1332 if (!on_wq)
1333 flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
4d22de3e
DLR		 1334}
1335
1336static void schedule_chk_task(struct adapter *adap)
1337{
1338 unsigned int timeo;
1339
1340 timeo = adap->params.linkpoll_period ?
1341 (HZ * adap->params.linkpoll_period) / 10 :
1342 adap->params.stats_update_period * HZ;
1343 if (timeo)
1344 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1345}
1346
1347static int offload_open(struct net_device *dev)
1348{
5fbf816f
DLR		 1349 struct port_info *pi = netdev_priv(dev);
1350 struct adapter *adapter = pi->adapter;
1351 struct t3cdev *tdev = dev2t3cdev(dev);
4d22de3e 1352 int adap_up = adapter->open_device_map & PORT_MASK;
c54f5c24 1353 int err;
4d22de3e
DLR		 1354
1355 if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1356 return 0;
1357
1358 if (!adap_up && (err = cxgb_up(adapter)) < 0)
48c4b6db 1359 goto out;
4d22de3e
DLR		 1360
1361 t3_tp_set_offload_mode(adapter, 1);
1362 tdev->lldev = adapter->port[0];
1363 err = cxgb3_offload_activate(adapter);
1364 if (err)
1365 goto out;
1366
1367 init_port_mtus(adapter);
1368 t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1369 adapter->params.b_wnd,
1370 adapter->params.rev == 0 ?
1371 adapter->port[0]->mtu : 0xffff);
1372 init_smt(adapter);
1373
d96a51f6
DN		 1374 if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1375 dev_dbg(&dev->dev, "cannot create sysfs group\n");
4d22de3e
DLR		 1376
1377 /* Call back all registered clients */
1378 cxgb3_add_clients(tdev);
1379
1380out:
1381 /* restore them in case the offload module has changed them */
1382 if (err) {
1383 t3_tp_set_offload_mode(adapter, 0);
1384 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1385 cxgb3_set_dummy_ops(tdev);
1386 }
1387 return err;
1388}
1389
1390static int offload_close(struct t3cdev *tdev)
1391{
1392 struct adapter *adapter = tdev2adap(tdev);
23f333a2 1393 struct t3c_data *td = T3C_DATA(tdev);
4d22de3e
DLR		 1394
1395 if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1396 return 0;
1397
1398 /* Call back all registered clients */
1399 cxgb3_remove_clients(tdev);
1400
0ee8d33c 1401 sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
4d22de3e 1402
c80b0c28 1403 /* Flush work scheduled while releasing TIDs */
43829731 1404 flush_work(&td->tid_release_task);
c80b0c28 1405
4d22de3e
DLR		 1406 tdev->lldev = NULL;
1407 cxgb3_set_dummy_ops(tdev);
1408 t3_tp_set_offload_mode(adapter, 0);
1409 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1410
1411 if (!adapter->open_device_map)
55bc3228 1412 cxgb_down(adapter, 0);
4d22de3e
DLR		 1413
1414 cxgb3_offload_deactivate(adapter);
1415 return 0;
1416}
1417
1418static int cxgb_open(struct net_device *dev)
1419{
4d22de3e 1420 struct port_info *pi = netdev_priv(dev);
5fbf816f 1421 struct adapter *adapter = pi->adapter;
4d22de3e 1422 int other_ports = adapter->open_device_map & PORT_MASK;
5fbf816f 1423 int err;
4d22de3e 1424
48c4b6db 1425 if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
4d22de3e
DLR		 1426 return err;
1427
1428 set_bit(pi->port_id, &adapter->open_device_map);
8ac3ba68 1429 if (is_offload(adapter) && !ofld_disable) {
4d22de3e
DLR		 1430 err = offload_open(dev);
1431 if (err)
428ac43f 1432 pr_warn("Could not initialize offload capabilities\n");
4d22de3e
DLR		 1433 }
1434
19221e75
BH		 1435 netif_set_real_num_tx_queues(dev, pi->nqsets);
1436 err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1437 if (err)
1438 return err;
4d22de3e
DLR		 1439 link_start(dev);
1440 t3_port_intr_enable(adapter, pi->port_id);
82ad3329 1441 netif_tx_start_all_queues(dev);
4d22de3e
DLR		 1442 if (!other_ports)
1443 schedule_chk_task(adapter);
1444
fa0d4c11 1445 cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
4d22de3e
DLR		 1446 return 0;
1447}
1448
55bc3228 1449static int __cxgb_close(struct net_device *dev, int on_wq)
4d22de3e 1450{
5fbf816f
DLR		 1451 struct port_info *pi = netdev_priv(dev);
1452 struct adapter *adapter = pi->adapter;
4d22de3e 1453
e8d19370
DLR		 1454
1455 if (!adapter->open_device_map)
1456 return 0;
1457
bf792094
DLR		 1458 /* Stop link fault interrupts */
1459 t3_xgm_intr_disable(adapter, pi->port_id);
1460 t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1461
5fbf816f 1462 t3_port_intr_disable(adapter, pi->port_id);
82ad3329 1463 netif_tx_stop_all_queues(dev);
5fbf816f 1464 pi->phy.ops->power_down(&pi->phy, 1);
4d22de3e 1465 netif_carrier_off(dev);
5fbf816f 1466 t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
4d22de3e 1467
20d3fc11 1468 spin_lock_irq(&adapter->work_lock); /* sync with update task */
5fbf816f 1469 clear_bit(pi->port_id, &adapter->open_device_map);
20d3fc11 1470 spin_unlock_irq(&adapter->work_lock);
4d22de3e
DLR		 1471
1472 if (!(adapter->open_device_map & PORT_MASK))
c80b0c28 1473 cancel_delayed_work_sync(&adapter->adap_check_task);
4d22de3e
DLR		 1474
1475 if (!adapter->open_device_map)
55bc3228 1476 cxgb_down(adapter, on_wq);
4d22de3e 1477
fa0d4c11 1478 cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
4d22de3e
DLR		 1479 return 0;
1480}
1481
55bc3228
CL		 1482static int cxgb_close(struct net_device *dev)
1483{
1484 return __cxgb_close(dev, 0);
1485}
1486
4d22de3e
DLR		 1487static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1488{
5fbf816f
DLR		 1489 struct port_info *pi = netdev_priv(dev);
1490 struct adapter *adapter = pi->adapter;
1491 struct net_device_stats *ns = &pi->netstats;
4d22de3e
DLR		 1492 const struct mac_stats *pstats;
1493
1494 spin_lock(&adapter->stats_lock);
5fbf816f 1495 pstats = t3_mac_update_stats(&pi->mac);
4d22de3e
DLR		 1496 spin_unlock(&adapter->stats_lock);
1497
1498 ns->tx_bytes = pstats->tx_octets;
1499 ns->tx_packets = pstats->tx_frames;
1500 ns->rx_bytes = pstats->rx_octets;
1501 ns->rx_packets = pstats->rx_frames;
1502 ns->multicast = pstats->rx_mcast_frames;
1503
1504 ns->tx_errors = pstats->tx_underrun;
1505 ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1506 pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1507 pstats->rx_fifo_ovfl;
1508
1509 /* detailed rx_errors */
1510 ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1511 ns->rx_over_errors = 0;
1512 ns->rx_crc_errors = pstats->rx_fcs_errs;
1513 ns->rx_frame_errors = pstats->rx_symbol_errs;
1514 ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1515 ns->rx_missed_errors = pstats->rx_cong_drops;
1516
1517 /* detailed tx_errors */
1518 ns->tx_aborted_errors = 0;
1519 ns->tx_carrier_errors = 0;
1520 ns->tx_fifo_errors = pstats->tx_underrun;
1521 ns->tx_heartbeat_errors = 0;
1522 ns->tx_window_errors = 0;
1523 return ns;
1524}
1525
1526static u32 get_msglevel(struct net_device *dev)
1527{
5fbf816f
DLR		 1528 struct port_info *pi = netdev_priv(dev);
1529 struct adapter *adapter = pi->adapter;
4d22de3e
DLR		 1530
1531 return adapter->msg_enable;
1532}
1533
1534static void set_msglevel(struct net_device *dev, u32 val)
1535{
5fbf816f
DLR		 1536 struct port_info *pi = netdev_priv(dev);
1537 struct adapter *adapter = pi->adapter;
4d22de3e
DLR		 1538
1539 adapter->msg_enable = val;
1540}
1541
9ca683c6 1542static const char stats_strings[][ETH_GSTRING_LEN] = {
4d22de3e
DLR		 1543 "TxOctetsOK ",
1544 "TxFramesOK ",
1545 "TxMulticastFramesOK",
1546 "TxBroadcastFramesOK",
1547 "TxPauseFrames ",
1548 "TxUnderrun ",
1549 "TxExtUnderrun ",
1550
1551 "TxFrames64 ",
1552 "TxFrames65To127 ",
1553 "TxFrames128To255 ",
1554 "TxFrames256To511 ",
1555 "TxFrames512To1023 ",
1556 "TxFrames1024To1518 ",
1557 "TxFrames1519ToMax ",
1558
1559 "RxOctetsOK ",
1560 "RxFramesOK ",
1561 "RxMulticastFramesOK",
1562 "RxBroadcastFramesOK",
1563 "RxPauseFrames ",
1564 "RxFCSErrors ",
1565 "RxSymbolErrors ",
1566 "RxShortErrors ",
1567 "RxJabberErrors ",
1568 "RxLengthErrors ",
1569 "RxFIFOoverflow ",
1570
1571 "RxFrames64 ",
1572 "RxFrames65To127 ",
1573 "RxFrames128To255 ",
1574 "RxFrames256To511 ",
1575 "RxFrames512To1023 ",
1576 "RxFrames1024To1518 ",
1577 "RxFrames1519ToMax ",
1578
1579 "PhyFIFOErrors ",
1580 "TSO ",
1581 "VLANextractions ",
1582 "VLANinsertions ",
1583 "TxCsumOffload ",
1584 "RxCsumGood ",
b47385bd
DLR		 1585 "LroAggregated ",
1586 "LroFlushed ",
1587 "LroNoDesc ",
fc90664e
DLR		 1588 "RxDrops ",
1589
1590 "CheckTXEnToggled ",
1591 "CheckResets ",
1592
bf792094 1593 "LinkFaults ",
4d22de3e
DLR		 1594};
1595
b9f2c044 1596static int get_sset_count(struct net_device *dev, int sset)
4d22de3e 1597{
b9f2c044
JG		 1598 switch (sset) {
1599 case ETH_SS_STATS:
1600 return ARRAY_SIZE(stats_strings);
1601 default:
1602 return -EOPNOTSUPP;
1603 }
4d22de3e
DLR		 1604}
1605
1606#define T3_REGMAP_SIZE (3 * 1024)
1607
1608static int get_regs_len(struct net_device *dev)
1609{
1610 return T3_REGMAP_SIZE;
1611}
1612
1613static int get_eeprom_len(struct net_device *dev)
1614{
1615 return EEPROMSIZE;
1616}
1617
1618static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1619{
5fbf816f
DLR		 1620 struct port_info *pi = netdev_priv(dev);
1621 struct adapter *adapter = pi->adapter;
4d22de3e 1622 u32 fw_vers = 0;
47330077 1623 u32 tp_vers = 0;
4d22de3e 1624
cf3760da 1625 spin_lock(&adapter->stats_lock);
4d22de3e 1626 t3_get_fw_version(adapter, &fw_vers);
47330077 1627 t3_get_tp_version(adapter, &tp_vers);
cf3760da 1628 spin_unlock(&adapter->stats_lock);
4d22de3e 1629
23020ab3
RJ		 1630 strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1631 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1632 strlcpy(info->bus_info, pci_name(adapter->pdev),
1633 sizeof(info->bus_info));
84b40501 1634 if (fw_vers)
4d22de3e 1635 snprintf(info->fw_version, sizeof(info->fw_version),
47330077 1636 "%s %u.%u.%u TP %u.%u.%u",
4aac3899
DLR		 1637 G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1638 G_FW_VERSION_MAJOR(fw_vers),
1639 G_FW_VERSION_MINOR(fw_vers),
47330077
DLR		 1640 G_FW_VERSION_MICRO(fw_vers),
1641 G_TP_VERSION_MAJOR(tp_vers),
1642 G_TP_VERSION_MINOR(tp_vers),
1643 G_TP_VERSION_MICRO(tp_vers));
4d22de3e
DLR		 1644}
1645
1646static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1647{
1648 if (stringset == ETH_SS_STATS)
1649 memcpy(data, stats_strings, sizeof(stats_strings));
1650}
1651
1652static unsigned long collect_sge_port_stats(struct adapter *adapter,
1653 struct port_info *p, int idx)
1654{
1655 int i;
1656 unsigned long tot = 0;
1657
8c263761
DLR		 1658 for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1659 tot += adapter->sge.qs[i].port_stats[idx];
4d22de3e
DLR		 1660 return tot;
1661}
1662
1663static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1664 u64 *data)
1665{
4d22de3e 1666 struct port_info *pi = netdev_priv(dev);
5fbf816f 1667 struct adapter *adapter = pi->adapter;
4d22de3e
DLR		 1668 const struct mac_stats *s;
1669
1670 spin_lock(&adapter->stats_lock);
1671 s = t3_mac_update_stats(&pi->mac);
1672 spin_unlock(&adapter->stats_lock);
1673
1674 *data++ = s->tx_octets;
1675 *data++ = s->tx_frames;
1676 *data++ = s->tx_mcast_frames;
1677 *data++ = s->tx_bcast_frames;
1678 *data++ = s->tx_pause;
1679 *data++ = s->tx_underrun;
1680 *data++ = s->tx_fifo_urun;
1681
1682 *data++ = s->tx_frames_64;
1683 *data++ = s->tx_frames_65_127;
1684 *data++ = s->tx_frames_128_255;
1685 *data++ = s->tx_frames_256_511;
1686 *data++ = s->tx_frames_512_1023;
1687 *data++ = s->tx_frames_1024_1518;
1688 *data++ = s->tx_frames_1519_max;
1689
1690 *data++ = s->rx_octets;
1691 *data++ = s->rx_frames;
1692 *data++ = s->rx_mcast_frames;
1693 *data++ = s->rx_bcast_frames;
1694 *data++ = s->rx_pause;
1695 *data++ = s->rx_fcs_errs;
1696 *data++ = s->rx_symbol_errs;
1697 *data++ = s->rx_short;
1698 *data++ = s->rx_jabber;
1699 *data++ = s->rx_too_long;
1700 *data++ = s->rx_fifo_ovfl;
1701
1702 *data++ = s->rx_frames_64;
1703 *data++ = s->rx_frames_65_127;
1704 *data++ = s->rx_frames_128_255;
1705 *data++ = s->rx_frames_256_511;
1706 *data++ = s->rx_frames_512_1023;
1707 *data++ = s->rx_frames_1024_1518;
1708 *data++ = s->rx_frames_1519_max;
1709
1710 *data++ = pi->phy.fifo_errors;
1711
1712 *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1713 *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1714 *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1715 *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1716 *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
7be2df45
HX		 1717 *data++ = 0;
1718 *data++ = 0;
1719 *data++ = 0;
4d22de3e 1720 *data++ = s->rx_cong_drops;
fc90664e
DLR		 1721
1722 *data++ = s->num_toggled;
1723 *data++ = s->num_resets;
bf792094
DLR		 1724
1725 *data++ = s->link_faults;
4d22de3e
DLR		 1726}
1727
1728static inline void reg_block_dump(struct adapter *ap, void *buf,
1729 unsigned int start, unsigned int end)
1730{
1731 u32 *p = buf + start;
1732
1733 for (; start <= end; start += sizeof(u32))
1734 *p++ = t3_read_reg(ap, start);
1735}
1736
1737static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1738 void *buf)
1739{
5fbf816f
DLR		 1740 struct port_info *pi = netdev_priv(dev);
1741 struct adapter *ap = pi->adapter;
4d22de3e
DLR		 1742
1743 /*
1744 * Version scheme:
1745 * bits 0..9: chip version
1746 * bits 10..15: chip revision
1747 * bit 31: set for PCIe cards
1748 */
1749 regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1750
1751 /*
1752 * We skip the MAC statistics registers because they are clear-on-read.
1753 * Also reading multi-register stats would need to synchronize with the
1754 * periodic mac stats accumulation. Hard to justify the complexity.
1755 */
1756 memset(buf, 0, T3_REGMAP_SIZE);
1757 reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1758 reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1759 reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1760 reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1761 reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1762 reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1763 XGM_REG(A_XGM_SERDES_STAT3, 1));
1764 reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1765 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1766}
1767
1768static int restart_autoneg(struct net_device *dev)
1769{
1770 struct port_info *p = netdev_priv(dev);
1771
1772 if (!netif_running(dev))
1773 return -EAGAIN;
1774 if (p->link_config.autoneg != AUTONEG_ENABLE)
1775 return -EINVAL;
1776 p->phy.ops->autoneg_restart(&p->phy);
1777 return 0;
1778}
1779
12fcf941 1780static int set_phys_id(struct net_device *dev,
1781 enum ethtool_phys_id_state state)
4d22de3e 1782{
5fbf816f
DLR		 1783 struct port_info *pi = netdev_priv(dev);
1784 struct adapter *adapter = pi->adapter;
4d22de3e 1785
12fcf941 1786 switch (state) {
1787 case ETHTOOL_ID_ACTIVE:
fce55922 1788 return 1; /* cycle on/off once per second */
12fcf941 1789
1790 case ETHTOOL_ID_OFF:
1791 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1792 break;
4d22de3e 1793
12fcf941 1794 case ETHTOOL_ID_ON:
1795 case ETHTOOL_ID_INACTIVE:
4d22de3e 1796 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
4d22de3e 1797 F_GPIO0_OUT_VAL);
12fcf941 1798 }
1799
4d22de3e
DLR		 1800 return 0;
1801}
1802
1803static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1804{
1805 struct port_info *p = netdev_priv(dev);
1806
1807 cmd->supported = p->link_config.supported;
1808 cmd->advertising = p->link_config.advertising;
1809
1810 if (netif_carrier_ok(dev)) {
70739497 1811 ethtool_cmd_speed_set(cmd, p->link_config.speed);
4d22de3e
DLR		 1812 cmd->duplex = p->link_config.duplex;
1813 } else {
537fae01
JP		 1814 ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
1815 cmd->duplex = DUPLEX_UNKNOWN;
4d22de3e
DLR		 1816 }
1817
1818 cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
0f07c4ee 1819 cmd->phy_address = p->phy.mdio.prtad;
4d22de3e
DLR		 1820 cmd->transceiver = XCVR_EXTERNAL;
1821 cmd->autoneg = p->link_config.autoneg;
1822 cmd->maxtxpkt = 0;
1823 cmd->maxrxpkt = 0;
1824 return 0;
1825}
1826
1827static int speed_duplex_to_caps(int speed, int duplex)
1828{
1829 int cap = 0;
1830
1831 switch (speed) {
1832 case SPEED_10:
1833 if (duplex == DUPLEX_FULL)
1834 cap = SUPPORTED_10baseT_Full;
1835 else
1836 cap = SUPPORTED_10baseT_Half;
1837 break;
1838 case SPEED_100:
1839 if (duplex == DUPLEX_FULL)
1840 cap = SUPPORTED_100baseT_Full;
1841 else
1842 cap = SUPPORTED_100baseT_Half;
1843 break;
1844 case SPEED_1000:
1845 if (duplex == DUPLEX_FULL)
1846 cap = SUPPORTED_1000baseT_Full;
1847 else
1848 cap = SUPPORTED_1000baseT_Half;
1849 break;
1850 case SPEED_10000:
1851 if (duplex == DUPLEX_FULL)
1852 cap = SUPPORTED_10000baseT_Full;
1853 }
1854 return cap;
1855}
1856
1857#define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1858 ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1859 ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1860 ADVERTISED_10000baseT_Full)
1861
1862static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1863{
1864 struct port_info *p = netdev_priv(dev);
1865 struct link_config *lc = &p->link_config;
1866
9b1e3656
DLR		 1867 if (!(lc->supported & SUPPORTED_Autoneg)) {
1868 /*
1869 * PHY offers a single speed/duplex. See if that's what's
1870 * being requested.
1871 */
1872 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338
DD		 1873 u32 speed = ethtool_cmd_speed(cmd);
1874 int cap = speed_duplex_to_caps(speed, cmd->duplex);
9b1e3656
DLR		 1875 if (lc->supported & cap)
1876 return 0;
1877 }
1878 return -EINVAL;
1879 }
4d22de3e
DLR		 1880
1881 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338
DD		 1882 u32 speed = ethtool_cmd_speed(cmd);
1883 int cap = speed_duplex_to_caps(speed, cmd->duplex);
4d22de3e 1884
25db0338 1885 if (!(lc->supported & cap) || (speed == SPEED_1000))
4d22de3e 1886 return -EINVAL;
25db0338 1887 lc->requested_speed = speed;
4d22de3e
DLR		 1888 lc->requested_duplex = cmd->duplex;
1889 lc->advertising = 0;
1890 } else {
1891 cmd->advertising &= ADVERTISED_MASK;
1892 cmd->advertising &= lc->supported;
1893 if (!cmd->advertising)
1894 return -EINVAL;
1895 lc->requested_speed = SPEED_INVALID;
1896 lc->requested_duplex = DUPLEX_INVALID;
1897 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1898 }
1899 lc->autoneg = cmd->autoneg;
1900 if (netif_running(dev))
1901 t3_link_start(&p->phy, &p->mac, lc);
1902 return 0;
1903}
1904
1905static void get_pauseparam(struct net_device *dev,
1906 struct ethtool_pauseparam *epause)
1907{
1908 struct port_info *p = netdev_priv(dev);
1909
1910 epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1911 epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1912 epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1913}
1914
1915static int set_pauseparam(struct net_device *dev,
1916 struct ethtool_pauseparam *epause)
1917{
1918 struct port_info *p = netdev_priv(dev);
1919 struct link_config *lc = &p->link_config;
1920
1921 if (epause->autoneg == AUTONEG_DISABLE)
1922 lc->requested_fc = 0;
1923 else if (lc->supported & SUPPORTED_Autoneg)
1924 lc->requested_fc = PAUSE_AUTONEG;
1925 else
1926 return -EINVAL;
1927
1928 if (epause->rx_pause)
1929 lc->requested_fc |= PAUSE_RX;
1930 if (epause->tx_pause)
1931 lc->requested_fc |= PAUSE_TX;
1932 if (lc->autoneg == AUTONEG_ENABLE) {
1933 if (netif_running(dev))
1934 t3_link_start(&p->phy, &p->mac, lc);
1935 } else {
1936 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1937 if (netif_running(dev))
1938 t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1939 }
1940 return 0;
1941}
1942
4d22de3e
DLR		 1943static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1944{
5fbf816f
DLR		 1945 struct port_info *pi = netdev_priv(dev);
1946 struct adapter *adapter = pi->adapter;
05b97b30 1947 const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
4d22de3e
DLR		 1948
1949 e->rx_max_pending = MAX_RX_BUFFERS;
4d22de3e
DLR		 1950 e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1951 e->tx_max_pending = MAX_TXQ_ENTRIES;
1952
05b97b30
DLR		 1953 e->rx_pending = q->fl_size;
1954 e->rx_mini_pending = q->rspq_size;
1955 e->rx_jumbo_pending = q->jumbo_size;
1956 e->tx_pending = q->txq_size[0];
4d22de3e
DLR		 1957}
1958
1959static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1960{
5fbf816f
DLR		 1961 struct port_info *pi = netdev_priv(dev);
1962 struct adapter *adapter = pi->adapter;
05b97b30 1963 struct qset_params *q;
5fbf816f 1964 int i;
4d22de3e
DLR		 1965
1966 if (e->rx_pending > MAX_RX_BUFFERS ||
1967 e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1968 e->tx_pending > MAX_TXQ_ENTRIES ||
1969 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1970 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1971 e->rx_pending < MIN_FL_ENTRIES ||
1972 e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1973 e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1974 return -EINVAL;
1975
1976 if (adapter->flags & FULL_INIT_DONE)
1977 return -EBUSY;
1978
05b97b30
DLR		 1979 q = &adapter->params.sge.qset[pi->first_qset];
1980 for (i = 0; i < pi->nqsets; ++i, ++q) {
4d22de3e
DLR		 1981 q->rspq_size = e->rx_mini_pending;
1982 q->fl_size = e->rx_pending;
1983 q->jumbo_size = e->rx_jumbo_pending;
1984 q->txq_size[0] = e->tx_pending;
1985 q->txq_size[1] = e->tx_pending;
1986 q->txq_size[2] = e->tx_pending;
1987 }
1988 return 0;
1989}
1990
1991static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1992{
5fbf816f
DLR		 1993 struct port_info *pi = netdev_priv(dev);
1994 struct adapter *adapter = pi->adapter;
c211c969
AB		 1995 struct qset_params *qsp;
1996 struct sge_qset *qs;
1997 int i;
4d22de3e
DLR		 1998
1999 if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
2000 return -EINVAL;
2001
c211c969
AB		 2002 for (i = 0; i < pi->nqsets; i++) {
2003 qsp = &adapter->params.sge.qset[i];
2004 qs = &adapter->sge.qs[i];
2005 qsp->coalesce_usecs = c->rx_coalesce_usecs;
2006 t3_update_qset_coalesce(qs, qsp);
2007 }
2008
4d22de3e
DLR		 2009 return 0;
2010}
2011
2012static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2013{
5fbf816f
DLR		 2014 struct port_info *pi = netdev_priv(dev);
2015 struct adapter *adapter = pi->adapter;
4d22de3e
DLR		 2016 struct qset_params *q = adapter->params.sge.qset;
2017
2018 c->rx_coalesce_usecs = q->coalesce_usecs;
2019 return 0;
2020}
2021
2022static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2023 u8 * data)
2024{
5fbf816f
DLR		 2025 struct port_info *pi = netdev_priv(dev);
2026 struct adapter *adapter = pi->adapter;
4d22de3e 2027 int i, err = 0;
4d22de3e
DLR		 2028
2029 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2030 if (!buf)
2031 return -ENOMEM;
2032
2033 e->magic = EEPROM_MAGIC;
2034 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
05e5c116 2035 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
4d22de3e
DLR		 2036
2037 if (!err)
2038 memcpy(data, buf + e->offset, e->len);
2039 kfree(buf);
2040 return err;
2041}
2042
2043static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2044 u8 * data)
2045{
5fbf816f
DLR		 2046 struct port_info *pi = netdev_priv(dev);
2047 struct adapter *adapter = pi->adapter;
05e5c116
AV		 2048 u32 aligned_offset, aligned_len;
2049 __le32 *p;
4d22de3e 2050 u8 *buf;
c54f5c24 2051 int err;
4d22de3e
DLR		 2052
2053 if (eeprom->magic != EEPROM_MAGIC)
2054 return -EINVAL;
2055
2056 aligned_offset = eeprom->offset & ~3;
2057 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2058
2059 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2060 buf = kmalloc(aligned_len, GFP_KERNEL);
2061 if (!buf)
2062 return -ENOMEM;
05e5c116 2063 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
4d22de3e
DLR		 2064 if (!err && aligned_len > 4)
2065 err = t3_seeprom_read(adapter,
2066 aligned_offset + aligned_len - 4,
05e5c116 2067 (__le32 *) & buf[aligned_len - 4]);
4d22de3e
DLR		 2068 if (err)
2069 goto out;
2070 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2071 } else
2072 buf = data;
2073
2074 err = t3_seeprom_wp(adapter, 0);
2075 if (err)
2076 goto out;
2077
05e5c116 2078 for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
4d22de3e
DLR		 2079 err = t3_seeprom_write(adapter, aligned_offset, *p);
2080 aligned_offset += 4;
2081 }
2082
2083 if (!err)
2084 err = t3_seeprom_wp(adapter, 1);
2085out:
2086 if (buf != data)
2087 kfree(buf);
2088 return err;
2089}
2090
2091static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2092{
2093 wol->supported = 0;
2094 wol->wolopts = 0;
2095 memset(&wol->sopass, 0, sizeof(wol->sopass));
2096}
2097
2098static const struct ethtool_ops cxgb_ethtool_ops = {
2099 .get_settings = get_settings,
2100 .set_settings = set_settings,
2101 .get_drvinfo = get_drvinfo,
2102 .get_msglevel = get_msglevel,
2103 .set_msglevel = set_msglevel,
2104 .get_ringparam = get_sge_param,
2105 .set_ringparam = set_sge_param,
2106 .get_coalesce = get_coalesce,
2107 .set_coalesce = set_coalesce,
2108 .get_eeprom_len = get_eeprom_len,
2109 .get_eeprom = get_eeprom,
2110 .set_eeprom = set_eeprom,
2111 .get_pauseparam = get_pauseparam,
2112 .set_pauseparam = set_pauseparam,
4d22de3e
DLR		 2113 .get_link = ethtool_op_get_link,
2114 .get_strings = get_strings,
12fcf941 2115 .set_phys_id = set_phys_id,
4d22de3e 2116 .nway_reset = restart_autoneg,
b9f2c044 2117 .get_sset_count = get_sset_count,
4d22de3e
DLR		 2118 .get_ethtool_stats = get_stats,
2119 .get_regs_len = get_regs_len,
2120 .get_regs = get_regs,
2121 .get_wol = get_wol,
4d22de3e
DLR		 2122};
2123
2124static int in_range(int val, int lo, int hi)
2125{
2126 return val < 0 || (val <= hi && val >= lo);
2127}
2128
2129static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2130{
5fbf816f
DLR		 2131 struct port_info *pi = netdev_priv(dev);
2132 struct adapter *adapter = pi->adapter;
4d22de3e 2133 u32 cmd;
5fbf816f 2134 int ret;
4d22de3e
DLR		 2135
2136 if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2137 return -EFAULT;
2138
2139 switch (cmd) {
4d22de3e
DLR		 2140 case CHELSIO_SET_QSET_PARAMS:{
2141 int i;
2142 struct qset_params *q;
2143 struct ch_qset_params t;
8c263761
DLR		 2144 int q1 = pi->first_qset;
2145 int nqsets = pi->nqsets;
4d22de3e
DLR		 2146
2147 if (!capable(CAP_NET_ADMIN))
2148 return -EPERM;
2149 if (copy_from_user(&t, useraddr, sizeof(t)))
2150 return -EFAULT;
2151 if (t.qset_idx >= SGE_QSETS)
2152 return -EINVAL;
2153 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
8e95a202
JP		 2154 !in_range(t.cong_thres, 0, 255) ||
2155 !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2156 MAX_TXQ_ENTRIES) ||
2157 !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2158 MAX_TXQ_ENTRIES) ||
2159 !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2160 MAX_CTRL_TXQ_ENTRIES) ||
2161 !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2162 MAX_RX_BUFFERS) ||
2163 !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2164 MAX_RX_JUMBO_BUFFERS) ||
2165 !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2166 MAX_RSPQ_ENTRIES))
4d22de3e 2167 return -EINVAL;
8c263761 2168
4d22de3e
DLR		 2169 if ((adapter->flags & FULL_INIT_DONE) &&
2170 (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2171 t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2172 t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2173 t.polling >= 0 || t.cong_thres >= 0))
2174 return -EBUSY;
2175
8c263761
DLR		 2176 /* Allow setting of any available qset when offload enabled */
2177 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2178 q1 = 0;
2179 for_each_port(adapter, i) {
2180 pi = adap2pinfo(adapter, i);
2181 nqsets += pi->first_qset + pi->nqsets;
2182 }
2183 }
2184
2185 if (t.qset_idx < q1)
2186 return -EINVAL;
2187 if (t.qset_idx > q1 + nqsets - 1)
2188 return -EINVAL;
2189
4d22de3e
DLR		 2190 q = &adapter->params.sge.qset[t.qset_idx];
2191
2192 if (t.rspq_size >= 0)
2193 q->rspq_size = t.rspq_size;
2194 if (t.fl_size[0] >= 0)
2195 q->fl_size = t.fl_size[0];
2196 if (t.fl_size[1] >= 0)
2197 q->jumbo_size = t.fl_size[1];
2198 if (t.txq_size[0] >= 0)
2199 q->txq_size[0] = t.txq_size[0];
2200 if (t.txq_size[1] >= 0)
2201 q->txq_size[1] = t.txq_size[1];
2202 if (t.txq_size[2] >= 0)
2203 q->txq_size[2] = t.txq_size[2];
2204 if (t.cong_thres >= 0)
2205 q->cong_thres = t.cong_thres;
2206 if (t.intr_lat >= 0) {
2207 struct sge_qset *qs =
2208 &adapter->sge.qs[t.qset_idx];
2209
2210 q->coalesce_usecs = t.intr_lat;
2211 t3_update_qset_coalesce(qs, q);
2212 }
2213 if (t.polling >= 0) {
2214 if (adapter->flags & USING_MSIX)
2215 q->polling = t.polling;
2216 else {
2217 /* No polling with INTx for T3A */
2218 if (adapter->params.rev == 0 &&
2219 !(adapter->flags & USING_MSI))
2220 t.polling = 0;
2221
2222 for (i = 0; i < SGE_QSETS; i++) {
2223 q = &adapter->params.sge.
2224 qset[i];
2225 q->polling = t.polling;
2226 }
2227 }
2228 }
d2fe2755
MM		 2229
2230 if (t.lro >= 0) {
2231 if (t.lro)
2232 dev->wanted_features |= NETIF_F_GRO;
2233 else
2234 dev->wanted_features &= ~NETIF_F_GRO;
2235 netdev_update_features(dev);
2236 }
04ecb072 2237
4d22de3e
DLR		 2238 break;
2239 }
2240 case CHELSIO_GET_QSET_PARAMS:{
2241 struct qset_params *q;
2242 struct ch_qset_params t;
8c263761
DLR		 2243 int q1 = pi->first_qset;
2244 int nqsets = pi->nqsets;
2245 int i;
4d22de3e
DLR		 2246
2247 if (copy_from_user(&t, useraddr, sizeof(t)))
2248 return -EFAULT;
8c263761
DLR		 2249
2250 /* Display qsets for all ports when offload enabled */
2251 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2252 q1 = 0;
2253 for_each_port(adapter, i) {
2254 pi = adap2pinfo(adapter, i);
2255 nqsets = pi->first_qset + pi->nqsets;
2256 }
2257 }
2258
2259 if (t.qset_idx >= nqsets)
4d22de3e
DLR		 2260 return -EINVAL;
2261
8c263761 2262 q = &adapter->params.sge.qset[q1 + t.qset_idx];
4d22de3e
DLR		 2263 t.rspq_size = q->rspq_size;
2264 t.txq_size[0] = q->txq_size[0];
2265 t.txq_size[1] = q->txq_size[1];
2266 t.txq_size[2] = q->txq_size[2];
2267 t.fl_size[0] = q->fl_size;
2268 t.fl_size[1] = q->jumbo_size;
2269 t.polling = q->polling;
d2fe2755 2270 t.lro = !!(dev->features & NETIF_F_GRO);
4d22de3e
DLR		 2271 t.intr_lat = q->coalesce_usecs;
2272 t.cong_thres = q->cong_thres;
8c263761
DLR		 2273 t.qnum = q1;
2274
2275 if (adapter->flags & USING_MSIX)
2276 t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2277 else
2278 t.vector = adapter->pdev->irq;
4d22de3e
DLR		 2279
2280 if (copy_to_user(useraddr, &t, sizeof(t)))
2281 return -EFAULT;
2282 break;
2283 }
2284 case CHELSIO_SET_QSET_NUM:{
2285 struct ch_reg edata;
4d22de3e
DLR		 2286 unsigned int i, first_qset = 0, other_qsets = 0;
2287
2288 if (!capable(CAP_NET_ADMIN))
2289 return -EPERM;
2290 if (adapter->flags & FULL_INIT_DONE)
2291 return -EBUSY;
2292 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2293 return -EFAULT;
2294 if (edata.val < 1 ||
2295 (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2296 return -EINVAL;
2297
2298 for_each_port(adapter, i)
2299 if (adapter->port[i] && adapter->port[i] != dev)
2300 other_qsets += adap2pinfo(adapter, i)->nqsets;
2301
2302 if (edata.val + other_qsets > SGE_QSETS)
2303 return -EINVAL;
2304
2305 pi->nqsets = edata.val;
2306
2307 for_each_port(adapter, i)
2308 if (adapter->port[i]) {
2309 pi = adap2pinfo(adapter, i);
2310 pi->first_qset = first_qset;
2311 first_qset += pi->nqsets;
2312 }
2313 break;
2314 }
2315 case CHELSIO_GET_QSET_NUM:{
2316 struct ch_reg edata;
4d22de3e 2317
49c37c03
DR		 2318 memset(&edata, 0, sizeof(struct ch_reg));
2319
4d22de3e
DLR		 2320 edata.cmd = CHELSIO_GET_QSET_NUM;
2321 edata.val = pi->nqsets;
2322 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2323 return -EFAULT;
2324 break;
2325 }
2326 case CHELSIO_LOAD_FW:{
2327 u8 *fw_data;
2328 struct ch_mem_range t;
2329
1b3aa7af 2330 if (!capable(CAP_SYS_RAWIO))
4d22de3e
DLR		 2331 return -EPERM;
2332 if (copy_from_user(&t, useraddr, sizeof(t)))
2333 return -EFAULT;
1b3aa7af 2334 /* Check t.len sanity ? */
c5dc9a35
JL		 2335 fw_data = memdup_user(useraddr + sizeof(t), t.len);
2336 if (IS_ERR(fw_data))
2337 return PTR_ERR(fw_data);
4d22de3e
DLR		 2338
2339 ret = t3_load_fw(adapter, fw_data, t.len);
2340 kfree(fw_data);
2341 if (ret)
2342 return ret;
2343 break;
2344 }
2345 case CHELSIO_SETMTUTAB:{
2346 struct ch_mtus m;
2347 int i;
2348
2349 if (!is_offload(adapter))
2350 return -EOPNOTSUPP;
2351 if (!capable(CAP_NET_ADMIN))
2352 return -EPERM;
2353 if (offload_running(adapter))
2354 return -EBUSY;
2355 if (copy_from_user(&m, useraddr, sizeof(m)))
2356 return -EFAULT;
2357 if (m.nmtus != NMTUS)
2358 return -EINVAL;
2359 if (m.mtus[0] < 81) /* accommodate SACK */
2360 return -EINVAL;
2361
2362 /* MTUs must be in ascending order */
2363 for (i = 1; i < NMTUS; ++i)
2364 if (m.mtus[i] < m.mtus[i - 1])
2365 return -EINVAL;
2366
2367 memcpy(adapter->params.mtus, m.mtus,
2368 sizeof(adapter->params.mtus));
2369 break;
2370 }
2371 case CHELSIO_GET_PM:{
2372 struct tp_params *p = &adapter->params.tp;
2373 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2374
2375 if (!is_offload(adapter))
2376 return -EOPNOTSUPP;
2377 m.tx_pg_sz = p->tx_pg_size;
2378 m.tx_num_pg = p->tx_num_pgs;
2379 m.rx_pg_sz = p->rx_pg_size;
2380 m.rx_num_pg = p->rx_num_pgs;
2381 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2382 if (copy_to_user(useraddr, &m, sizeof(m)))
2383 return -EFAULT;
2384 break;
2385 }
2386 case CHELSIO_SET_PM:{
2387 struct ch_pm m;
2388 struct tp_params *p = &adapter->params.tp;
2389
2390 if (!is_offload(adapter))
2391 return -EOPNOTSUPP;
2392 if (!capable(CAP_NET_ADMIN))
2393 return -EPERM;
2394 if (adapter->flags & FULL_INIT_DONE)
2395 return -EBUSY;
2396 if (copy_from_user(&m, useraddr, sizeof(m)))
2397 return -EFAULT;
d9da466a 2398 if (!is_power_of_2(m.rx_pg_sz) ||
2399 !is_power_of_2(m.tx_pg_sz))
4d22de3e
DLR		 2400 return -EINVAL; /* not power of 2 */
2401 if (!(m.rx_pg_sz & 0x14000))
2402 return -EINVAL; /* not 16KB or 64KB */
2403 if (!(m.tx_pg_sz & 0x1554000))
2404 return -EINVAL;
2405 if (m.tx_num_pg == -1)
2406 m.tx_num_pg = p->tx_num_pgs;
2407 if (m.rx_num_pg == -1)
2408 m.rx_num_pg = p->rx_num_pgs;
2409 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2410 return -EINVAL;
2411 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2412 m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2413 return -EINVAL;
2414 p->rx_pg_size = m.rx_pg_sz;
2415 p->tx_pg_size = m.tx_pg_sz;
2416 p->rx_num_pgs = m.rx_num_pg;
2417 p->tx_num_pgs = m.tx_num_pg;
2418 break;
2419 }
2420 case CHELSIO_GET_MEM:{
2421 struct ch_mem_range t;
2422 struct mc7 *mem;
2423 u64 buf[32];
2424
2425 if (!is_offload(adapter))
2426 return -EOPNOTSUPP;
2427 if (!(adapter->flags & FULL_INIT_DONE))
2428 return -EIO; /* need the memory controllers */
2429 if (copy_from_user(&t, useraddr, sizeof(t)))
2430 return -EFAULT;
2431 if ((t.addr & 7) || (t.len & 7))
2432 return -EINVAL;
2433 if (t.mem_id == MEM_CM)
2434 mem = &adapter->cm;
2435 else if (t.mem_id == MEM_PMRX)
2436 mem = &adapter->pmrx;
2437 else if (t.mem_id == MEM_PMTX)
2438 mem = &adapter->pmtx;
2439 else
2440 return -EINVAL;
2441
2442 /*
1825494a
DLR		 2443 * Version scheme:
2444 * bits 0..9: chip version
2445 * bits 10..15: chip revision
2446 */
4d22de3e
DLR		 2447 t.version = 3 | (adapter->params.rev << 10);
2448 if (copy_to_user(useraddr, &t, sizeof(t)))
2449 return -EFAULT;
2450
2451 /*
2452 * Read 256 bytes at a time as len can be large and we don't
2453 * want to use huge intermediate buffers.
2454 */
2455 useraddr += sizeof(t); /* advance to start of buffer */
2456 while (t.len) {
2457 unsigned int chunk =
2458 min_t(unsigned int, t.len, sizeof(buf));
2459
2460 ret =
2461 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2462 buf);
2463 if (ret)
2464 return ret;
2465 if (copy_to_user(useraddr, buf, chunk))
2466 return -EFAULT;
2467 useraddr += chunk;
2468 t.addr += chunk;
2469 t.len -= chunk;
2470 }
2471 break;
2472 }
2473 case CHELSIO_SET_TRACE_FILTER:{
2474 struct ch_trace t;
2475 const struct trace_params *tp;
2476
2477 if (!capable(CAP_NET_ADMIN))
2478 return -EPERM;
2479 if (!offload_running(adapter))
2480 return -EAGAIN;
2481 if (copy_from_user(&t, useraddr, sizeof(t)))
2482 return -EFAULT;
2483
2484 tp = (const struct trace_params *)&t.sip;
2485 if (t.config_tx)
2486 t3_config_trace_filter(adapter, tp, 0,
2487 t.invert_match,
2488 t.trace_tx);
2489 if (t.config_rx)
2490 t3_config_trace_filter(adapter, tp, 1,
2491 t.invert_match,
2492 t.trace_rx);
2493 break;
2494 }
4d22de3e
DLR		 2495 default:
2496 return -EOPNOTSUPP;
2497 }
2498 return 0;
2499}
2500
2501static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2502{
4d22de3e 2503 struct mii_ioctl_data *data = if_mii(req);
5fbf816f
DLR		 2504 struct port_info *pi = netdev_priv(dev);
2505 struct adapter *adapter = pi->adapter;
4d22de3e
DLR		 2506
2507 switch (cmd) {
0f07c4ee
BH		 2508 case SIOCGMIIREG:
2509 case SIOCSMIIREG:
2510 /* Convert phy_id from older PRTAD/DEVAD format */
2511 if (is_10G(adapter) &&
2512 !mdio_phy_id_is_c45(data->phy_id) &&
2513 (data->phy_id & 0x1f00) &&
2514 !(data->phy_id & 0xe0e0))
2515 data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2516 data->phy_id & 0x1f);
4d22de3e 2517 /* FALLTHRU */
0f07c4ee
BH		 2518 case SIOCGMIIPHY:
2519 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
4d22de3e
DLR		 2520 case SIOCCHIOCTL:
2521 return cxgb_extension_ioctl(dev, req->ifr_data);
2522 default:
2523 return -EOPNOTSUPP;
2524 }
4d22de3e
DLR		 2525}
2526
2527static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2528{
4d22de3e 2529 struct port_info *pi = netdev_priv(dev);
5fbf816f
DLR		 2530 struct adapter *adapter = pi->adapter;
2531 int ret;
4d22de3e
DLR		 2532
2533 if (new_mtu < 81) /* accommodate SACK */
2534 return -EINVAL;
2535 if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2536 return ret;
2537 dev->mtu = new_mtu;
2538 init_port_mtus(adapter);
2539 if (adapter->params.rev == 0 && offload_running(adapter))
2540 t3_load_mtus(adapter, adapter->params.mtus,
2541 adapter->params.a_wnd, adapter->params.b_wnd,
2542 adapter->port[0]->mtu);
2543 return 0;
2544}
2545
2546static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2547{
4d22de3e 2548 struct port_info *pi = netdev_priv(dev);
5fbf816f 2549 struct adapter *adapter = pi->adapter;
4d22de3e
DLR		 2550 struct sockaddr *addr = p;
2551
2552 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 2553 return -EADDRNOTAVAIL;
4d22de3e
DLR		 2554
2555 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
f14d42f3 2556 t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
4d22de3e
DLR		 2557 if (offload_running(adapter))
2558 write_smt_entry(adapter, pi->port_id);
2559 return 0;
2560}
2561
c8f44aff
MM		 2562static netdev_features_t cxgb_fix_features(struct net_device *dev,
2563 netdev_features_t features)
892ef5d8
JP		 2564{
2565 /*
2566 * Since there is no support for separate rx/tx vlan accel
2567 * enable/disable make sure tx flag is always in same state as rx.
2568 */
f646968f
PM		 2569 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2570 features |= NETIF_F_HW_VLAN_CTAG_TX;
892ef5d8 2571 else
f646968f 2572 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
892ef5d8
JP		 2573
2574 return features;
2575}
2576
c8f44aff 2577static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
892ef5d8 2578{
c8f44aff 2579 netdev_features_t changed = dev->features ^ features;
892ef5d8 2580
f646968f 2581 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
892ef5d8
JP		 2582 cxgb_vlan_mode(dev, features);
2583
2584 return 0;
2585}
2586
4d22de3e
DLR		 2587#ifdef CONFIG_NET_POLL_CONTROLLER
2588static void cxgb_netpoll(struct net_device *dev)
2589{
890de332 2590 struct port_info *pi = netdev_priv(dev);
5fbf816f 2591 struct adapter *adapter = pi->adapter;
890de332 2592 int qidx;
4d22de3e 2593
890de332
DLR		 2594 for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2595 struct sge_qset *qs = &adapter->sge.qs[qidx];
2596 void *source;
2eab17ab 2597
890de332
DLR		 2598 if (adapter->flags & USING_MSIX)
2599 source = qs;
2600 else
2601 source = adapter;
2602
2603 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2604 }
4d22de3e
DLR		 2605}
2606#endif
2607
2608/*
2609 * Periodic accumulation of MAC statistics.
2610 */
2611static void mac_stats_update(struct adapter *adapter)
2612{
2613 int i;
2614
2615 for_each_port(adapter, i) {
2616 struct net_device *dev = adapter->port[i];
2617 struct port_info *p = netdev_priv(dev);
2618
2619 if (netif_running(dev)) {
2620 spin_lock(&adapter->stats_lock);
2621 t3_mac_update_stats(&p->mac);
2622 spin_unlock(&adapter->stats_lock);
2623 }
2624 }
2625}
2626
2627static void check_link_status(struct adapter *adapter)
2628{
2629 int i;
2630
2631 for_each_port(adapter, i) {
2632 struct net_device *dev = adapter->port[i];
2633 struct port_info *p = netdev_priv(dev);
c22c8149 2634 int link_fault;
4d22de3e 2635
bf792094 2636 spin_lock_irq(&adapter->work_lock);
c22c8149
DLR		 2637 link_fault = p->link_fault;
2638 spin_unlock_irq(&adapter->work_lock);
2639
2640 if (link_fault) {
3851c66c 2641 t3_link_fault(adapter, i);
bf792094
DLR		 2642 continue;
2643 }
bf792094
DLR		 2644
2645 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2646 t3_xgm_intr_disable(adapter, i);
2647 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2648
4d22de3e 2649 t3_link_changed(adapter, i);
bf792094
DLR		 2650 t3_xgm_intr_enable(adapter, i);
2651 }
4d22de3e
DLR		 2652 }
2653}
2654
fc90664e
DLR		 2655static void check_t3b2_mac(struct adapter *adapter)
2656{
2657 int i;
2658
f2d961c9
DLR		 2659 if (!rtnl_trylock()) /* synchronize with ifdown */
2660 return;
2661
fc90664e
DLR		 2662 for_each_port(adapter, i) {
2663 struct net_device *dev = adapter->port[i];
2664 struct port_info *p = netdev_priv(dev);
2665 int status;
2666
2667 if (!netif_running(dev))
2668 continue;
2669
2670 status = 0;
6d6dabac 2671 if (netif_running(dev) && netif_carrier_ok(dev))
fc90664e
DLR		 2672 status = t3b2_mac_watchdog_task(&p->mac);
2673 if (status == 1)
2674 p->mac.stats.num_toggled++;
2675 else if (status == 2) {
2676 struct cmac *mac = &p->mac;
2677
2678 t3_mac_set_mtu(mac, dev->mtu);
f14d42f3 2679 t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
fc90664e
DLR		 2680 cxgb_set_rxmode(dev);
2681 t3_link_start(&p->phy, mac, &p->link_config);
2682 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2683 t3_port_intr_enable(adapter, p->port_id);
2684 p->mac.stats.num_resets++;
2685 }
2686 }
2687 rtnl_unlock();
2688}
2689
2690
4d22de3e
DLR		 2691static void t3_adap_check_task(struct work_struct *work)
2692{
2693 struct adapter *adapter = container_of(work, struct adapter,
2694 adap_check_task.work);
2695 const struct adapter_params *p = &adapter->params;
fc882196
DLR		 2696 int port;
2697 unsigned int v, status, reset;
4d22de3e
DLR		 2698
2699 adapter->check_task_cnt++;
2700
3851c66c 2701 check_link_status(adapter);
4d22de3e
DLR		 2702
2703 /* Accumulate MAC stats if needed */
2704 if (!p->linkpoll_period ||
2705 (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2706 p->stats_update_period) {
2707 mac_stats_update(adapter);
2708 adapter->check_task_cnt = 0;
2709 }
2710
fc90664e
DLR		 2711 if (p->rev == T3_REV_B2)
2712 check_t3b2_mac(adapter);
2713
fc882196
DLR		 2714 /*
2715 * Scan the XGMAC's to check for various conditions which we want to
2716 * monitor in a periodic polling manner rather than via an interrupt
2717 * condition. This is used for conditions which would otherwise flood
2718 * the system with interrupts and we only really need to know that the
2719 * conditions are "happening" ... For each condition we count the
2720 * detection of the condition and reset it for the next polling loop.
2721 */
2722 for_each_port(adapter, port) {
2723 struct cmac *mac = &adap2pinfo(adapter, port)->mac;
2724 u32 cause;
2725
2726 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2727 reset = 0;
2728 if (cause & F_RXFIFO_OVERFLOW) {
2729 mac->stats.rx_fifo_ovfl++;
2730 reset |= F_RXFIFO_OVERFLOW;
2731 }
2732
2733 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2734 }
2735
2736 /*
2737 * We do the same as above for FL_EMPTY interrupts.
2738 */
2739 status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2740 reset = 0;
2741
2742 if (status & F_FLEMPTY) {
2743 struct sge_qset *qs = &adapter->sge.qs[0];
2744 int i = 0;
2745
2746 reset |= F_FLEMPTY;
2747
2748 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2749 0xffff;
2750
2751 while (v) {
2752 qs->fl[i].empty += (v & 1);
2753 if (i)
2754 qs++;
2755 i ^= 1;
2756 v >>= 1;
2757 }
2758 }
2759
2760 t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2761
4d22de3e 2762 /* Schedule the next check update if any port is active. */
20d3fc11 2763 spin_lock_irq(&adapter->work_lock);
4d22de3e
DLR		 2764 if (adapter->open_device_map & PORT_MASK)
2765 schedule_chk_task(adapter);
20d3fc11 2766 spin_unlock_irq(&adapter->work_lock);
4d22de3e
DLR		 2767}
2768
e998f245
SW		 2769static void db_full_task(struct work_struct *work)
2770{
2771 struct adapter *adapter = container_of(work, struct adapter,
2772 db_full_task);
2773
2774 cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2775}
2776
2777static void db_empty_task(struct work_struct *work)
2778{
2779 struct adapter *adapter = container_of(work, struct adapter,
2780 db_empty_task);
2781
2782 cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2783}
2784
2785static void db_drop_task(struct work_struct *work)
2786{
2787 struct adapter *adapter = container_of(work, struct adapter,
2788 db_drop_task);
2789 unsigned long delay = 1000;
2790 unsigned short r;
2791
2792 cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2793
2794 /*
2795 * Sleep a while before ringing the driver qset dbs.
2796 * The delay is between 1000-2023 usecs.
2797 */
2798 get_random_bytes(&r, 2);
2799 delay += r & 1023;
2800 set_current_state(TASK_UNINTERRUPTIBLE);
2801 schedule_timeout(usecs_to_jiffies(delay));
2802 ring_dbs(adapter);
2803}
2804
4d22de3e
DLR		 2805/*
2806 * Processes external (PHY) interrupts in process context.
2807 */
2808static void ext_intr_task(struct work_struct *work)
2809{
2810 struct adapter *adapter = container_of(work, struct adapter,
2811 ext_intr_handler_task);
bf792094
DLR		 2812 int i;
2813
2814 /* Disable link fault interrupts */
2815 for_each_port(adapter, i) {
2816 struct net_device *dev = adapter->port[i];
2817 struct port_info *p = netdev_priv(dev);
2818
2819 t3_xgm_intr_disable(adapter, i);
2820 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2821 }
4d22de3e 2822
bf792094 2823 /* Re-enable link fault interrupts */
4d22de3e
DLR		 2824 t3_phy_intr_handler(adapter);
2825
bf792094
DLR		 2826 for_each_port(adapter, i)
2827 t3_xgm_intr_enable(adapter, i);
2828
4d22de3e
DLR		 2829 /* Now reenable external interrupts */
2830 spin_lock_irq(&adapter->work_lock);
2831 if (adapter->slow_intr_mask) {
2832 adapter->slow_intr_mask |= F_T3DBG;
2833 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2834 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2835 adapter->slow_intr_mask);
2836 }
2837 spin_unlock_irq(&adapter->work_lock);
2838}
2839
2840/*
2841 * Interrupt-context handler for external (PHY) interrupts.
2842 */
2843void t3_os_ext_intr_handler(struct adapter *adapter)
2844{
2845 /*
2846 * Schedule a task to handle external interrupts as they may be slow
2847 * and we use a mutex to protect MDIO registers. We disable PHY
2848 * interrupts in the meantime and let the task reenable them when
2849 * it's done.
2850 */
2851 spin_lock(&adapter->work_lock);
2852 if (adapter->slow_intr_mask) {
2853 adapter->slow_intr_mask &= ~F_T3DBG;
2854 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2855 adapter->slow_intr_mask);
2856 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2857 }
2858 spin_unlock(&adapter->work_lock);
2859}
2860
bf792094
DLR		 2861void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2862{
2863 struct net_device *netdev = adapter->port[port_id];
2864 struct port_info *pi = netdev_priv(netdev);
2865
2866 spin_lock(&adapter->work_lock);
2867 pi->link_fault = 1;
bf792094
DLR		 2868 spin_unlock(&adapter->work_lock);
2869}
2870
55bc3228 2871static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
20d3fc11
DLR		 2872{
2873 int i, ret = 0;
2874
cb0bc205
DLR		 2875 if (is_offload(adapter) &&
2876 test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
fa0d4c11 2877 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
cb0bc205
DLR		 2878 offload_close(&adapter->tdev);
2879 }
2880
20d3fc11
DLR		 2881 /* Stop all ports */
2882 for_each_port(adapter, i) {
2883 struct net_device *netdev = adapter->port[i];
2884
2885 if (netif_running(netdev))
55bc3228 2886 __cxgb_close(netdev, on_wq);
20d3fc11
DLR		 2887 }
2888
20d3fc11
DLR		 2889 /* Stop SGE timers */
2890 t3_stop_sge_timers(adapter);
2891
2892 adapter->flags &= ~FULL_INIT_DONE;
2893
2894 if (reset)
2895 ret = t3_reset_adapter(adapter);
2896
2897 pci_disable_device(adapter->pdev);
2898
2899 return ret;
2900}
2901
2902static int t3_reenable_adapter(struct adapter *adapter)
2903{
2904 if (pci_enable_device(adapter->pdev)) {
2905 dev_err(&adapter->pdev->dev,
2906 "Cannot re-enable PCI device after reset.\n");
2907 goto err;
2908 }
2909 pci_set_master(adapter->pdev);
2910 pci_restore_state(adapter->pdev);
ccdddf50 2911 pci_save_state(adapter->pdev);
20d3fc11
DLR		 2912
2913 /* Free sge resources */
2914 t3_free_sge_resources(adapter);
2915
2916 if (t3_replay_prep_adapter(adapter))
2917 goto err;
2918
2919 return 0;
2920err:
2921 return -1;
2922}
2923
2924static void t3_resume_ports(struct adapter *adapter)
2925{
2926 int i;
2927
2928 /* Restart the ports */
2929 for_each_port(adapter, i) {
2930 struct net_device *netdev = adapter->port[i];
2931
2932 if (netif_running(netdev)) {
2933 if (cxgb_open(netdev)) {
2934 dev_err(&adapter->pdev->dev,
2935 "can't bring device back up"
2936 " after reset\n");
2937 continue;
2938 }
2939 }
2940 }
cb0bc205
DLR		 2941
2942 if (is_offload(adapter) && !ofld_disable)
fa0d4c11 2943 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
20d3fc11
DLR		 2944}
2945
2946/*
2947 * processes a fatal error.
2948 * Bring the ports down, reset the chip, bring the ports back up.
2949 */
2950static void fatal_error_task(struct work_struct *work)
2951{
2952 struct adapter *adapter = container_of(work, struct adapter,
2953 fatal_error_handler_task);
2954 int err = 0;
2955
2956 rtnl_lock();
55bc3228 2957 err = t3_adapter_error(adapter, 1, 1);
20d3fc11
DLR		 2958 if (!err)
2959 err = t3_reenable_adapter(adapter);
2960 if (!err)
2961 t3_resume_ports(adapter);
2962
2963 CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2964 rtnl_unlock();
2965}
2966
4d22de3e
DLR		 2967void t3_fatal_err(struct adapter *adapter)
2968{
2969 unsigned int fw_status[4];
2970
2971 if (adapter->flags & FULL_INIT_DONE) {
2972 t3_sge_stop(adapter);
c64c2eae
DLR		 2973 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2974 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2975 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2976 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
20d3fc11
DLR		 2977
2978 spin_lock(&adapter->work_lock);
4d22de3e 2979 t3_intr_disable(adapter);
20d3fc11
DLR		 2980 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2981 spin_unlock(&adapter->work_lock);
4d22de3e
DLR		 2982 }
2983 CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2984 if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2985 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2986 fw_status[0], fw_status[1],
2987 fw_status[2], fw_status[3]);
4d22de3e
DLR		 2988}
2989
91a6b50c
DLR		 2990/**
2991 * t3_io_error_detected - called when PCI error is detected
2992 * @pdev: Pointer to PCI device
2993 * @state: The current pci connection state
2994 *
2995 * This function is called after a PCI bus error affecting
2996 * this device has been detected.
2997 */
2998static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2999 pci_channel_state_t state)
3000{
bc4b6b52 3001 struct adapter *adapter = pci_get_drvdata(pdev);
91a6b50c 3002
e8d19370
DLR		 3003 if (state == pci_channel_io_perm_failure)
3004 return PCI_ERS_RESULT_DISCONNECT;
3005
c661c4a2 3006 t3_adapter_error(adapter, 0, 0);
91a6b50c 3007
48c4b6db 3008 /* Request a slot reset. */
91a6b50c
DLR		 3009 return PCI_ERS_RESULT_NEED_RESET;
3010}
3011
3012/**
3013 * t3_io_slot_reset - called after the pci bus has been reset.
3014 * @pdev: Pointer to PCI device
3015 *
3016 * Restart the card from scratch, as if from a cold-boot.
3017 */
3018static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3019{
bc4b6b52 3020 struct adapter *adapter = pci_get_drvdata(pdev);
91a6b50c 3021
20d3fc11
DLR		 3022 if (!t3_reenable_adapter(adapter))
3023 return PCI_ERS_RESULT_RECOVERED;
91a6b50c 3024
48c4b6db 3025 return PCI_ERS_RESULT_DISCONNECT;
91a6b50c
DLR		 3026}
3027
3028/**
3029 * t3_io_resume - called when traffic can start flowing again.
3030 * @pdev: Pointer to PCI device
3031 *
3032 * This callback is called when the error recovery driver tells us that
3033 * its OK to resume normal operation.
3034 */
3035static void t3_io_resume(struct pci_dev *pdev)
3036{
bc4b6b52 3037 struct adapter *adapter = pci_get_drvdata(pdev);
91a6b50c 3038
68f40c10
DLR		 3039 CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3040 t3_read_reg(adapter, A_PCIE_PEX_ERR));
3041
7cc47d13 3042 rtnl_lock();
20d3fc11 3043 t3_resume_ports(adapter);
7cc47d13 3044 rtnl_unlock();
91a6b50c
DLR		 3045}
3046
3646f0e5 3047static const struct pci_error_handlers t3_err_handler = {
91a6b50c
DLR		 3048 .error_detected = t3_io_error_detected,
3049 .slot_reset = t3_io_slot_reset,
3050 .resume = t3_io_resume,
3051};
3052
8c263761
DLR		 3053/*
3054 * Set the number of qsets based on the number of CPUs and the number of ports,
3055 * not to exceed the number of available qsets, assuming there are enough qsets
3056 * per port in HW.
3057 */
3058static void set_nqsets(struct adapter *adap)
3059{
3060 int i, j = 0;
dbfa6001 3061 int num_cpus = netif_get_num_default_rss_queues();
8c263761 3062 int hwports = adap->params.nports;
5cda9364 3063 int nqsets = adap->msix_nvectors - 1;
8c263761 3064
f9ee3882 3065 if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
8c263761
DLR		 3066 if (hwports == 2 &&
3067 (hwports * nqsets > SGE_QSETS ||
3068 num_cpus >= nqsets / hwports))
3069 nqsets /= hwports;
3070 if (nqsets > num_cpus)
3071 nqsets = num_cpus;
3072 if (nqsets < 1 || hwports == 4)
3073 nqsets = 1;
3074 } else
3075 nqsets = 1;
3076
3077 for_each_port(adap, i) {
3078 struct port_info *pi = adap2pinfo(adap, i);
3079
3080 pi->first_qset = j;
3081 pi->nqsets = nqsets;
3082 j = pi->first_qset + nqsets;
3083
3084 dev_info(&adap->pdev->dev,
3085 "Port %d using %d queue sets.\n", i, nqsets);
3086 }
3087}
3088
2109eaab 3089static int cxgb_enable_msix(struct adapter *adap)
4d22de3e
DLR		 3090{
3091 struct msix_entry entries[SGE_QSETS + 1];
5cda9364 3092 int vectors;
fc1d0bf1 3093 int i;
4d22de3e 3094
5cda9364
DLR		 3095 vectors = ARRAY_SIZE(entries);
3096 for (i = 0; i < vectors; ++i)
4d22de3e
DLR		 3097 entries[i].entry = i;
3098
fc1d0bf1
AG		 3099 vectors = pci_enable_msix_range(adap->pdev, entries,
3100 adap->params.nports + 1, vectors);
3101 if (vectors < 0)
3102 return vectors;
5cda9364 3103
fc1d0bf1
AG		 3104 for (i = 0; i < vectors; ++i)
3105 adap->msix_info[i].vec = entries[i].vector;
3106 adap->msix_nvectors = vectors;
5cda9364 3107
fc1d0bf1 3108 return 0;
4d22de3e
DLR		 3109}
3110
1dd06ae8 3111static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
4d22de3e
DLR		 3112{
3113 static const char *pci_variant[] = {
3114 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3115 };
3116
3117 int i;
3118 char buf[80];
3119
3120 if (is_pcie(adap))
3121 snprintf(buf, sizeof(buf), "%s x%d",
3122 pci_variant[adap->params.pci.variant],
3123 adap->params.pci.width);
3124 else
3125 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3126 pci_variant[adap->params.pci.variant],
3127 adap->params.pci.speed, adap->params.pci.width);
3128
3129 for_each_port(adap, i) {
3130 struct net_device *dev = adap->port[i];
3131 const struct port_info *pi = netdev_priv(dev);
3132
3133 if (!test_bit(i, &adap->registered_device_map))
3134 continue;
428ac43f
JP		 3135 netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3136 ai->desc, pi->phy.desc,
3137 is_offload(adap) ? "R" : "", adap->params.rev, buf,
3138 (adap->flags & USING_MSIX) ? " MSI-X" :
3139 (adap->flags & USING_MSI) ? " MSI" : "");
4d22de3e 3140 if (adap->name == dev->name && adap->params.vpd.mclk)
428ac43f 3141 pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
4d22de3e
DLR		 3142 adap->name, t3_mc7_size(&adap->cm) >> 20,
3143 t3_mc7_size(&adap->pmtx) >> 20,
167cdf5f
DLR		 3144 t3_mc7_size(&adap->pmrx) >> 20,
3145 adap->params.vpd.sn);
4d22de3e
DLR		 3146 }
3147}
3148
dd752696
SH		 3149static const struct net_device_ops cxgb_netdev_ops = {
3150 .ndo_open = cxgb_open,
3151 .ndo_stop = cxgb_close,
43a944f3 3152 .ndo_start_xmit = t3_eth_xmit,
dd752696
SH		 3153 .ndo_get_stats = cxgb_get_stats,
3154 .ndo_validate_addr = eth_validate_addr,
afc4b13d 3155 .ndo_set_rx_mode = cxgb_set_rxmode,
dd752696
SH		 3156 .ndo_do_ioctl = cxgb_ioctl,
3157 .ndo_change_mtu = cxgb_change_mtu,
3158 .ndo_set_mac_address = cxgb_set_mac_addr,
892ef5d8
JP		 3159 .ndo_fix_features = cxgb_fix_features,
3160 .ndo_set_features = cxgb_set_features,
dd752696
SH		 3161#ifdef CONFIG_NET_POLL_CONTROLLER
3162 .ndo_poll_controller = cxgb_netpoll,
3163#endif
3164};
3165
2109eaab 3166static void cxgb3_init_iscsi_mac(struct net_device *dev)
f14d42f3
KX		 3167{
3168 struct port_info *pi = netdev_priv(dev);
3169
3170 memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3171 pi->iscsic.mac_addr[3] |= 0x80;
3172}
3173
1d962ecf 3174#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3175#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3176 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
1dd06ae8 3177static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
4d22de3e 3178{
4d22de3e 3179 int i, err, pci_using_dac = 0;
68f40c10 3180 resource_size_t mmio_start, mmio_len;
4d22de3e
DLR		 3181 const struct adapter_info *ai;
3182 struct adapter *adapter = NULL;
3183 struct port_info *pi;
3184
428ac43f 3185 pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
4d22de3e
DLR		 3186
3187 if (!cxgb3_wq) {
3188 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3189 if (!cxgb3_wq) {
428ac43f 3190 pr_err("cannot initialize work queue\n");
4d22de3e
DLR		 3191 return -ENOMEM;
3192 }
3193 }
3194
7aaaaa1e 3195 err = pci_enable_device(pdev);
4d22de3e 3196 if (err) {
7aaaaa1e
KV		 3197 dev_err(&pdev->dev, "cannot enable PCI device\n");
3198 goto out;
4d22de3e
DLR		 3199 }
3200
7aaaaa1e 3201 err = pci_request_regions(pdev, DRV_NAME);
4d22de3e 3202 if (err) {
7aaaaa1e
KV		 3203 /* Just info, some other driver may have claimed the device. */
3204 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3205 goto out_disable_device;
4d22de3e
DLR		 3206 }
3207
6a35528a 3208 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
4d22de3e 3209 pci_using_dac = 1;
6a35528a 3210 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
4d22de3e
DLR		 3211 if (err) {
3212 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3213 "coherent allocations\n");
7aaaaa1e 3214 goto out_release_regions;
4d22de3e 3215 }
284901a9 3216 } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
4d22de3e 3217 dev_err(&pdev->dev, "no usable DMA configuration\n");
7aaaaa1e 3218 goto out_release_regions;
4d22de3e
DLR		 3219 }
3220
3221 pci_set_master(pdev);
204e2f98 3222 pci_save_state(pdev);
4d22de3e
DLR		 3223
3224 mmio_start = pci_resource_start(pdev, 0);
3225 mmio_len = pci_resource_len(pdev, 0);
3226 ai = t3_get_adapter_info(ent->driver_data);
3227
3228 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3229 if (!adapter) {
3230 err = -ENOMEM;
7aaaaa1e 3231 goto out_release_regions;
4d22de3e
DLR		 3232 }
3233
74b793e1
DLR		 3234 adapter->nofail_skb =
3235 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3236 if (!adapter->nofail_skb) {
3237 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3238 err = -ENOMEM;
3239 goto out_free_adapter;
3240 }
3241
4d22de3e
DLR		 3242 adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3243 if (!adapter->regs) {
3244 dev_err(&pdev->dev, "cannot map device registers\n");
3245 err = -ENOMEM;
3246 goto out_free_adapter;
3247 }
3248
3249 adapter->pdev = pdev;
3250 adapter->name = pci_name(pdev);
3251 adapter->msg_enable = dflt_msg_enable;
3252 adapter->mmio_len = mmio_len;
3253
3254 mutex_init(&adapter->mdio_lock);
3255 spin_lock_init(&adapter->work_lock);
3256 spin_lock_init(&adapter->stats_lock);
3257
3258 INIT_LIST_HEAD(&adapter->adapter_list);
3259 INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
20d3fc11 3260 INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
e998f245
SW		 3261
3262 INIT_WORK(&adapter->db_full_task, db_full_task);
3263 INIT_WORK(&adapter->db_empty_task, db_empty_task);
3264 INIT_WORK(&adapter->db_drop_task, db_drop_task);
3265
4d22de3e
DLR		 3266 INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3267
952cdf33 3268 for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
4d22de3e
DLR		 3269 struct net_device *netdev;
3270
82ad3329 3271 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
4d22de3e
DLR		 3272 if (!netdev) {
3273 err = -ENOMEM;
3274 goto out_free_dev;
3275 }
3276
4d22de3e
DLR		 3277 SET_NETDEV_DEV(netdev, &pdev->dev);
3278
3279 adapter->port[i] = netdev;
3280 pi = netdev_priv(netdev);
5fbf816f 3281 pi->adapter = adapter;
4d22de3e
DLR		 3282 pi->port_id = i;
3283 netif_carrier_off(netdev);
3284 netdev->irq = pdev->irq;
3285 netdev->mem_start = mmio_start;
3286 netdev->mem_end = mmio_start + mmio_len - 1;
d2fe2755 3287 netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
f646968f
PM		 3288 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
3289 netdev->features |= netdev->hw_features |
3290 NETIF_F_HW_VLAN_CTAG_TX;
1d962ecf 3291 netdev->vlan_features |= netdev->features & VLAN_FEAT;
4d22de3e
DLR		 3292 if (pci_using_dac)
3293 netdev->features |= NETIF_F_HIGHDMA;
3294
dd752696 3295 netdev->netdev_ops = &cxgb_netdev_ops;
7ad24ea4 3296 netdev->ethtool_ops = &cxgb_ethtool_ops;
4d22de3e
DLR		 3297 }
3298
5fbf816f 3299 pci_set_drvdata(pdev, adapter);
4d22de3e
DLR		 3300 if (t3_prep_adapter(adapter, ai, 1) < 0) {
3301 err = -ENODEV;
3302 goto out_free_dev;
3303 }
2eab17ab 3304
4d22de3e
DLR		 3305 /*
3306 * The card is now ready to go. If any errors occur during device
3307 * registration we do not fail the whole card but rather proceed only
3308 * with the ports we manage to register successfully. However we must
3309 * register at least one net device.
3310 */
3311 for_each_port(adapter, i) {
3312 err = register_netdev(adapter->port[i]);
3313 if (err)
3314 dev_warn(&pdev->dev,
3315 "cannot register net device %s, skipping\n",
3316 adapter->port[i]->name);
3317 else {
3318 /*
3319 * Change the name we use for messages to the name of
3320 * the first successfully registered interface.
3321 */
3322 if (!adapter->registered_device_map)
3323 adapter->name = adapter->port[i]->name;
3324
3325 __set_bit(i, &adapter->registered_device_map);
3326 }
3327 }
3328 if (!adapter->registered_device_map) {
3329 dev_err(&pdev->dev, "could not register any net devices\n");
3330 goto out_free_dev;
3331 }
3332
f14d42f3
KX		 3333 for_each_port(adapter, i)
3334 cxgb3_init_iscsi_mac(adapter->port[i]);
3335
4d22de3e
DLR		 3336 /* Driver's ready. Reflect it on LEDs */
3337 t3_led_ready(adapter);
3338
3339 if (is_offload(adapter)) {
3340 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3341 cxgb3_adapter_ofld(adapter);
3342 }
3343
3344 /* See what interrupts we'll be using */
3345 if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3346 adapter->flags |= USING_MSIX;
3347 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3348 adapter->flags |= USING_MSI;
3349
8c263761
DLR		 3350 set_nqsets(adapter);
3351
0ee8d33c 3352 err = sysfs_create_group(&adapter->port[0]->dev.kobj,
4d22de3e
DLR		 3353 &cxgb3_attr_group);
3354
3355 print_port_info(adapter, ai);
3356 return 0;
3357
3358out_free_dev:
3359 iounmap(adapter->regs);
952cdf33 3360 for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
4d22de3e
DLR		 3361 if (adapter->port[i])
3362 free_netdev(adapter->port[i]);
3363
3364out_free_adapter:
3365 kfree(adapter);
3366
4d22de3e
DLR		 3367out_release_regions:
3368 pci_release_regions(pdev);
7aaaaa1e
KV		 3369out_disable_device:
3370 pci_disable_device(pdev);
7aaaaa1e 3371out:
4d22de3e
DLR		 3372 return err;
3373}
3374
2109eaab 3375static void remove_one(struct pci_dev *pdev)
4d22de3e 3376{
5fbf816f 3377 struct adapter *adapter = pci_get_drvdata(pdev);
4d22de3e 3378
5fbf816f 3379 if (adapter) {
4d22de3e 3380 int i;
4d22de3e
DLR		 3381
3382 t3_sge_stop(adapter);
0ee8d33c 3383 sysfs_remove_group(&adapter->port[0]->dev.kobj,
4d22de3e
DLR		 3384 &cxgb3_attr_group);
3385
4d22de3e
DLR		 3386 if (is_offload(adapter)) {
3387 cxgb3_adapter_unofld(adapter);
3388 if (test_bit(OFFLOAD_DEVMAP_BIT,
3389 &adapter->open_device_map))
3390 offload_close(&adapter->tdev);
3391 }
3392
67d92ab7
DLR		 3393 for_each_port(adapter, i)
3394 if (test_bit(i, &adapter->registered_device_map))
3395 unregister_netdev(adapter->port[i]);
3396
0ca41c04 3397 t3_stop_sge_timers(adapter);
4d22de3e
DLR		 3398 t3_free_sge_resources(adapter);
3399 cxgb_disable_msi(adapter);
3400
4d22de3e
DLR		 3401 for_each_port(adapter, i)
3402 if (adapter->port[i])
3403 free_netdev(adapter->port[i]);
3404
3405 iounmap(adapter->regs);
74b793e1
DLR		 3406 if (adapter->nofail_skb)
3407 kfree_skb(adapter->nofail_skb);
4d22de3e
DLR		 3408 kfree(adapter);
3409 pci_release_regions(pdev);
3410 pci_disable_device(pdev);
4d22de3e
DLR		 3411 }
3412}
3413
3414static struct pci_driver driver = {
3415 .name = DRV_NAME,
3416 .id_table = cxgb3_pci_tbl,
3417 .probe = init_one,
2109eaab 3418 .remove = remove_one,
91a6b50c 3419 .err_handler = &t3_err_handler,
4d22de3e
DLR		 3420};
3421
3422static int __init cxgb3_init_module(void)
3423{
3424 int ret;
3425
3426 cxgb3_offload_init();
3427
3428 ret = pci_register_driver(&driver);
3429 return ret;
3430}
3431
3432static void __exit cxgb3_cleanup_module(void)
3433{
3434 pci_unregister_driver(&driver);
3435 if (cxgb3_wq)
3436 destroy_workqueue(cxgb3_wq);
3437}
3438
3439module_init(cxgb3_init_module);
3440module_exit(cxgb3_cleanup_module);
Linux 6.x block layer and io_uring tree(s)