projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4: Add support for T4 configuration file
[linux-2.6-block.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
63#include <asm/uaccess.h>
64
65#include "cxgb4.h"
66#include "t4_regs.h"
67#include "t4_msg.h"
68#include "t4fw_api.h"
69#include "l2t.h"
70
99e6d065 71#define DRV_VERSION "1.3.0-ko"
b8ff05a9
DM		 72#define DRV_DESC "Chelsio T4 Network Driver"
73
74/*
75 * Max interrupt hold-off timer value in us. Queues fall back to this value
76 * under extreme memory pressure so it's largish to give the system time to
77 * recover.
78 */
79#define MAX_SGE_TIMERVAL 200U
80
7ee9ff94
CL		 81#ifdef CONFIG_PCI_IOV
82/*
83 * Virtual Function provisioning constants. We need two extra Ingress Queues
84 * with Interrupt capability to serve as the VF's Firmware Event Queue and
85 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
86 * Lists associated with them). For each Ethernet/Control Egress Queue and
87 * for each Free List, we need an Egress Context.
88 */
89enum {
90 VFRES_NPORTS = 1, /* # of "ports" per VF */
91 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
92
93 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
94 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
95 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
96 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
97 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
98 VFRES_TC = 0, /* PCI-E traffic class */
99 VFRES_NEXACTF = 16, /* # of exact MPS filters */
100
101 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
102 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
103};
104
105/*
106 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
107 * static and likely not to be useful in the long run. We really need to
108 * implement some form of persistent configuration which the firmware
109 * controls.
110 */
111static unsigned int pfvfres_pmask(struct adapter *adapter,
112 unsigned int pf, unsigned int vf)
113{
114 unsigned int portn, portvec;
115
116 /*
117 * Give PF's access to all of the ports.
118 */
119 if (vf == 0)
120 return FW_PFVF_CMD_PMASK_MASK;
121
122 /*
123 * For VFs, we'll assign them access to the ports based purely on the
124 * PF. We assign active ports in order, wrapping around if there are
125 * fewer active ports than PFs: e.g. active port[pf % nports].
126 * Unfortunately the adapter's port_info structs haven't been
127 * initialized yet so we have to compute this.
128 */
129 if (adapter->params.nports == 0)
130 return 0;
131
132 portn = pf % adapter->params.nports;
133 portvec = adapter->params.portvec;
134 for (;;) {
135 /*
136 * Isolate the lowest set bit in the port vector. If we're at
137 * the port number that we want, return that as the pmask.
138 * otherwise mask that bit out of the port vector and
139 * decrement our port number ...
140 */
141 unsigned int pmask = portvec ^ (portvec & (portvec-1));
142 if (portn == 0)
143 return pmask;
144 portn--;
145 portvec &= ~pmask;
146 }
147 /*NOTREACHED*/
148}
149#endif
150
b8ff05a9
DM		 151enum {
152 MAX_TXQ_ENTRIES = 16384,
153 MAX_CTRL_TXQ_ENTRIES = 1024,
154 MAX_RSPQ_ENTRIES = 16384,
155 MAX_RX_BUFFERS = 16384,
156 MIN_TXQ_ENTRIES = 32,
157 MIN_CTRL_TXQ_ENTRIES = 32,
158 MIN_RSPQ_ENTRIES = 128,
159 MIN_FL_ENTRIES = 16
160};
161
162#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
163 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
164 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
165
060e0c75 166#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
b8ff05a9
DM		 167
168static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
060e0c75 169 CH_DEVICE(0xa000, 0), /* PE10K */
ccea790e
DM		 170 CH_DEVICE(0x4001, -1),
171 CH_DEVICE(0x4002, -1),
172 CH_DEVICE(0x4003, -1),
173 CH_DEVICE(0x4004, -1),
174 CH_DEVICE(0x4005, -1),
175 CH_DEVICE(0x4006, -1),
176 CH_DEVICE(0x4007, -1),
177 CH_DEVICE(0x4008, -1),
178 CH_DEVICE(0x4009, -1),
179 CH_DEVICE(0x400a, -1),
180 CH_DEVICE(0x4401, 4),
181 CH_DEVICE(0x4402, 4),
182 CH_DEVICE(0x4403, 4),
183 CH_DEVICE(0x4404, 4),
184 CH_DEVICE(0x4405, 4),
185 CH_DEVICE(0x4406, 4),
186 CH_DEVICE(0x4407, 4),
187 CH_DEVICE(0x4408, 4),
188 CH_DEVICE(0x4409, 4),
189 CH_DEVICE(0x440a, 4),
f637d577
VP		 190 CH_DEVICE(0x440d, 4),
191 CH_DEVICE(0x440e, 4),
b8ff05a9
DM		 192 { 0, }
193};
194
195#define FW_FNAME "cxgb4/t4fw.bin"
636f9d37 196#define FW_CFNAME "cxgb4/t4-config.txt"
b8ff05a9
DM		 197
198MODULE_DESCRIPTION(DRV_DESC);
199MODULE_AUTHOR("Chelsio Communications");
200MODULE_LICENSE("Dual BSD/GPL");
201MODULE_VERSION(DRV_VERSION);
202MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
203MODULE_FIRMWARE(FW_FNAME);
204
636f9d37
VP		 205/*
206 * Normally we're willing to become the firmware's Master PF but will be happy
207 * if another PF has already become the Master and initialized the adapter.
208 * Setting "force_init" will cause this driver to forcibly establish itself as
209 * the Master PF and initialize the adapter.
210 */
211static uint force_init;
212
213module_param(force_init, uint, 0644);
214MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
215
b8ff05a9
DM		 216static int dflt_msg_enable = DFLT_MSG_ENABLE;
217
218module_param(dflt_msg_enable, int, 0644);
219MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
220
221/*
222 * The driver uses the best interrupt scheme available on a platform in the
223 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
224 * of these schemes the driver may consider as follows:
225 *
226 * msi = 2: choose from among all three options
227 * msi = 1: only consider MSI and INTx interrupts
228 * msi = 0: force INTx interrupts
229 */
230static int msi = 2;
231
232module_param(msi, int, 0644);
233MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
234
235/*
236 * Queue interrupt hold-off timer values. Queues default to the first of these
237 * upon creation.
238 */
239static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
240
241module_param_array(intr_holdoff, uint, NULL, 0644);
242MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
243 "0..4 in microseconds");
244
245static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
246
247module_param_array(intr_cnt, uint, NULL, 0644);
248MODULE_PARM_DESC(intr_cnt,
249 "thresholds 1..3 for queue interrupt packet counters");
250
636f9d37
VP		 251/*
252 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
253 * offset by 2 bytes in order to have the IP headers line up on 4-byte
254 * boundaries. This is a requirement for many architectures which will throw
255 * a machine check fault if an attempt is made to access one of the 4-byte IP
256 * header fields on a non-4-byte boundary. And it's a major performance issue
257 * even on some architectures which allow it like some implementations of the
258 * x86 ISA. However, some architectures don't mind this and for some very
259 * edge-case performance sensitive applications (like forwarding large volumes
260 * of small packets), setting this DMA offset to 0 will decrease the number of
261 * PCI-E Bus transfers enough to measurably affect performance.
262 */
263static int rx_dma_offset = 2;
264
eb939922 265static bool vf_acls;
b8ff05a9
DM		 266
267#ifdef CONFIG_PCI_IOV
268module_param(vf_acls, bool, 0644);
269MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
270
271static unsigned int num_vf[4];
272
273module_param_array(num_vf, uint, NULL, 0644);
274MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
275#endif
276
277static struct dentry *cxgb4_debugfs_root;
278
279static LIST_HEAD(adapter_list);
280static DEFINE_MUTEX(uld_mutex);
281static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
282static const char *uld_str[] = { "RDMA", "iSCSI" };
283
284static void link_report(struct net_device *dev)
285{
286 if (!netif_carrier_ok(dev))
287 netdev_info(dev, "link down\n");
288 else {
289 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
290
291 const char *s = "10Mbps";
292 const struct port_info *p = netdev_priv(dev);
293
294 switch (p->link_cfg.speed) {
295 case SPEED_10000:
296 s = "10Gbps";
297 break;
298 case SPEED_1000:
299 s = "1000Mbps";
300 break;
301 case SPEED_100:
302 s = "100Mbps";
303 break;
304 }
305
306 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
307 fc[p->link_cfg.fc]);
308 }
309}
310
311void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
312{
313 struct net_device *dev = adapter->port[port_id];
314
315 /* Skip changes from disabled ports. */
316 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
317 if (link_stat)
318 netif_carrier_on(dev);
319 else
320 netif_carrier_off(dev);
321
322 link_report(dev);
323 }
324}
325
326void t4_os_portmod_changed(const struct adapter *adap, int port_id)
327{
328 static const char *mod_str[] = {
a0881cab 329 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 330 };
331
332 const struct net_device *dev = adap->port[port_id];
333 const struct port_info *pi = netdev_priv(dev);
334
335 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
336 netdev_info(dev, "port module unplugged\n");
a0881cab 337 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 338 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
339}
340
341/*
342 * Configure the exact and hash address filters to handle a port's multicast
343 * and secondary unicast MAC addresses.
344 */
345static int set_addr_filters(const struct net_device *dev, bool sleep)
346{
347 u64 mhash = 0;
348 u64 uhash = 0;
349 bool free = true;
350 u16 filt_idx[7];
351 const u8 *addr[7];
352 int ret, naddr = 0;
b8ff05a9
DM		 353 const struct netdev_hw_addr *ha;
354 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 355 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 356 const struct port_info *pi = netdev_priv(dev);
060e0c75 357 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 358
359 /* first do the secondary unicast addresses */
360 netdev_for_each_uc_addr(ha, dev) {
361 addr[naddr++] = ha->addr;
362 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 363 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 364 naddr, addr, filt_idx, &uhash, sleep);
365 if (ret < 0)
366 return ret;
367
368 free = false;
369 naddr = 0;
370 }
371 }
372
373 /* next set up the multicast addresses */
4a35ecf8
DM		 374 netdev_for_each_mc_addr(ha, dev) {
375 addr[naddr++] = ha->addr;
376 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 377 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 378 naddr, addr, filt_idx, &mhash, sleep);
379 if (ret < 0)
380 return ret;
381
382 free = false;
383 naddr = 0;
384 }
385 }
386
060e0c75 387 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 388 uhash | mhash, sleep);
389}
390
3069ee9b
VP		 391int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
392module_param(dbfifo_int_thresh, int, 0644);
393MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
394
395int dbfifo_drain_delay = 1000; /* usecs to sleep while draining the dbfifo */
396module_param(dbfifo_drain_delay, int, 0644);
397MODULE_PARM_DESC(dbfifo_drain_delay,
398 "usecs to sleep while draining the dbfifo");
399
b8ff05a9
DM		 400/*
401 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
402 * If @mtu is -1 it is left unchanged.
403 */
404static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
405{
406 int ret;
407 struct port_info *pi = netdev_priv(dev);
408
409 ret = set_addr_filters(dev, sleep_ok);
410 if (ret == 0)
060e0c75 411 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 412 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 413 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 414 sleep_ok);
415 return ret;
416}
417
3069ee9b
VP		 418static struct workqueue_struct *workq;
419
b8ff05a9
DM		 420/**
421 * link_start - enable a port
422 * @dev: the port to enable
423 *
424 * Performs the MAC and PHY actions needed to enable a port.
425 */
426static int link_start(struct net_device *dev)
427{
428 int ret;
429 struct port_info *pi = netdev_priv(dev);
060e0c75 430 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 431
432 /*
433 * We do not set address filters and promiscuity here, the stack does
434 * that step explicitly.
435 */
060e0c75 436 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
19ecae2c 437 !!(dev->features & NETIF_F_HW_VLAN_RX), true);
b8ff05a9 438 if (ret == 0) {
060e0c75 439 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 440 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 441 true);
b8ff05a9
DM		 442 if (ret >= 0) {
443 pi->xact_addr_filt = ret;
444 ret = 0;
445 }
446 }
447 if (ret == 0)
060e0c75
DM		 448 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
449 &pi->link_cfg);
b8ff05a9 450 if (ret == 0)
060e0c75 451 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
b8ff05a9
DM		 452 return ret;
453}
454
455/*
456 * Response queue handler for the FW event queue.
457 */
458static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
459 const struct pkt_gl *gl)
460{
461 u8 opcode = ((const struct rss_header *)rsp)->opcode;
462
463 rsp++; /* skip RSS header */
464 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
465 const struct cpl_sge_egr_update *p = (void *)rsp;
466 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 467 struct sge_txq *txq;
b8ff05a9 468
e46dab4d 469 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 470 txq->restarts++;
e46dab4d 471 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 472 struct sge_eth_txq *eq;
473
474 eq = container_of(txq, struct sge_eth_txq, q);
475 netif_tx_wake_queue(eq->txq);
476 } else {
477 struct sge_ofld_txq *oq;
478
479 oq = container_of(txq, struct sge_ofld_txq, q);
480 tasklet_schedule(&oq->qresume_tsk);
481 }
482 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
483 const struct cpl_fw6_msg *p = (void *)rsp;
484
485 if (p->type == 0)
486 t4_handle_fw_rpl(q->adap, p->data);
487 } else if (opcode == CPL_L2T_WRITE_RPL) {
488 const struct cpl_l2t_write_rpl *p = (void *)rsp;
489
490 do_l2t_write_rpl(q->adap, p);
491 } else
492 dev_err(q->adap->pdev_dev,
493 "unexpected CPL %#x on FW event queue\n", opcode);
494 return 0;
495}
496
497/**
498 * uldrx_handler - response queue handler for ULD queues
499 * @q: the response queue that received the packet
500 * @rsp: the response queue descriptor holding the offload message
501 * @gl: the gather list of packet fragments
502 *
503 * Deliver an ingress offload packet to a ULD. All processing is done by
504 * the ULD, we just maintain statistics.
505 */
506static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
507 const struct pkt_gl *gl)
508{
509 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
510
511 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
512 rxq->stats.nomem++;
513 return -1;
514 }
515 if (gl == NULL)
516 rxq->stats.imm++;
517 else if (gl == CXGB4_MSG_AN)
518 rxq->stats.an++;
519 else
520 rxq->stats.pkts++;
521 return 0;
522}
523
524static void disable_msi(struct adapter *adapter)
525{
526 if (adapter->flags & USING_MSIX) {
527 pci_disable_msix(adapter->pdev);
528 adapter->flags &= ~USING_MSIX;
529 } else if (adapter->flags & USING_MSI) {
530 pci_disable_msi(adapter->pdev);
531 adapter->flags &= ~USING_MSI;
532 }
533}
534
535/*
536 * Interrupt handler for non-data events used with MSI-X.
537 */
538static irqreturn_t t4_nondata_intr(int irq, void *cookie)
539{
540 struct adapter *adap = cookie;
541
542 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
543 if (v & PFSW) {
544 adap->swintr = 1;
545 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
546 }
547 t4_slow_intr_handler(adap);
548 return IRQ_HANDLED;
549}
550
551/*
552 * Name the MSI-X interrupts.
553 */
554static void name_msix_vecs(struct adapter *adap)
555{
ba27816c 556 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 557
558 /* non-data interrupts */
b1a3c2b6 559 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 560
561 /* FW events */
b1a3c2b6
DM		 562 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
563 adap->port[0]->name);
b8ff05a9
DM		 564
565 /* Ethernet queues */
566 for_each_port(adap, j) {
567 struct net_device *d = adap->port[j];
568 const struct port_info *pi = netdev_priv(d);
569
ba27816c 570 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 571 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
572 d->name, i);
b8ff05a9
DM		 573 }
574
575 /* offload queues */
ba27816c
DM		 576 for_each_ofldrxq(&adap->sge, i)
577 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 578 adap->port[0]->name, i);
ba27816c
DM		 579
580 for_each_rdmarxq(&adap->sge, i)
581 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 582 adap->port[0]->name, i);
b8ff05a9
DM		 583}
584
585static int request_msix_queue_irqs(struct adapter *adap)
586{
587 struct sge *s = &adap->sge;
588 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
589
590 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
591 adap->msix_info[1].desc, &s->fw_evtq);
592 if (err)
593 return err;
594
595 for_each_ethrxq(s, ethqidx) {
596 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
597 adap->msix_info[msi].desc,
598 &s->ethrxq[ethqidx].rspq);
599 if (err)
600 goto unwind;
601 msi++;
602 }
603 for_each_ofldrxq(s, ofldqidx) {
604 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
605 adap->msix_info[msi].desc,
606 &s->ofldrxq[ofldqidx].rspq);
607 if (err)
608 goto unwind;
609 msi++;
610 }
611 for_each_rdmarxq(s, rdmaqidx) {
612 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
613 adap->msix_info[msi].desc,
614 &s->rdmarxq[rdmaqidx].rspq);
615 if (err)
616 goto unwind;
617 msi++;
618 }
619 return 0;
620
621unwind:
622 while (--rdmaqidx >= 0)
623 free_irq(adap->msix_info[--msi].vec,
624 &s->rdmarxq[rdmaqidx].rspq);
625 while (--ofldqidx >= 0)
626 free_irq(adap->msix_info[--msi].vec,
627 &s->ofldrxq[ofldqidx].rspq);
628 while (--ethqidx >= 0)
629 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
630 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
631 return err;
632}
633
634static void free_msix_queue_irqs(struct adapter *adap)
635{
636 int i, msi = 2;
637 struct sge *s = &adap->sge;
638
639 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
640 for_each_ethrxq(s, i)
641 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
642 for_each_ofldrxq(s, i)
643 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
644 for_each_rdmarxq(s, i)
645 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
646}
647
671b0060
DM		 648/**
649 * write_rss - write the RSS table for a given port
650 * @pi: the port
651 * @queues: array of queue indices for RSS
652 *
653 * Sets up the portion of the HW RSS table for the port's VI to distribute
654 * packets to the Rx queues in @queues.
655 */
656static int write_rss(const struct port_info *pi, const u16 *queues)
657{
658 u16 *rss;
659 int i, err;
660 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
661
662 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
663 if (!rss)
664 return -ENOMEM;
665
666 /* map the queue indices to queue ids */
667 for (i = 0; i < pi->rss_size; i++, queues++)
668 rss[i] = q[*queues].rspq.abs_id;
669
060e0c75
DM		 670 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
671 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 672 kfree(rss);
673 return err;
674}
675
b8ff05a9
DM		 676/**
677 * setup_rss - configure RSS
678 * @adap: the adapter
679 *
671b0060 680 * Sets up RSS for each port.
b8ff05a9
DM		 681 */
682static int setup_rss(struct adapter *adap)
683{
671b0060 684 int i, err;
b8ff05a9
DM		 685
686 for_each_port(adap, i) {
687 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 688
671b0060 689 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 690 if (err)
691 return err;
692 }
693 return 0;
694}
695
e46dab4d
DM		 696/*
697 * Return the channel of the ingress queue with the given qid.
698 */
699static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
700{
701 qid -= p->ingr_start;
702 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
703}
704
b8ff05a9
DM		 705/*
706 * Wait until all NAPI handlers are descheduled.
707 */
708static void quiesce_rx(struct adapter *adap)
709{
710 int i;
711
712 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
713 struct sge_rspq *q = adap->sge.ingr_map[i];
714
715 if (q && q->handler)
716 napi_disable(&q->napi);
717 }
718}
719
720/*
721 * Enable NAPI scheduling and interrupt generation for all Rx queues.
722 */
723static void enable_rx(struct adapter *adap)
724{
725 int i;
726
727 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
728 struct sge_rspq *q = adap->sge.ingr_map[i];
729
730 if (!q)
731 continue;
732 if (q->handler)
733 napi_enable(&q->napi);
734 /* 0-increment GTS to start the timer and enable interrupts */
735 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
736 SEINTARM(q->intr_params) |
737 INGRESSQID(q->cntxt_id));
738 }
739}
740
741/**
742 * setup_sge_queues - configure SGE Tx/Rx/response queues
743 * @adap: the adapter
744 *
745 * Determines how many sets of SGE queues to use and initializes them.
746 * We support multiple queue sets per port if we have MSI-X, otherwise
747 * just one queue set per port.
748 */
749static int setup_sge_queues(struct adapter *adap)
750{
751 int err, msi_idx, i, j;
752 struct sge *s = &adap->sge;
753
754 bitmap_zero(s->starving_fl, MAX_EGRQ);
755 bitmap_zero(s->txq_maperr, MAX_EGRQ);
756
757 if (adap->flags & USING_MSIX)
758 msi_idx = 1; /* vector 0 is for non-queue interrupts */
759 else {
760 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
761 NULL, NULL);
762 if (err)
763 return err;
764 msi_idx = -((int)s->intrq.abs_id + 1);
765 }
766
767 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
768 msi_idx, NULL, fwevtq_handler);
769 if (err) {
770freeout: t4_free_sge_resources(adap);
771 return err;
772 }
773
774 for_each_port(adap, i) {
775 struct net_device *dev = adap->port[i];
776 struct port_info *pi = netdev_priv(dev);
777 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
778 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
779
780 for (j = 0; j < pi->nqsets; j++, q++) {
781 if (msi_idx > 0)
782 msi_idx++;
783 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
784 msi_idx, &q->fl,
785 t4_ethrx_handler);
786 if (err)
787 goto freeout;
788 q->rspq.idx = j;
789 memset(&q->stats, 0, sizeof(q->stats));
790 }
791 for (j = 0; j < pi->nqsets; j++, t++) {
792 err = t4_sge_alloc_eth_txq(adap, t, dev,
793 netdev_get_tx_queue(dev, j),
794 s->fw_evtq.cntxt_id);
795 if (err)
796 goto freeout;
797 }
798 }
799
800 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
801 for_each_ofldrxq(s, i) {
802 struct sge_ofld_rxq *q = &s->ofldrxq[i];
803 struct net_device *dev = adap->port[i / j];
804
805 if (msi_idx > 0)
806 msi_idx++;
807 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
808 &q->fl, uldrx_handler);
809 if (err)
810 goto freeout;
811 memset(&q->stats, 0, sizeof(q->stats));
812 s->ofld_rxq[i] = q->rspq.abs_id;
813 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
814 s->fw_evtq.cntxt_id);
815 if (err)
816 goto freeout;
817 }
818
819 for_each_rdmarxq(s, i) {
820 struct sge_ofld_rxq *q = &s->rdmarxq[i];
821
822 if (msi_idx > 0)
823 msi_idx++;
824 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
825 msi_idx, &q->fl, uldrx_handler);
826 if (err)
827 goto freeout;
828 memset(&q->stats, 0, sizeof(q->stats));
829 s->rdma_rxq[i] = q->rspq.abs_id;
830 }
831
832 for_each_port(adap, i) {
833 /*
834 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
835 * have RDMA queues, and that's the right value.
836 */
837 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
838 s->fw_evtq.cntxt_id,
839 s->rdmarxq[i].rspq.cntxt_id);
840 if (err)
841 goto freeout;
842 }
843
844 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
845 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
846 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
847 return 0;
848}
849
850/*
851 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
852 * started but failed, and a negative errno if flash load couldn't start.
853 */
854static int upgrade_fw(struct adapter *adap)
855{
856 int ret;
857 u32 vers;
858 const struct fw_hdr *hdr;
859 const struct firmware *fw;
860 struct device *dev = adap->pdev_dev;
861
862 ret = request_firmware(&fw, FW_FNAME, dev);
863 if (ret < 0) {
864 dev_err(dev, "unable to load firmware image " FW_FNAME
865 ", error %d\n", ret);
866 return ret;
867 }
868
869 hdr = (const struct fw_hdr *)fw->data;
870 vers = ntohl(hdr->fw_ver);
871 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
872 ret = -EINVAL; /* wrong major version, won't do */
873 goto out;
874 }
875
876 /*
877 * If the flash FW is unusable or we found something newer, load it.
878 */
879 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
880 vers > adap->params.fw_vers) {
881 ret = -t4_load_fw(adap, fw->data, fw->size);
882 if (!ret)
883 dev_info(dev, "firmware upgraded to version %pI4 from "
884 FW_FNAME "\n", &hdr->fw_ver);
885 }
886out: release_firmware(fw);
887 return ret;
888}
889
890/*
891 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
892 * The allocated memory is cleared.
893 */
894void *t4_alloc_mem(size_t size)
895{
89bf67f1 896 void *p = kzalloc(size, GFP_KERNEL);
b8ff05a9
DM		 897
898 if (!p)
89bf67f1 899 p = vzalloc(size);
b8ff05a9
DM		 900 return p;
901}
902
903/*
904 * Free memory allocated through alloc_mem().
905 */
31b9c19b 906static void t4_free_mem(void *addr)
b8ff05a9
DM		 907{
908 if (is_vmalloc_addr(addr))
909 vfree(addr);
910 else
911 kfree(addr);
912}
913
914static inline int is_offload(const struct adapter *adap)
915{
916 return adap->params.offload;
917}
918
919/*
920 * Implementation of ethtool operations.
921 */
922
923static u32 get_msglevel(struct net_device *dev)
924{
925 return netdev2adap(dev)->msg_enable;
926}
927
928static void set_msglevel(struct net_device *dev, u32 val)
929{
930 netdev2adap(dev)->msg_enable = val;
931}
932
933static char stats_strings[][ETH_GSTRING_LEN] = {
934 "TxOctetsOK ",
935 "TxFramesOK ",
936 "TxBroadcastFrames ",
937 "TxMulticastFrames ",
938 "TxUnicastFrames ",
939 "TxErrorFrames ",
940
941 "TxFrames64 ",
942 "TxFrames65To127 ",
943 "TxFrames128To255 ",
944 "TxFrames256To511 ",
945 "TxFrames512To1023 ",
946 "TxFrames1024To1518 ",
947 "TxFrames1519ToMax ",
948
949 "TxFramesDropped ",
950 "TxPauseFrames ",
951 "TxPPP0Frames ",
952 "TxPPP1Frames ",
953 "TxPPP2Frames ",
954 "TxPPP3Frames ",
955 "TxPPP4Frames ",
956 "TxPPP5Frames ",
957 "TxPPP6Frames ",
958 "TxPPP7Frames ",
959
960 "RxOctetsOK ",
961 "RxFramesOK ",
962 "RxBroadcastFrames ",
963 "RxMulticastFrames ",
964 "RxUnicastFrames ",
965
966 "RxFramesTooLong ",
967 "RxJabberErrors ",
968 "RxFCSErrors ",
969 "RxLengthErrors ",
970 "RxSymbolErrors ",
971 "RxRuntFrames ",
972
973 "RxFrames64 ",
974 "RxFrames65To127 ",
975 "RxFrames128To255 ",
976 "RxFrames256To511 ",
977 "RxFrames512To1023 ",
978 "RxFrames1024To1518 ",
979 "RxFrames1519ToMax ",
980
981 "RxPauseFrames ",
982 "RxPPP0Frames ",
983 "RxPPP1Frames ",
984 "RxPPP2Frames ",
985 "RxPPP3Frames ",
986 "RxPPP4Frames ",
987 "RxPPP5Frames ",
988 "RxPPP6Frames ",
989 "RxPPP7Frames ",
990
991 "RxBG0FramesDropped ",
992 "RxBG1FramesDropped ",
993 "RxBG2FramesDropped ",
994 "RxBG3FramesDropped ",
995 "RxBG0FramesTrunc ",
996 "RxBG1FramesTrunc ",
997 "RxBG2FramesTrunc ",
998 "RxBG3FramesTrunc ",
999
1000 "TSO ",
1001 "TxCsumOffload ",
1002 "RxCsumGood ",
1003 "VLANextractions ",
1004 "VLANinsertions ",
4a6346d4
DM		 1005 "GROpackets ",
1006 "GROmerged ",
b8ff05a9
DM		 1007};
1008
1009static int get_sset_count(struct net_device *dev, int sset)
1010{
1011 switch (sset) {
1012 case ETH_SS_STATS:
1013 return ARRAY_SIZE(stats_strings);
1014 default:
1015 return -EOPNOTSUPP;
1016 }
1017}
1018
1019#define T4_REGMAP_SIZE (160 * 1024)
1020
1021static int get_regs_len(struct net_device *dev)
1022{
1023 return T4_REGMAP_SIZE;
1024}
1025
1026static int get_eeprom_len(struct net_device *dev)
1027{
1028 return EEPROMSIZE;
1029}
1030
1031static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1032{
1033 struct adapter *adapter = netdev2adap(dev);
1034
23020ab3
RJ		 1035 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1036 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1037 strlcpy(info->bus_info, pci_name(adapter->pdev),
1038 sizeof(info->bus_info));
b8ff05a9 1039
84b40501 1040 if (adapter->params.fw_vers)
b8ff05a9
DM		 1041 snprintf(info->fw_version, sizeof(info->fw_version),
1042 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1043 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1044 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1045 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1046 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1047 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1048 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1049 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1050 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1051}
1052
1053static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1054{
1055 if (stringset == ETH_SS_STATS)
1056 memcpy(data, stats_strings, sizeof(stats_strings));
1057}
1058
1059/*
1060 * port stats maintained per queue of the port. They should be in the same
1061 * order as in stats_strings above.
1062 */
1063struct queue_port_stats {
1064 u64 tso;
1065 u64 tx_csum;
1066 u64 rx_csum;
1067 u64 vlan_ex;
1068 u64 vlan_ins;
4a6346d4
DM		 1069 u64 gro_pkts;
1070 u64 gro_merged;
b8ff05a9
DM		 1071};
1072
1073static void collect_sge_port_stats(const struct adapter *adap,
1074 const struct port_info *p, struct queue_port_stats *s)
1075{
1076 int i;
1077 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1078 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1079
1080 memset(s, 0, sizeof(*s));
1081 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1082 s->tso += tx->tso;
1083 s->tx_csum += tx->tx_cso;
1084 s->rx_csum += rx->stats.rx_cso;
1085 s->vlan_ex += rx->stats.vlan_ex;
1086 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1087 s->gro_pkts += rx->stats.lro_pkts;
1088 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1089 }
1090}
1091
1092static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1093 u64 *data)
1094{
1095 struct port_info *pi = netdev_priv(dev);
1096 struct adapter *adapter = pi->adapter;
1097
1098 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1099
1100 data += sizeof(struct port_stats) / sizeof(u64);
1101 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1102}
1103
1104/*
1105 * Return a version number to identify the type of adapter. The scheme is:
1106 * - bits 0..9: chip version
1107 * - bits 10..15: chip revision
835bb606 1108 * - bits 16..23: register dump version
b8ff05a9
DM		 1109 */
1110static inline unsigned int mk_adap_vers(const struct adapter *ap)
1111{
835bb606 1112 return 4 | (ap->params.rev << 10) | (1 << 16);
b8ff05a9
DM		 1113}
1114
1115static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1116 unsigned int end)
1117{
1118 u32 *p = buf + start;
1119
1120 for ( ; start <= end; start += sizeof(u32))
1121 *p++ = t4_read_reg(ap, start);
1122}
1123
1124static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1125 void *buf)
1126{
1127 static const unsigned int reg_ranges[] = {
1128 0x1008, 0x1108,
1129 0x1180, 0x11b4,
1130 0x11fc, 0x123c,
1131 0x1300, 0x173c,
1132 0x1800, 0x18fc,
1133 0x3000, 0x30d8,
1134 0x30e0, 0x5924,
1135 0x5960, 0x59d4,
1136 0x5a00, 0x5af8,
1137 0x6000, 0x6098,
1138 0x6100, 0x6150,
1139 0x6200, 0x6208,
1140 0x6240, 0x6248,
1141 0x6280, 0x6338,
1142 0x6370, 0x638c,
1143 0x6400, 0x643c,
1144 0x6500, 0x6524,
1145 0x6a00, 0x6a38,
1146 0x6a60, 0x6a78,
1147 0x6b00, 0x6b84,
1148 0x6bf0, 0x6c84,
1149 0x6cf0, 0x6d84,
1150 0x6df0, 0x6e84,
1151 0x6ef0, 0x6f84,
1152 0x6ff0, 0x7084,
1153 0x70f0, 0x7184,
1154 0x71f0, 0x7284,
1155 0x72f0, 0x7384,
1156 0x73f0, 0x7450,
1157 0x7500, 0x7530,
1158 0x7600, 0x761c,
1159 0x7680, 0x76cc,
1160 0x7700, 0x7798,
1161 0x77c0, 0x77fc,
1162 0x7900, 0x79fc,
1163 0x7b00, 0x7c38,
1164 0x7d00, 0x7efc,
1165 0x8dc0, 0x8e1c,
1166 0x8e30, 0x8e78,
1167 0x8ea0, 0x8f6c,
1168 0x8fc0, 0x9074,
1169 0x90fc, 0x90fc,
1170 0x9400, 0x9458,
1171 0x9600, 0x96bc,
1172 0x9800, 0x9808,
1173 0x9820, 0x983c,
1174 0x9850, 0x9864,
1175 0x9c00, 0x9c6c,
1176 0x9c80, 0x9cec,
1177 0x9d00, 0x9d6c,
1178 0x9d80, 0x9dec,
1179 0x9e00, 0x9e6c,
1180 0x9e80, 0x9eec,
1181 0x9f00, 0x9f6c,
1182 0x9f80, 0x9fec,
1183 0xd004, 0xd03c,
1184 0xdfc0, 0xdfe0,
1185 0xe000, 0xea7c,
1186 0xf000, 0x11190,
835bb606
DM		 1187 0x19040, 0x1906c,
1188 0x19078, 0x19080,
1189 0x1908c, 0x19124,
b8ff05a9
DM		 1190 0x19150, 0x191b0,
1191 0x191d0, 0x191e8,
1192 0x19238, 0x1924c,
1193 0x193f8, 0x19474,
1194 0x19490, 0x194f8,
1195 0x19800, 0x19f30,
1196 0x1a000, 0x1a06c,
1197 0x1a0b0, 0x1a120,
1198 0x1a128, 0x1a138,
1199 0x1a190, 0x1a1c4,
1200 0x1a1fc, 0x1a1fc,
1201 0x1e040, 0x1e04c,
835bb606 1202 0x1e284, 0x1e28c,
b8ff05a9
DM		 1203 0x1e2c0, 0x1e2c0,
1204 0x1e2e0, 0x1e2e0,
1205 0x1e300, 0x1e384,
1206 0x1e3c0, 0x1e3c8,
1207 0x1e440, 0x1e44c,
835bb606 1208 0x1e684, 0x1e68c,
b8ff05a9
DM		 1209 0x1e6c0, 0x1e6c0,
1210 0x1e6e0, 0x1e6e0,
1211 0x1e700, 0x1e784,
1212 0x1e7c0, 0x1e7c8,
1213 0x1e840, 0x1e84c,
835bb606 1214 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1215 0x1eac0, 0x1eac0,
1216 0x1eae0, 0x1eae0,
1217 0x1eb00, 0x1eb84,
1218 0x1ebc0, 0x1ebc8,
1219 0x1ec40, 0x1ec4c,
835bb606 1220 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1221 0x1eec0, 0x1eec0,
1222 0x1eee0, 0x1eee0,
1223 0x1ef00, 0x1ef84,
1224 0x1efc0, 0x1efc8,
1225 0x1f040, 0x1f04c,
835bb606 1226 0x1f284, 0x1f28c,
b8ff05a9
DM		 1227 0x1f2c0, 0x1f2c0,
1228 0x1f2e0, 0x1f2e0,
1229 0x1f300, 0x1f384,
1230 0x1f3c0, 0x1f3c8,
1231 0x1f440, 0x1f44c,
835bb606 1232 0x1f684, 0x1f68c,
b8ff05a9
DM		 1233 0x1f6c0, 0x1f6c0,
1234 0x1f6e0, 0x1f6e0,
1235 0x1f700, 0x1f784,
1236 0x1f7c0, 0x1f7c8,
1237 0x1f840, 0x1f84c,
835bb606 1238 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1239 0x1fac0, 0x1fac0,
1240 0x1fae0, 0x1fae0,
1241 0x1fb00, 0x1fb84,
1242 0x1fbc0, 0x1fbc8,
1243 0x1fc40, 0x1fc4c,
835bb606 1244 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1245 0x1fec0, 0x1fec0,
1246 0x1fee0, 0x1fee0,
1247 0x1ff00, 0x1ff84,
1248 0x1ffc0, 0x1ffc8,
1249 0x20000, 0x2002c,
1250 0x20100, 0x2013c,
1251 0x20190, 0x201c8,
1252 0x20200, 0x20318,
1253 0x20400, 0x20528,
1254 0x20540, 0x20614,
1255 0x21000, 0x21040,
1256 0x2104c, 0x21060,
1257 0x210c0, 0x210ec,
1258 0x21200, 0x21268,
1259 0x21270, 0x21284,
1260 0x212fc, 0x21388,
1261 0x21400, 0x21404,
1262 0x21500, 0x21518,
1263 0x2152c, 0x2153c,
1264 0x21550, 0x21554,
1265 0x21600, 0x21600,
1266 0x21608, 0x21628,
1267 0x21630, 0x2163c,
1268 0x21700, 0x2171c,
1269 0x21780, 0x2178c,
1270 0x21800, 0x21c38,
1271 0x21c80, 0x21d7c,
1272 0x21e00, 0x21e04,
1273 0x22000, 0x2202c,
1274 0x22100, 0x2213c,
1275 0x22190, 0x221c8,
1276 0x22200, 0x22318,
1277 0x22400, 0x22528,
1278 0x22540, 0x22614,
1279 0x23000, 0x23040,
1280 0x2304c, 0x23060,
1281 0x230c0, 0x230ec,
1282 0x23200, 0x23268,
1283 0x23270, 0x23284,
1284 0x232fc, 0x23388,
1285 0x23400, 0x23404,
1286 0x23500, 0x23518,
1287 0x2352c, 0x2353c,
1288 0x23550, 0x23554,
1289 0x23600, 0x23600,
1290 0x23608, 0x23628,
1291 0x23630, 0x2363c,
1292 0x23700, 0x2371c,
1293 0x23780, 0x2378c,
1294 0x23800, 0x23c38,
1295 0x23c80, 0x23d7c,
1296 0x23e00, 0x23e04,
1297 0x24000, 0x2402c,
1298 0x24100, 0x2413c,
1299 0x24190, 0x241c8,
1300 0x24200, 0x24318,
1301 0x24400, 0x24528,
1302 0x24540, 0x24614,
1303 0x25000, 0x25040,
1304 0x2504c, 0x25060,
1305 0x250c0, 0x250ec,
1306 0x25200, 0x25268,
1307 0x25270, 0x25284,
1308 0x252fc, 0x25388,
1309 0x25400, 0x25404,
1310 0x25500, 0x25518,
1311 0x2552c, 0x2553c,
1312 0x25550, 0x25554,
1313 0x25600, 0x25600,
1314 0x25608, 0x25628,
1315 0x25630, 0x2563c,
1316 0x25700, 0x2571c,
1317 0x25780, 0x2578c,
1318 0x25800, 0x25c38,
1319 0x25c80, 0x25d7c,
1320 0x25e00, 0x25e04,
1321 0x26000, 0x2602c,
1322 0x26100, 0x2613c,
1323 0x26190, 0x261c8,
1324 0x26200, 0x26318,
1325 0x26400, 0x26528,
1326 0x26540, 0x26614,
1327 0x27000, 0x27040,
1328 0x2704c, 0x27060,
1329 0x270c0, 0x270ec,
1330 0x27200, 0x27268,
1331 0x27270, 0x27284,
1332 0x272fc, 0x27388,
1333 0x27400, 0x27404,
1334 0x27500, 0x27518,
1335 0x2752c, 0x2753c,
1336 0x27550, 0x27554,
1337 0x27600, 0x27600,
1338 0x27608, 0x27628,
1339 0x27630, 0x2763c,
1340 0x27700, 0x2771c,
1341 0x27780, 0x2778c,
1342 0x27800, 0x27c38,
1343 0x27c80, 0x27d7c,
1344 0x27e00, 0x27e04
1345 };
1346
1347 int i;
1348 struct adapter *ap = netdev2adap(dev);
1349
1350 regs->version = mk_adap_vers(ap);
1351
1352 memset(buf, 0, T4_REGMAP_SIZE);
1353 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1354 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1355}
1356
1357static int restart_autoneg(struct net_device *dev)
1358{
1359 struct port_info *p = netdev_priv(dev);
1360
1361 if (!netif_running(dev))
1362 return -EAGAIN;
1363 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1364 return -EINVAL;
060e0c75 1365 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 1366 return 0;
1367}
1368
c5e06360
DM		 1369static int identify_port(struct net_device *dev,
1370 enum ethtool_phys_id_state state)
b8ff05a9 1371{
c5e06360 1372 unsigned int val;
060e0c75
DM		 1373 struct adapter *adap = netdev2adap(dev);
1374
c5e06360
DM		 1375 if (state == ETHTOOL_ID_ACTIVE)
1376 val = 0xffff;
1377 else if (state == ETHTOOL_ID_INACTIVE)
1378 val = 0;
1379 else
1380 return -EINVAL;
b8ff05a9 1381
c5e06360 1382 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 1383}
1384
1385static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1386{
1387 unsigned int v = 0;
1388
a0881cab
DM		 1389 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1390 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 1391 v |= SUPPORTED_TP;
1392 if (caps & FW_PORT_CAP_SPEED_100M)
1393 v |= SUPPORTED_100baseT_Full;
1394 if (caps & FW_PORT_CAP_SPEED_1G)
1395 v |= SUPPORTED_1000baseT_Full;
1396 if (caps & FW_PORT_CAP_SPEED_10G)
1397 v |= SUPPORTED_10000baseT_Full;
1398 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1399 v |= SUPPORTED_Backplane;
1400 if (caps & FW_PORT_CAP_SPEED_1G)
1401 v |= SUPPORTED_1000baseKX_Full;
1402 if (caps & FW_PORT_CAP_SPEED_10G)
1403 v |= SUPPORTED_10000baseKX4_Full;
1404 } else if (type == FW_PORT_TYPE_KR)
1405 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 1406 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 1407 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1408 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1409 else if (type == FW_PORT_TYPE_BP4_AP)
1410 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1411 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1412 SUPPORTED_10000baseKX4_Full;
a0881cab
DM		 1413 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1414 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9
DM		 1415 v |= SUPPORTED_FIBRE;
1416
1417 if (caps & FW_PORT_CAP_ANEG)
1418 v |= SUPPORTED_Autoneg;
1419 return v;
1420}
1421
1422static unsigned int to_fw_linkcaps(unsigned int caps)
1423{
1424 unsigned int v = 0;
1425
1426 if (caps & ADVERTISED_100baseT_Full)
1427 v |= FW_PORT_CAP_SPEED_100M;
1428 if (caps & ADVERTISED_1000baseT_Full)
1429 v |= FW_PORT_CAP_SPEED_1G;
1430 if (caps & ADVERTISED_10000baseT_Full)
1431 v |= FW_PORT_CAP_SPEED_10G;
1432 return v;
1433}
1434
1435static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1436{
1437 const struct port_info *p = netdev_priv(dev);
1438
1439 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 1440 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 1441 p->port_type == FW_PORT_TYPE_BT_XAUI)
1442 cmd->port = PORT_TP;
a0881cab
DM		 1443 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1444 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 1445 cmd->port = PORT_FIBRE;
a0881cab
DM		 1446 else if (p->port_type == FW_PORT_TYPE_SFP) {
1447 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1448 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1449 cmd->port = PORT_DA;
1450 else
1451 cmd->port = PORT_FIBRE;
1452 } else
b8ff05a9
DM		 1453 cmd->port = PORT_OTHER;
1454
1455 if (p->mdio_addr >= 0) {
1456 cmd->phy_address = p->mdio_addr;
1457 cmd->transceiver = XCVR_EXTERNAL;
1458 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1459 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1460 } else {
1461 cmd->phy_address = 0; /* not really, but no better option */
1462 cmd->transceiver = XCVR_INTERNAL;
1463 cmd->mdio_support = 0;
1464 }
1465
1466 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1467 cmd->advertising = from_fw_linkcaps(p->port_type,
1468 p->link_cfg.advertising);
70739497
DD		 1469 ethtool_cmd_speed_set(cmd,
1470 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 1471 cmd->duplex = DUPLEX_FULL;
1472 cmd->autoneg = p->link_cfg.autoneg;
1473 cmd->maxtxpkt = 0;
1474 cmd->maxrxpkt = 0;
1475 return 0;
1476}
1477
1478static unsigned int speed_to_caps(int speed)
1479{
1480 if (speed == SPEED_100)
1481 return FW_PORT_CAP_SPEED_100M;
1482 if (speed == SPEED_1000)
1483 return FW_PORT_CAP_SPEED_1G;
1484 if (speed == SPEED_10000)
1485 return FW_PORT_CAP_SPEED_10G;
1486 return 0;
1487}
1488
1489static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1490{
1491 unsigned int cap;
1492 struct port_info *p = netdev_priv(dev);
1493 struct link_config *lc = &p->link_cfg;
25db0338 1494 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 1495
1496 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1497 return -EINVAL;
1498
1499 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1500 /*
1501 * PHY offers a single speed. See if that's what's
1502 * being requested.
1503 */
1504 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 1505 (lc->supported & speed_to_caps(speed)))
1506 return 0;
b8ff05a9
DM		 1507 return -EINVAL;
1508 }
1509
1510 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 1511 cap = speed_to_caps(speed);
b8ff05a9 1512
25db0338
DD		 1513 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1514 (speed == SPEED_10000))
b8ff05a9
DM		 1515 return -EINVAL;
1516 lc->requested_speed = cap;
1517 lc->advertising = 0;
1518 } else {
1519 cap = to_fw_linkcaps(cmd->advertising);
1520 if (!(lc->supported & cap))
1521 return -EINVAL;
1522 lc->requested_speed = 0;
1523 lc->advertising = cap | FW_PORT_CAP_ANEG;
1524 }
1525 lc->autoneg = cmd->autoneg;
1526
1527 if (netif_running(dev))
060e0c75
DM		 1528 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1529 lc);
b8ff05a9
DM		 1530 return 0;
1531}
1532
1533static void get_pauseparam(struct net_device *dev,
1534 struct ethtool_pauseparam *epause)
1535{
1536 struct port_info *p = netdev_priv(dev);
1537
1538 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1539 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1540 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1541}
1542
1543static int set_pauseparam(struct net_device *dev,
1544 struct ethtool_pauseparam *epause)
1545{
1546 struct port_info *p = netdev_priv(dev);
1547 struct link_config *lc = &p->link_cfg;
1548
1549 if (epause->autoneg == AUTONEG_DISABLE)
1550 lc->requested_fc = 0;
1551 else if (lc->supported & FW_PORT_CAP_ANEG)
1552 lc->requested_fc = PAUSE_AUTONEG;
1553 else
1554 return -EINVAL;
1555
1556 if (epause->rx_pause)
1557 lc->requested_fc |= PAUSE_RX;
1558 if (epause->tx_pause)
1559 lc->requested_fc |= PAUSE_TX;
1560 if (netif_running(dev))
060e0c75
DM		 1561 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1562 lc);
b8ff05a9
DM		 1563 return 0;
1564}
1565
b8ff05a9
DM		 1566static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1567{
1568 const struct port_info *pi = netdev_priv(dev);
1569 const struct sge *s = &pi->adapter->sge;
1570
1571 e->rx_max_pending = MAX_RX_BUFFERS;
1572 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1573 e->rx_jumbo_max_pending = 0;
1574 e->tx_max_pending = MAX_TXQ_ENTRIES;
1575
1576 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1577 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1578 e->rx_jumbo_pending = 0;
1579 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1580}
1581
1582static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1583{
1584 int i;
1585 const struct port_info *pi = netdev_priv(dev);
1586 struct adapter *adapter = pi->adapter;
1587 struct sge *s = &adapter->sge;
1588
1589 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1590 e->tx_pending > MAX_TXQ_ENTRIES ||
1591 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1592 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1593 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1594 return -EINVAL;
1595
1596 if (adapter->flags & FULL_INIT_DONE)
1597 return -EBUSY;
1598
1599 for (i = 0; i < pi->nqsets; ++i) {
1600 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1601 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1602 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1603 }
1604 return 0;
1605}
1606
1607static int closest_timer(const struct sge *s, int time)
1608{
1609 int i, delta, match = 0, min_delta = INT_MAX;
1610
1611 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1612 delta = time - s->timer_val[i];
1613 if (delta < 0)
1614 delta = -delta;
1615 if (delta < min_delta) {
1616 min_delta = delta;
1617 match = i;
1618 }
1619 }
1620 return match;
1621}
1622
1623static int closest_thres(const struct sge *s, int thres)
1624{
1625 int i, delta, match = 0, min_delta = INT_MAX;
1626
1627 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1628 delta = thres - s->counter_val[i];
1629 if (delta < 0)
1630 delta = -delta;
1631 if (delta < min_delta) {
1632 min_delta = delta;
1633 match = i;
1634 }
1635 }
1636 return match;
1637}
1638
1639/*
1640 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1641 */
1642static unsigned int qtimer_val(const struct adapter *adap,
1643 const struct sge_rspq *q)
1644{
1645 unsigned int idx = q->intr_params >> 1;
1646
1647 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1648}
1649
1650/**
1651 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1652 * @adap: the adapter
1653 * @q: the Rx queue
1654 * @us: the hold-off time in us, or 0 to disable timer
1655 * @cnt: the hold-off packet count, or 0 to disable counter
1656 *
1657 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1658 * one of the two needs to be enabled for the queue to generate interrupts.
1659 */
1660static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1661 unsigned int us, unsigned int cnt)
1662{
1663 if ((us | cnt) == 0)
1664 cnt = 1;
1665
1666 if (cnt) {
1667 int err;
1668 u32 v, new_idx;
1669
1670 new_idx = closest_thres(&adap->sge, cnt);
1671 if (q->desc && q->pktcnt_idx != new_idx) {
1672 /* the queue has already been created, update it */
1673 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1674 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1675 FW_PARAMS_PARAM_YZ(q->cntxt_id);
060e0c75
DM		 1676 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1677 &new_idx);
b8ff05a9
DM		 1678 if (err)
1679 return err;
1680 }
1681 q->pktcnt_idx = new_idx;
1682 }
1683
1684 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1685 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1686 return 0;
1687}
1688
1689static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1690{
1691 const struct port_info *pi = netdev_priv(dev);
1692 struct adapter *adap = pi->adapter;
1693
1694 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1695 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1696}
1697
1698static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1699{
1700 const struct port_info *pi = netdev_priv(dev);
1701 const struct adapter *adap = pi->adapter;
1702 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1703
1704 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1705 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1706 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1707 return 0;
1708}
1709
1478b3ee
DM		 1710/**
1711 * eeprom_ptov - translate a physical EEPROM address to virtual
1712 * @phys_addr: the physical EEPROM address
1713 * @fn: the PCI function number
1714 * @sz: size of function-specific area
1715 *
1716 * Translate a physical EEPROM address to virtual. The first 1K is
1717 * accessed through virtual addresses starting at 31K, the rest is
1718 * accessed through virtual addresses starting at 0.
1719 *
1720 * The mapping is as follows:
1721 * [0..1K) -> [31K..32K)
1722 * [1K..1K+A) -> [31K-A..31K)
1723 * [1K+A..ES) -> [0..ES-A-1K)
1724 *
1725 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 1726 */
1478b3ee 1727static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 1728{
1478b3ee 1729 fn *= sz;
b8ff05a9
DM		 1730 if (phys_addr < 1024)
1731 return phys_addr + (31 << 10);
1478b3ee
DM		 1732 if (phys_addr < 1024 + fn)
1733 return 31744 - fn + phys_addr - 1024;
b8ff05a9 1734 if (phys_addr < EEPROMSIZE)
1478b3ee 1735 return phys_addr - 1024 - fn;
b8ff05a9
DM		 1736 return -EINVAL;
1737}
1738
1739/*
1740 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 1741 */
1742static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1743{
1478b3ee 1744 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 1745
1746 if (vaddr >= 0)
1747 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1748 return vaddr < 0 ? vaddr : 0;
1749}
1750
1751static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1752{
1478b3ee 1753 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 1754
1755 if (vaddr >= 0)
1756 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1757 return vaddr < 0 ? vaddr : 0;
1758}
1759
1760#define EEPROM_MAGIC 0x38E2F10C
1761
1762static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1763 u8 *data)
1764{
1765 int i, err = 0;
1766 struct adapter *adapter = netdev2adap(dev);
1767
1768 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1769 if (!buf)
1770 return -ENOMEM;
1771
1772 e->magic = EEPROM_MAGIC;
1773 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1774 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1775
1776 if (!err)
1777 memcpy(data, buf + e->offset, e->len);
1778 kfree(buf);
1779 return err;
1780}
1781
1782static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1783 u8 *data)
1784{
1785 u8 *buf;
1786 int err = 0;
1787 u32 aligned_offset, aligned_len, *p;
1788 struct adapter *adapter = netdev2adap(dev);
1789
1790 if (eeprom->magic != EEPROM_MAGIC)
1791 return -EINVAL;
1792
1793 aligned_offset = eeprom->offset & ~3;
1794 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1795
1478b3ee
DM		 1796 if (adapter->fn > 0) {
1797 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1798
1799 if (aligned_offset < start ||
1800 aligned_offset + aligned_len > start + EEPROMPFSIZE)
1801 return -EPERM;
1802 }
1803
b8ff05a9
DM		 1804 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1805 /*
1806 * RMW possibly needed for first or last words.
1807 */
1808 buf = kmalloc(aligned_len, GFP_KERNEL);
1809 if (!buf)
1810 return -ENOMEM;
1811 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1812 if (!err && aligned_len > 4)
1813 err = eeprom_rd_phys(adapter,
1814 aligned_offset + aligned_len - 4,
1815 (u32 *)&buf[aligned_len - 4]);
1816 if (err)
1817 goto out;
1818 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1819 } else
1820 buf = data;
1821
1822 err = t4_seeprom_wp(adapter, false);
1823 if (err)
1824 goto out;
1825
1826 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1827 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1828 aligned_offset += 4;
1829 }
1830
1831 if (!err)
1832 err = t4_seeprom_wp(adapter, true);
1833out:
1834 if (buf != data)
1835 kfree(buf);
1836 return err;
1837}
1838
1839static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1840{
1841 int ret;
1842 const struct firmware *fw;
1843 struct adapter *adap = netdev2adap(netdev);
1844
1845 ef->data[sizeof(ef->data) - 1] = '\0';
1846 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1847 if (ret < 0)
1848 return ret;
1849
1850 ret = t4_load_fw(adap, fw->data, fw->size);
1851 release_firmware(fw);
1852 if (!ret)
1853 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1854 return ret;
1855}
1856
1857#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1858#define BCAST_CRC 0xa0ccc1a6
1859
1860static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1861{
1862 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1863 wol->wolopts = netdev2adap(dev)->wol;
1864 memset(&wol->sopass, 0, sizeof(wol->sopass));
1865}
1866
1867static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1868{
1869 int err = 0;
1870 struct port_info *pi = netdev_priv(dev);
1871
1872 if (wol->wolopts & ~WOL_SUPPORTED)
1873 return -EINVAL;
1874 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1875 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1876 if (wol->wolopts & WAKE_BCAST) {
1877 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1878 ~0ULL, 0, false);
1879 if (!err)
1880 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1881 ~6ULL, ~0ULL, BCAST_CRC, true);
1882 } else
1883 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1884 return err;
1885}
1886
c8f44aff 1887static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 1888{
2ed28baa 1889 const struct port_info *pi = netdev_priv(dev);
c8f44aff 1890 netdev_features_t changed = dev->features ^ features;
19ecae2c 1891 int err;
19ecae2c 1892
2ed28baa
MM		 1893 if (!(changed & NETIF_F_HW_VLAN_RX))
1894 return 0;
19ecae2c 1895
2ed28baa
MM		 1896 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1897 -1, -1, -1,
1898 !!(features & NETIF_F_HW_VLAN_RX), true);
1899 if (unlikely(err))
1900 dev->features = features ^ NETIF_F_HW_VLAN_RX;
19ecae2c 1901 return err;
87b6cf51
DM		 1902}
1903
7850f63f 1904static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 1905{
1906 const struct port_info *pi = netdev_priv(dev);
671b0060 1907
7850f63f
BH		 1908 return pi->rss_size;
1909}
1910
1911static int get_rss_table(struct net_device *dev, u32 *p)
1912{
1913 const struct port_info *pi = netdev_priv(dev);
1914 unsigned int n = pi->rss_size;
1915
671b0060 1916 while (n--)
7850f63f 1917 p[n] = pi->rss[n];
671b0060
DM		 1918 return 0;
1919}
1920
7850f63f 1921static int set_rss_table(struct net_device *dev, const u32 *p)
671b0060
DM		 1922{
1923 unsigned int i;
1924 struct port_info *pi = netdev_priv(dev);
1925
7850f63f
BH		 1926 for (i = 0; i < pi->rss_size; i++)
1927 pi->rss[i] = p[i];
671b0060
DM		 1928 if (pi->adapter->flags & FULL_INIT_DONE)
1929 return write_rss(pi, pi->rss);
1930 return 0;
1931}
1932
1933static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 1934 u32 *rules)
671b0060 1935{
f796564a
DM		 1936 const struct port_info *pi = netdev_priv(dev);
1937
671b0060 1938 switch (info->cmd) {
f796564a
DM		 1939 case ETHTOOL_GRXFH: {
1940 unsigned int v = pi->rss_mode;
1941
1942 info->data = 0;
1943 switch (info->flow_type) {
1944 case TCP_V4_FLOW:
1945 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1946 info->data = RXH_IP_SRC | RXH_IP_DST |
1947 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1948 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1949 info->data = RXH_IP_SRC | RXH_IP_DST;
1950 break;
1951 case UDP_V4_FLOW:
1952 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1953 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1954 info->data = RXH_IP_SRC | RXH_IP_DST |
1955 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1956 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1957 info->data = RXH_IP_SRC | RXH_IP_DST;
1958 break;
1959 case SCTP_V4_FLOW:
1960 case AH_ESP_V4_FLOW:
1961 case IPV4_FLOW:
1962 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1963 info->data = RXH_IP_SRC | RXH_IP_DST;
1964 break;
1965 case TCP_V6_FLOW:
1966 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1967 info->data = RXH_IP_SRC | RXH_IP_DST |
1968 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1969 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1970 info->data = RXH_IP_SRC | RXH_IP_DST;
1971 break;
1972 case UDP_V6_FLOW:
1973 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1974 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1975 info->data = RXH_IP_SRC | RXH_IP_DST |
1976 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1977 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1978 info->data = RXH_IP_SRC | RXH_IP_DST;
1979 break;
1980 case SCTP_V6_FLOW:
1981 case AH_ESP_V6_FLOW:
1982 case IPV6_FLOW:
1983 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1984 info->data = RXH_IP_SRC | RXH_IP_DST;
1985 break;
1986 }
1987 return 0;
1988 }
671b0060 1989 case ETHTOOL_GRXRINGS:
f796564a 1990 info->data = pi->nqsets;
671b0060
DM		 1991 return 0;
1992 }
1993 return -EOPNOTSUPP;
1994}
1995
9b07be4b 1996static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 1997 .get_settings = get_settings,
1998 .set_settings = set_settings,
1999 .get_drvinfo = get_drvinfo,
2000 .get_msglevel = get_msglevel,
2001 .set_msglevel = set_msglevel,
2002 .get_ringparam = get_sge_param,
2003 .set_ringparam = set_sge_param,
2004 .get_coalesce = get_coalesce,
2005 .set_coalesce = set_coalesce,
2006 .get_eeprom_len = get_eeprom_len,
2007 .get_eeprom = get_eeprom,
2008 .set_eeprom = set_eeprom,
2009 .get_pauseparam = get_pauseparam,
2010 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2011 .get_link = ethtool_op_get_link,
2012 .get_strings = get_strings,
c5e06360 2013 .set_phys_id = identify_port,
b8ff05a9
DM		 2014 .nway_reset = restart_autoneg,
2015 .get_sset_count = get_sset_count,
2016 .get_ethtool_stats = get_stats,
2017 .get_regs_len = get_regs_len,
2018 .get_regs = get_regs,
2019 .get_wol = get_wol,
2020 .set_wol = set_wol,
671b0060 2021 .get_rxnfc = get_rxnfc,
7850f63f 2022 .get_rxfh_indir_size = get_rss_table_size,
671b0060
DM		 2023 .get_rxfh_indir = get_rss_table,
2024 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 2025 .flash_device = set_flash,
2026};
2027
2028/*
2029 * debugfs support
2030 */
b8ff05a9
DM		 2031static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2032 loff_t *ppos)
2033{
2034 loff_t pos = *ppos;
2035 loff_t avail = file->f_path.dentry->d_inode->i_size;
2036 unsigned int mem = (uintptr_t)file->private_data & 3;
2037 struct adapter *adap = file->private_data - mem;
2038
2039 if (pos < 0)
2040 return -EINVAL;
2041 if (pos >= avail)
2042 return 0;
2043 if (count > avail - pos)
2044 count = avail - pos;
2045
2046 while (count) {
2047 size_t len;
2048 int ret, ofst;
2049 __be32 data[16];
2050
2051 if (mem == MEM_MC)
2052 ret = t4_mc_read(adap, pos, data, NULL);
2053 else
2054 ret = t4_edc_read(adap, mem, pos, data, NULL);
2055 if (ret)
2056 return ret;
2057
2058 ofst = pos % sizeof(data);
2059 len = min(count, sizeof(data) - ofst);
2060 if (copy_to_user(buf, (u8 *)data + ofst, len))
2061 return -EFAULT;
2062
2063 buf += len;
2064 pos += len;
2065 count -= len;
2066 }
2067 count = pos - *ppos;
2068 *ppos = pos;
2069 return count;
2070}
2071
2072static const struct file_operations mem_debugfs_fops = {
2073 .owner = THIS_MODULE,
234e3405 2074 .open = simple_open,
b8ff05a9 2075 .read = mem_read,
6038f373 2076 .llseek = default_llseek,
b8ff05a9
DM		 2077};
2078
2079static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2080 unsigned int idx, unsigned int size_mb)
2081{
2082 struct dentry *de;
2083
2084 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2085 (void *)adap + idx, &mem_debugfs_fops);
2086 if (de && de->d_inode)
2087 de->d_inode->i_size = size_mb << 20;
2088}
2089
2090static int __devinit setup_debugfs(struct adapter *adap)
2091{
2092 int i;
2093
2094 if (IS_ERR_OR_NULL(adap->debugfs_root))
2095 return -1;
2096
2097 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2098 if (i & EDRAM0_ENABLE)
2099 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2100 if (i & EDRAM1_ENABLE)
2101 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2102 if (i & EXT_MEM_ENABLE)
2103 add_debugfs_mem(adap, "mc", MEM_MC,
2104 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2105 if (adap->l2t)
2106 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2107 &t4_l2t_fops);
2108 return 0;
2109}
2110
2111/*
2112 * upper-layer driver support
2113 */
2114
2115/*
2116 * Allocate an active-open TID and set it to the supplied value.
2117 */
2118int cxgb4_alloc_atid(struct tid_info *t, void *data)
2119{
2120 int atid = -1;
2121
2122 spin_lock_bh(&t->atid_lock);
2123 if (t->afree) {
2124 union aopen_entry *p = t->afree;
2125
2126 atid = p - t->atid_tab;
2127 t->afree = p->next;
2128 p->data = data;
2129 t->atids_in_use++;
2130 }
2131 spin_unlock_bh(&t->atid_lock);
2132 return atid;
2133}
2134EXPORT_SYMBOL(cxgb4_alloc_atid);
2135
2136/*
2137 * Release an active-open TID.
2138 */
2139void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2140{
2141 union aopen_entry *p = &t->atid_tab[atid];
2142
2143 spin_lock_bh(&t->atid_lock);
2144 p->next = t->afree;
2145 t->afree = p;
2146 t->atids_in_use--;
2147 spin_unlock_bh(&t->atid_lock);
2148}
2149EXPORT_SYMBOL(cxgb4_free_atid);
2150
2151/*
2152 * Allocate a server TID and set it to the supplied value.
2153 */
2154int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2155{
2156 int stid;
2157
2158 spin_lock_bh(&t->stid_lock);
2159 if (family == PF_INET) {
2160 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2161 if (stid < t->nstids)
2162 __set_bit(stid, t->stid_bmap);
2163 else
2164 stid = -1;
2165 } else {
2166 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2167 if (stid < 0)
2168 stid = -1;
2169 }
2170 if (stid >= 0) {
2171 t->stid_tab[stid].data = data;
2172 stid += t->stid_base;
2173 t->stids_in_use++;
2174 }
2175 spin_unlock_bh(&t->stid_lock);
2176 return stid;
2177}
2178EXPORT_SYMBOL(cxgb4_alloc_stid);
2179
2180/*
2181 * Release a server TID.
2182 */
2183void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2184{
2185 stid -= t->stid_base;
2186 spin_lock_bh(&t->stid_lock);
2187 if (family == PF_INET)
2188 __clear_bit(stid, t->stid_bmap);
2189 else
2190 bitmap_release_region(t->stid_bmap, stid, 2);
2191 t->stid_tab[stid].data = NULL;
2192 t->stids_in_use--;
2193 spin_unlock_bh(&t->stid_lock);
2194}
2195EXPORT_SYMBOL(cxgb4_free_stid);
2196
2197/*
2198 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2199 */
2200static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2201 unsigned int tid)
2202{
2203 struct cpl_tid_release *req;
2204
2205 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2206 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2207 INIT_TP_WR(req, tid);
2208 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2209}
2210
2211/*
2212 * Queue a TID release request and if necessary schedule a work queue to
2213 * process it.
2214 */
31b9c19b 2215static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2216 unsigned int tid)
b8ff05a9
DM		 2217{
2218 void **p = &t->tid_tab[tid];
2219 struct adapter *adap = container_of(t, struct adapter, tids);
2220
2221 spin_lock_bh(&adap->tid_release_lock);
2222 *p = adap->tid_release_head;
2223 /* Low 2 bits encode the Tx channel number */
2224 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2225 if (!adap->tid_release_task_busy) {
2226 adap->tid_release_task_busy = true;
3069ee9b 2227 queue_work(workq, &adap->tid_release_task);
b8ff05a9
DM		 2228 }
2229 spin_unlock_bh(&adap->tid_release_lock);
2230}
b8ff05a9
DM		 2231
2232/*
2233 * Process the list of pending TID release requests.
2234 */
2235static void process_tid_release_list(struct work_struct *work)
2236{
2237 struct sk_buff *skb;
2238 struct adapter *adap;
2239
2240 adap = container_of(work, struct adapter, tid_release_task);
2241
2242 spin_lock_bh(&adap->tid_release_lock);
2243 while (adap->tid_release_head) {
2244 void **p = adap->tid_release_head;
2245 unsigned int chan = (uintptr_t)p & 3;
2246 p = (void *)p - chan;
2247
2248 adap->tid_release_head = *p;
2249 *p = NULL;
2250 spin_unlock_bh(&adap->tid_release_lock);
2251
2252 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2253 GFP_KERNEL)))
2254 schedule_timeout_uninterruptible(1);
2255
2256 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2257 t4_ofld_send(adap, skb);
2258 spin_lock_bh(&adap->tid_release_lock);
2259 }
2260 adap->tid_release_task_busy = false;
2261 spin_unlock_bh(&adap->tid_release_lock);
2262}
2263
2264/*
2265 * Release a TID and inform HW. If we are unable to allocate the release
2266 * message we defer to a work queue.
2267 */
2268void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2269{
2270 void *old;
2271 struct sk_buff *skb;
2272 struct adapter *adap = container_of(t, struct adapter, tids);
2273
2274 old = t->tid_tab[tid];
2275 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2276 if (likely(skb)) {
2277 t->tid_tab[tid] = NULL;
2278 mk_tid_release(skb, chan, tid);
2279 t4_ofld_send(adap, skb);
2280 } else
2281 cxgb4_queue_tid_release(t, chan, tid);
2282 if (old)
2283 atomic_dec(&t->tids_in_use);
2284}
2285EXPORT_SYMBOL(cxgb4_remove_tid);
2286
2287/*
2288 * Allocate and initialize the TID tables. Returns 0 on success.
2289 */
2290static int tid_init(struct tid_info *t)
2291{
2292 size_t size;
2293 unsigned int natids = t->natids;
2294
2295 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2296 t->nstids * sizeof(*t->stid_tab) +
2297 BITS_TO_LONGS(t->nstids) * sizeof(long);
2298 t->tid_tab = t4_alloc_mem(size);
2299 if (!t->tid_tab)
2300 return -ENOMEM;
2301
2302 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2303 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2304 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2305 spin_lock_init(&t->stid_lock);
2306 spin_lock_init(&t->atid_lock);
2307
2308 t->stids_in_use = 0;
2309 t->afree = NULL;
2310 t->atids_in_use = 0;
2311 atomic_set(&t->tids_in_use, 0);
2312
2313 /* Setup the free list for atid_tab and clear the stid bitmap. */
2314 if (natids) {
2315 while (--natids)
2316 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2317 t->afree = t->atid_tab;
2318 }
2319 bitmap_zero(t->stid_bmap, t->nstids);
2320 return 0;
2321}
2322
2323/**
2324 * cxgb4_create_server - create an IP server
2325 * @dev: the device
2326 * @stid: the server TID
2327 * @sip: local IP address to bind server to
2328 * @sport: the server's TCP port
2329 * @queue: queue to direct messages from this server to
2330 *
2331 * Create an IP server for the given port and address.
2332 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2333 */
2334int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2335 __be32 sip, __be16 sport, unsigned int queue)
2336{
2337 unsigned int chan;
2338 struct sk_buff *skb;
2339 struct adapter *adap;
2340 struct cpl_pass_open_req *req;
2341
2342 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2343 if (!skb)
2344 return -ENOMEM;
2345
2346 adap = netdev2adap(dev);
2347 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2348 INIT_TP_WR(req, 0);
2349 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2350 req->local_port = sport;
2351 req->peer_port = htons(0);
2352 req->local_ip = sip;
2353 req->peer_ip = htonl(0);
e46dab4d 2354 chan = rxq_to_chan(&adap->sge, queue);
b8ff05a9
DM		 2355 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2356 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2357 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2358 return t4_mgmt_tx(adap, skb);
2359}
2360EXPORT_SYMBOL(cxgb4_create_server);
2361
b8ff05a9
DM		 2362/**
2363 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2364 * @mtus: the HW MTU table
2365 * @mtu: the target MTU
2366 * @idx: index of selected entry in the MTU table
2367 *
2368 * Returns the index and the value in the HW MTU table that is closest to
2369 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2370 * table, in which case that smallest available value is selected.
2371 */
2372unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2373 unsigned int *idx)
2374{
2375 unsigned int i = 0;
2376
2377 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2378 ++i;
2379 if (idx)
2380 *idx = i;
2381 return mtus[i];
2382}
2383EXPORT_SYMBOL(cxgb4_best_mtu);
2384
2385/**
2386 * cxgb4_port_chan - get the HW channel of a port
2387 * @dev: the net device for the port
2388 *
2389 * Return the HW Tx channel of the given port.
2390 */
2391unsigned int cxgb4_port_chan(const struct net_device *dev)
2392{
2393 return netdev2pinfo(dev)->tx_chan;
2394}
2395EXPORT_SYMBOL(cxgb4_port_chan);
2396
881806bc
VP		 2397unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
2398{
2399 struct adapter *adap = netdev2adap(dev);
2400 u32 v;
2401
2402 v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2403 return lpfifo ? G_LP_COUNT(v) : G_HP_COUNT(v);
2404}
2405EXPORT_SYMBOL(cxgb4_dbfifo_count);
2406
b8ff05a9
DM		 2407/**
2408 * cxgb4_port_viid - get the VI id of a port
2409 * @dev: the net device for the port
2410 *
2411 * Return the VI id of the given port.
2412 */
2413unsigned int cxgb4_port_viid(const struct net_device *dev)
2414{
2415 return netdev2pinfo(dev)->viid;
2416}
2417EXPORT_SYMBOL(cxgb4_port_viid);
2418
2419/**
2420 * cxgb4_port_idx - get the index of a port
2421 * @dev: the net device for the port
2422 *
2423 * Return the index of the given port.
2424 */
2425unsigned int cxgb4_port_idx(const struct net_device *dev)
2426{
2427 return netdev2pinfo(dev)->port_id;
2428}
2429EXPORT_SYMBOL(cxgb4_port_idx);
2430
b8ff05a9
DM		 2431void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2432 struct tp_tcp_stats *v6)
2433{
2434 struct adapter *adap = pci_get_drvdata(pdev);
2435
2436 spin_lock(&adap->stats_lock);
2437 t4_tp_get_tcp_stats(adap, v4, v6);
2438 spin_unlock(&adap->stats_lock);
2439}
2440EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2441
2442void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2443 const unsigned int *pgsz_order)
2444{
2445 struct adapter *adap = netdev2adap(dev);
2446
2447 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2448 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2449 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2450 HPZ3(pgsz_order[3]));
2451}
2452EXPORT_SYMBOL(cxgb4_iscsi_init);
2453
3069ee9b
VP		 2454int cxgb4_flush_eq_cache(struct net_device *dev)
2455{
2456 struct adapter *adap = netdev2adap(dev);
2457 int ret;
2458
2459 ret = t4_fwaddrspace_write(adap, adap->mbox,
2460 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
2461 return ret;
2462}
2463EXPORT_SYMBOL(cxgb4_flush_eq_cache);
2464
2465static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
2466{
2467 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
2468 __be64 indices;
2469 int ret;
2470
2471 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
2472 if (!ret) {
2473 indices = be64_to_cpu(indices);
2474 *cidx = (indices >> 25) & 0xffff;
2475 *pidx = (indices >> 9) & 0xffff;
2476 }
2477 return ret;
2478}
2479
2480int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
2481 u16 size)
2482{
2483 struct adapter *adap = netdev2adap(dev);
2484 u16 hw_pidx, hw_cidx;
2485 int ret;
2486
2487 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
2488 if (ret)
2489 goto out;
2490
2491 if (pidx != hw_pidx) {
2492 u16 delta;
2493
2494 if (pidx >= hw_pidx)
2495 delta = pidx - hw_pidx;
2496 else
2497 delta = size - hw_pidx + pidx;
2498 wmb();
840f3000
VP		 2499 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
2500 QID(qid) | PIDX(delta));
3069ee9b
VP		 2501 }
2502out:
2503 return ret;
2504}
2505EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
2506
b8ff05a9
DM		 2507static struct pci_driver cxgb4_driver;
2508
2509static void check_neigh_update(struct neighbour *neigh)
2510{
2511 const struct device *parent;
2512 const struct net_device *netdev = neigh->dev;
2513
2514 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2515 netdev = vlan_dev_real_dev(netdev);
2516 parent = netdev->dev.parent;
2517 if (parent && parent->driver == &cxgb4_driver.driver)
2518 t4_l2t_update(dev_get_drvdata(parent), neigh);
2519}
2520
2521static int netevent_cb(struct notifier_block *nb, unsigned long event,
2522 void *data)
2523{
2524 switch (event) {
2525 case NETEVENT_NEIGH_UPDATE:
2526 check_neigh_update(data);
2527 break;
b8ff05a9
DM		 2528 case NETEVENT_REDIRECT:
2529 default:
2530 break;
2531 }
2532 return 0;
2533}
2534
2535static bool netevent_registered;
2536static struct notifier_block cxgb4_netevent_nb = {
2537 .notifier_call = netevent_cb
2538};
2539
3069ee9b
VP		 2540static void drain_db_fifo(struct adapter *adap, int usecs)
2541{
2542 u32 v;
2543
2544 do {
2545 set_current_state(TASK_UNINTERRUPTIBLE);
2546 schedule_timeout(usecs_to_jiffies(usecs));
2547 v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2548 if (G_LP_COUNT(v) == 0 && G_HP_COUNT(v) == 0)
2549 break;
2550 } while (1);
2551}
2552
2553static void disable_txq_db(struct sge_txq *q)
2554{
2555 spin_lock_irq(&q->db_lock);
2556 q->db_disabled = 1;
2557 spin_unlock_irq(&q->db_lock);
2558}
2559
2560static void enable_txq_db(struct sge_txq *q)
2561{
2562 spin_lock_irq(&q->db_lock);
2563 q->db_disabled = 0;
2564 spin_unlock_irq(&q->db_lock);
2565}
2566
2567static void disable_dbs(struct adapter *adap)
2568{
2569 int i;
2570
2571 for_each_ethrxq(&adap->sge, i)
2572 disable_txq_db(&adap->sge.ethtxq[i].q);
2573 for_each_ofldrxq(&adap->sge, i)
2574 disable_txq_db(&adap->sge.ofldtxq[i].q);
2575 for_each_port(adap, i)
2576 disable_txq_db(&adap->sge.ctrlq[i].q);
2577}
2578
2579static void enable_dbs(struct adapter *adap)
2580{
2581 int i;
2582
2583 for_each_ethrxq(&adap->sge, i)
2584 enable_txq_db(&adap->sge.ethtxq[i].q);
2585 for_each_ofldrxq(&adap->sge, i)
2586 enable_txq_db(&adap->sge.ofldtxq[i].q);
2587 for_each_port(adap, i)
2588 enable_txq_db(&adap->sge.ctrlq[i].q);
2589}
2590
2591static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
2592{
2593 u16 hw_pidx, hw_cidx;
2594 int ret;
2595
2596 spin_lock_bh(&q->db_lock);
2597 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
2598 if (ret)
2599 goto out;
2600 if (q->db_pidx != hw_pidx) {
2601 u16 delta;
2602
2603 if (q->db_pidx >= hw_pidx)
2604 delta = q->db_pidx - hw_pidx;
2605 else
2606 delta = q->size - hw_pidx + q->db_pidx;
2607 wmb();
840f3000
VP		 2608 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
2609 QID(q->cntxt_id) | PIDX(delta));
3069ee9b
VP		 2610 }
2611out:
2612 q->db_disabled = 0;
2613 spin_unlock_bh(&q->db_lock);
2614 if (ret)
2615 CH_WARN(adap, "DB drop recovery failed.\n");
2616}
2617static void recover_all_queues(struct adapter *adap)
2618{
2619 int i;
2620
2621 for_each_ethrxq(&adap->sge, i)
2622 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
2623 for_each_ofldrxq(&adap->sge, i)
2624 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
2625 for_each_port(adap, i)
2626 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
2627}
2628
881806bc
VP		 2629static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
2630{
2631 mutex_lock(&uld_mutex);
2632 if (adap->uld_handle[CXGB4_ULD_RDMA])
2633 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
2634 cmd);
2635 mutex_unlock(&uld_mutex);
2636}
2637
2638static void process_db_full(struct work_struct *work)
2639{
2640 struct adapter *adap;
881806bc
VP		 2641
2642 adap = container_of(work, struct adapter, db_full_task);
2643
881806bc 2644 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3069ee9b 2645 drain_db_fifo(adap, dbfifo_drain_delay);
840f3000
VP		 2646 t4_set_reg_field(adap, SGE_INT_ENABLE3,
2647 DBFIFO_HP_INT | DBFIFO_LP_INT,
2648 DBFIFO_HP_INT | DBFIFO_LP_INT);
881806bc 2649 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
881806bc
VP		 2650}
2651
2652static void process_db_drop(struct work_struct *work)
2653{
2654 struct adapter *adap;
881806bc 2655
3069ee9b 2656 adap = container_of(work, struct adapter, db_drop_task);
881806bc 2657
3069ee9b
VP		 2658 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
2659 disable_dbs(adap);
881806bc 2660 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3069ee9b
VP		 2661 drain_db_fifo(adap, 1);
2662 recover_all_queues(adap);
2663 enable_dbs(adap);
881806bc
VP		 2664}
2665
2666void t4_db_full(struct adapter *adap)
2667{
840f3000
VP		 2668 t4_set_reg_field(adap, SGE_INT_ENABLE3,
2669 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3069ee9b 2670 queue_work(workq, &adap->db_full_task);
881806bc
VP		 2671}
2672
2673void t4_db_dropped(struct adapter *adap)
2674{
3069ee9b 2675 queue_work(workq, &adap->db_drop_task);
881806bc
VP		 2676}
2677
b8ff05a9
DM		 2678static void uld_attach(struct adapter *adap, unsigned int uld)
2679{
2680 void *handle;
2681 struct cxgb4_lld_info lli;
2682
2683 lli.pdev = adap->pdev;
2684 lli.l2t = adap->l2t;
2685 lli.tids = &adap->tids;
2686 lli.ports = adap->port;
2687 lli.vr = &adap->vres;
2688 lli.mtus = adap->params.mtus;
2689 if (uld == CXGB4_ULD_RDMA) {
2690 lli.rxq_ids = adap->sge.rdma_rxq;
2691 lli.nrxq = adap->sge.rdmaqs;
2692 } else if (uld == CXGB4_ULD_ISCSI) {
2693 lli.rxq_ids = adap->sge.ofld_rxq;
2694 lli.nrxq = adap->sge.ofldqsets;
2695 }
2696 lli.ntxq = adap->sge.ofldqsets;
2697 lli.nchan = adap->params.nports;
2698 lli.nports = adap->params.nports;
2699 lli.wr_cred = adap->params.ofldq_wr_cred;
2700 lli.adapter_type = adap->params.rev;
2701 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2702 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 2703 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2704 (adap->fn * 4));
b8ff05a9 2705 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 2706 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2707 (adap->fn * 4));
b8ff05a9
DM		 2708 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2709 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2710 lli.fw_vers = adap->params.fw_vers;
3069ee9b 2711 lli.dbfifo_int_thresh = dbfifo_int_thresh;
b8ff05a9
DM		 2712
2713 handle = ulds[uld].add(&lli);
2714 if (IS_ERR(handle)) {
2715 dev_warn(adap->pdev_dev,
2716 "could not attach to the %s driver, error %ld\n",
2717 uld_str[uld], PTR_ERR(handle));
2718 return;
2719 }
2720
2721 adap->uld_handle[uld] = handle;
2722
2723 if (!netevent_registered) {
2724 register_netevent_notifier(&cxgb4_netevent_nb);
2725 netevent_registered = true;
2726 }
e29f5dbc
DM		 2727
2728 if (adap->flags & FULL_INIT_DONE)
2729 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 2730}
2731
2732static void attach_ulds(struct adapter *adap)
2733{
2734 unsigned int i;
2735
2736 mutex_lock(&uld_mutex);
2737 list_add_tail(&adap->list_node, &adapter_list);
2738 for (i = 0; i < CXGB4_ULD_MAX; i++)
2739 if (ulds[i].add)
2740 uld_attach(adap, i);
2741 mutex_unlock(&uld_mutex);
2742}
2743
2744static void detach_ulds(struct adapter *adap)
2745{
2746 unsigned int i;
2747
2748 mutex_lock(&uld_mutex);
2749 list_del(&adap->list_node);
2750 for (i = 0; i < CXGB4_ULD_MAX; i++)
2751 if (adap->uld_handle[i]) {
2752 ulds[i].state_change(adap->uld_handle[i],
2753 CXGB4_STATE_DETACH);
2754 adap->uld_handle[i] = NULL;
2755 }
2756 if (netevent_registered && list_empty(&adapter_list)) {
2757 unregister_netevent_notifier(&cxgb4_netevent_nb);
2758 netevent_registered = false;
2759 }
2760 mutex_unlock(&uld_mutex);
2761}
2762
2763static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2764{
2765 unsigned int i;
2766
2767 mutex_lock(&uld_mutex);
2768 for (i = 0; i < CXGB4_ULD_MAX; i++)
2769 if (adap->uld_handle[i])
2770 ulds[i].state_change(adap->uld_handle[i], new_state);
2771 mutex_unlock(&uld_mutex);
2772}
2773
2774/**
2775 * cxgb4_register_uld - register an upper-layer driver
2776 * @type: the ULD type
2777 * @p: the ULD methods
2778 *
2779 * Registers an upper-layer driver with this driver and notifies the ULD
2780 * about any presently available devices that support its type. Returns
2781 * %-EBUSY if a ULD of the same type is already registered.
2782 */
2783int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2784{
2785 int ret = 0;
2786 struct adapter *adap;
2787
2788 if (type >= CXGB4_ULD_MAX)
2789 return -EINVAL;
2790 mutex_lock(&uld_mutex);
2791 if (ulds[type].add) {
2792 ret = -EBUSY;
2793 goto out;
2794 }
2795 ulds[type] = *p;
2796 list_for_each_entry(adap, &adapter_list, list_node)
2797 uld_attach(adap, type);
2798out: mutex_unlock(&uld_mutex);
2799 return ret;
2800}
2801EXPORT_SYMBOL(cxgb4_register_uld);
2802
2803/**
2804 * cxgb4_unregister_uld - unregister an upper-layer driver
2805 * @type: the ULD type
2806 *
2807 * Unregisters an existing upper-layer driver.
2808 */
2809int cxgb4_unregister_uld(enum cxgb4_uld type)
2810{
2811 struct adapter *adap;
2812
2813 if (type >= CXGB4_ULD_MAX)
2814 return -EINVAL;
2815 mutex_lock(&uld_mutex);
2816 list_for_each_entry(adap, &adapter_list, list_node)
2817 adap->uld_handle[type] = NULL;
2818 ulds[type].add = NULL;
2819 mutex_unlock(&uld_mutex);
2820 return 0;
2821}
2822EXPORT_SYMBOL(cxgb4_unregister_uld);
2823
2824/**
2825 * cxgb_up - enable the adapter
2826 * @adap: adapter being enabled
2827 *
2828 * Called when the first port is enabled, this function performs the
2829 * actions necessary to make an adapter operational, such as completing
2830 * the initialization of HW modules, and enabling interrupts.
2831 *
2832 * Must be called with the rtnl lock held.
2833 */
2834static int cxgb_up(struct adapter *adap)
2835{
aaefae9b 2836 int err;
b8ff05a9 2837
aaefae9b
DM		 2838 err = setup_sge_queues(adap);
2839 if (err)
2840 goto out;
2841 err = setup_rss(adap);
2842 if (err)
2843 goto freeq;
b8ff05a9
DM		 2844
2845 if (adap->flags & USING_MSIX) {
aaefae9b 2846 name_msix_vecs(adap);
b8ff05a9
DM		 2847 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2848 adap->msix_info[0].desc, adap);
2849 if (err)
2850 goto irq_err;
2851
2852 err = request_msix_queue_irqs(adap);
2853 if (err) {
2854 free_irq(adap->msix_info[0].vec, adap);
2855 goto irq_err;
2856 }
2857 } else {
2858 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2859 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 2860 adap->port[0]->name, adap);
b8ff05a9
DM		 2861 if (err)
2862 goto irq_err;
2863 }
2864 enable_rx(adap);
2865 t4_sge_start(adap);
2866 t4_intr_enable(adap);
aaefae9b 2867 adap->flags |= FULL_INIT_DONE;
b8ff05a9
DM		 2868 notify_ulds(adap, CXGB4_STATE_UP);
2869 out:
2870 return err;
2871 irq_err:
2872 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 2873 freeq:
2874 t4_free_sge_resources(adap);
b8ff05a9
DM		 2875 goto out;
2876}
2877
2878static void cxgb_down(struct adapter *adapter)
2879{
2880 t4_intr_disable(adapter);
2881 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 2882 cancel_work_sync(&adapter->db_full_task);
2883 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 2884 adapter->tid_release_task_busy = false;
204dc3c0 2885 adapter->tid_release_head = NULL;
b8ff05a9
DM		 2886
2887 if (adapter->flags & USING_MSIX) {
2888 free_msix_queue_irqs(adapter);
2889 free_irq(adapter->msix_info[0].vec, adapter);
2890 } else
2891 free_irq(adapter->pdev->irq, adapter);
2892 quiesce_rx(adapter);
aaefae9b
DM		 2893 t4_sge_stop(adapter);
2894 t4_free_sge_resources(adapter);
2895 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 2896}
2897
2898/*
2899 * net_device operations
2900 */
2901static int cxgb_open(struct net_device *dev)
2902{
2903 int err;
2904 struct port_info *pi = netdev_priv(dev);
2905 struct adapter *adapter = pi->adapter;
2906
6a3c869a
DM		 2907 netif_carrier_off(dev);
2908
aaefae9b
DM		 2909 if (!(adapter->flags & FULL_INIT_DONE)) {
2910 err = cxgb_up(adapter);
2911 if (err < 0)
2912 return err;
2913 }
b8ff05a9 2914
f68707b8
DM		 2915 err = link_start(dev);
2916 if (!err)
2917 netif_tx_start_all_queues(dev);
2918 return err;
b8ff05a9
DM		 2919}
2920
2921static int cxgb_close(struct net_device *dev)
2922{
b8ff05a9
DM		 2923 struct port_info *pi = netdev_priv(dev);
2924 struct adapter *adapter = pi->adapter;
2925
2926 netif_tx_stop_all_queues(dev);
2927 netif_carrier_off(dev);
060e0c75 2928 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 2929}
2930
f5152c90
DM		 2931static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2932 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 2933{
2934 struct port_stats stats;
2935 struct port_info *p = netdev_priv(dev);
2936 struct adapter *adapter = p->adapter;
b8ff05a9
DM		 2937
2938 spin_lock(&adapter->stats_lock);
2939 t4_get_port_stats(adapter, p->tx_chan, &stats);
2940 spin_unlock(&adapter->stats_lock);
2941
2942 ns->tx_bytes = stats.tx_octets;
2943 ns->tx_packets = stats.tx_frames;
2944 ns->rx_bytes = stats.rx_octets;
2945 ns->rx_packets = stats.rx_frames;
2946 ns->multicast = stats.rx_mcast_frames;
2947
2948 /* detailed rx_errors */
2949 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2950 stats.rx_runt;
2951 ns->rx_over_errors = 0;
2952 ns->rx_crc_errors = stats.rx_fcs_err;
2953 ns->rx_frame_errors = stats.rx_symbol_err;
2954 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2955 stats.rx_ovflow2 + stats.rx_ovflow3 +
2956 stats.rx_trunc0 + stats.rx_trunc1 +
2957 stats.rx_trunc2 + stats.rx_trunc3;
2958 ns->rx_missed_errors = 0;
2959
2960 /* detailed tx_errors */
2961 ns->tx_aborted_errors = 0;
2962 ns->tx_carrier_errors = 0;
2963 ns->tx_fifo_errors = 0;
2964 ns->tx_heartbeat_errors = 0;
2965 ns->tx_window_errors = 0;
2966
2967 ns->tx_errors = stats.tx_error_frames;
2968 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2969 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2970 return ns;
2971}
2972
2973static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2974{
060e0c75 2975 unsigned int mbox;
b8ff05a9
DM		 2976 int ret = 0, prtad, devad;
2977 struct port_info *pi = netdev_priv(dev);
2978 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2979
2980 switch (cmd) {
2981 case SIOCGMIIPHY:
2982 if (pi->mdio_addr < 0)
2983 return -EOPNOTSUPP;
2984 data->phy_id = pi->mdio_addr;
2985 break;
2986 case SIOCGMIIREG:
2987 case SIOCSMIIREG:
2988 if (mdio_phy_id_is_c45(data->phy_id)) {
2989 prtad = mdio_phy_id_prtad(data->phy_id);
2990 devad = mdio_phy_id_devad(data->phy_id);
2991 } else if (data->phy_id < 32) {
2992 prtad = data->phy_id;
2993 devad = 0;
2994 data->reg_num &= 0x1f;
2995 } else
2996 return -EINVAL;
2997
060e0c75 2998 mbox = pi->adapter->fn;
b8ff05a9 2999 if (cmd == SIOCGMIIREG)
060e0c75 3000 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 3001 data->reg_num, &data->val_out);
3002 else
060e0c75 3003 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 3004 data->reg_num, data->val_in);
3005 break;
3006 default:
3007 return -EOPNOTSUPP;
3008 }
3009 return ret;
3010}
3011
3012static void cxgb_set_rxmode(struct net_device *dev)
3013{
3014 /* unfortunately we can't return errors to the stack */
3015 set_rxmode(dev, -1, false);
3016}
3017
3018static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
3019{
3020 int ret;
3021 struct port_info *pi = netdev_priv(dev);
3022
3023 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
3024 return -EINVAL;
060e0c75
DM		 3025 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
3026 -1, -1, -1, true);
b8ff05a9
DM		 3027 if (!ret)
3028 dev->mtu = new_mtu;
3029 return ret;
3030}
3031
3032static int cxgb_set_mac_addr(struct net_device *dev, void *p)
3033{
3034 int ret;
3035 struct sockaddr *addr = p;
3036 struct port_info *pi = netdev_priv(dev);
3037
3038 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 3039 return -EADDRNOTAVAIL;
b8ff05a9 3040
060e0c75
DM		 3041 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
3042 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 3043 if (ret < 0)
3044 return ret;
3045
3046 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3047 pi->xact_addr_filt = ret;
3048 return 0;
3049}
3050
b8ff05a9
DM		 3051#ifdef CONFIG_NET_POLL_CONTROLLER
3052static void cxgb_netpoll(struct net_device *dev)
3053{
3054 struct port_info *pi = netdev_priv(dev);
3055 struct adapter *adap = pi->adapter;
3056
3057 if (adap->flags & USING_MSIX) {
3058 int i;
3059 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
3060
3061 for (i = pi->nqsets; i; i--, rx++)
3062 t4_sge_intr_msix(0, &rx->rspq);
3063 } else
3064 t4_intr_handler(adap)(0, adap);
3065}
3066#endif
3067
3068static const struct net_device_ops cxgb4_netdev_ops = {
3069 .ndo_open = cxgb_open,
3070 .ndo_stop = cxgb_close,
3071 .ndo_start_xmit = t4_eth_xmit,
9be793bf 3072 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 3073 .ndo_set_rx_mode = cxgb_set_rxmode,
3074 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 3075 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 3076 .ndo_validate_addr = eth_validate_addr,
3077 .ndo_do_ioctl = cxgb_ioctl,
3078 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 3079#ifdef CONFIG_NET_POLL_CONTROLLER
3080 .ndo_poll_controller = cxgb_netpoll,
3081#endif
3082};
3083
3084void t4_fatal_err(struct adapter *adap)
3085{
3086 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
3087 t4_intr_disable(adap);
3088 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
3089}
3090
3091static void setup_memwin(struct adapter *adap)
3092{
3093 u32 bar0;
3094
3095 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
3096 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
3097 (bar0 + MEMWIN0_BASE) | BIR(0) |
3098 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
3099 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
3100 (bar0 + MEMWIN1_BASE) | BIR(0) |
3101 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
3102 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
3103 (bar0 + MEMWIN2_BASE) | BIR(0) |
3104 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
636f9d37
VP		 3105}
3106
3107static void setup_memwin_rdma(struct adapter *adap)
3108{
1ae970e0
DM		 3109 if (adap->vres.ocq.size) {
3110 unsigned int start, sz_kb;
3111
3112 start = pci_resource_start(adap->pdev, 2) +
3113 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
3114 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
3115 t4_write_reg(adap,
3116 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
3117 start | BIR(1) | WINDOW(ilog2(sz_kb)));
3118 t4_write_reg(adap,
3119 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
3120 adap->vres.ocq.start);
3121 t4_read_reg(adap,
3122 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
3123 }
b8ff05a9
DM		 3124}
3125
02b5fb8e
DM		 3126static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
3127{
3128 u32 v;
3129 int ret;
3130
3131 /* get device capabilities */
3132 memset(c, 0, sizeof(*c));
3133 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3134 FW_CMD_REQUEST | FW_CMD_READ);
3135 c->retval_len16 = htonl(FW_LEN16(*c));
060e0c75 3136 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 3137 if (ret < 0)
3138 return ret;
3139
3140 /* select capabilities we'll be using */
3141 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3142 if (!vf_acls)
3143 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3144 else
3145 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3146 } else if (vf_acls) {
3147 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
3148 return ret;
3149 }
3150 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3151 FW_CMD_REQUEST | FW_CMD_WRITE);
060e0c75 3152 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 3153 if (ret < 0)
3154 return ret;
3155
060e0c75 3156 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e
DM		 3157 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3158 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
3159 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
3160 if (ret < 0)
3161 return ret;
3162
060e0c75
DM		 3163 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
3164 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 3165 if (ret < 0)
3166 return ret;
3167
3168 t4_sge_init(adap);
3169
02b5fb8e
DM		 3170 /* tweak some settings */
3171 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
3172 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
3173 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
3174 v = t4_read_reg(adap, TP_PIO_DATA);
3175 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
060e0c75
DM		 3176
3177 /* get basic stuff going */
3178 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 3179}
3180
b8ff05a9
DM		 3181/*
3182 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
3183 */
3184#define MAX_ATIDS 8192U
3185
636f9d37
VP		 3186/*
3187 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3188 *
3189 * If the firmware we're dealing with has Configuration File support, then
3190 * we use that to perform all configuration
3191 */
3192
3193/*
3194 * Tweak configuration based on module parameters, etc. Most of these have
3195 * defaults assigned to them by Firmware Configuration Files (if we're using
3196 * them) but need to be explicitly set if we're using hard-coded
3197 * initialization. But even in the case of using Firmware Configuration
3198 * Files, we'd like to expose the ability to change these via module
3199 * parameters so these are essentially common tweaks/settings for
3200 * Configuration Files and hard-coded initialization ...
3201 */
3202static int adap_init0_tweaks(struct adapter *adapter)
3203{
3204 /*
3205 * Fix up various Host-Dependent Parameters like Page Size, Cache
3206 * Line Size, etc. The firmware default is for a 4KB Page Size and
3207 * 64B Cache Line Size ...
3208 */
3209 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
3210
3211 /*
3212 * Process module parameters which affect early initialization.
3213 */
3214 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
3215 dev_err(&adapter->pdev->dev,
3216 "Ignoring illegal rx_dma_offset=%d, using 2\n",
3217 rx_dma_offset);
3218 rx_dma_offset = 2;
3219 }
3220 t4_set_reg_field(adapter, SGE_CONTROL,
3221 PKTSHIFT_MASK,
3222 PKTSHIFT(rx_dma_offset));
3223
3224 /*
3225 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
3226 * adds the pseudo header itself.
3227 */
3228 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
3229 CSUM_HAS_PSEUDO_HDR, 0);
3230
3231 return 0;
3232}
3233
3234/*
3235 * Attempt to initialize the adapter via a Firmware Configuration File.
3236 */
3237static int adap_init0_config(struct adapter *adapter, int reset)
3238{
3239 struct fw_caps_config_cmd caps_cmd;
3240 const struct firmware *cf;
3241 unsigned long mtype = 0, maddr = 0;
3242 u32 finiver, finicsum, cfcsum;
3243 int ret, using_flash;
3244
3245 /*
3246 * Reset device if necessary.
3247 */
3248 if (reset) {
3249 ret = t4_fw_reset(adapter, adapter->mbox,
3250 PIORSTMODE | PIORST);
3251 if (ret < 0)
3252 goto bye;
3253 }
3254
3255 /*
3256 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
3257 * then use that. Otherwise, use the configuration file stored
3258 * in the adapter flash ...
3259 */
3260 ret = request_firmware(&cf, FW_CFNAME, adapter->pdev_dev);
3261 if (ret < 0) {
3262 using_flash = 1;
3263 mtype = FW_MEMTYPE_CF_FLASH;
3264 maddr = t4_flash_cfg_addr(adapter);
3265 } else {
3266 u32 params[7], val[7];
3267
3268 using_flash = 0;
3269 if (cf->size >= FLASH_CFG_MAX_SIZE)
3270 ret = -ENOMEM;
3271 else {
3272 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3273 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
3274 ret = t4_query_params(adapter, adapter->mbox,
3275 adapter->fn, 0, 1, params, val);
3276 if (ret == 0) {
3277 /*
3278 * For t4_memory_write() below addresses and
3279 * sizes have to be in terms of multiples of 4
3280 * bytes. So, if the Configuration File isn't
3281 * a multiple of 4 bytes in length we'll have
3282 * to write that out separately since we can't
3283 * guarantee that the bytes following the
3284 * residual byte in the buffer returned by
3285 * request_firmware() are zeroed out ...
3286 */
3287 size_t resid = cf->size & 0x3;
3288 size_t size = cf->size & ~0x3;
3289 __be32 *data = (__be32 *)cf->data;
3290
3291 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
3292 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
3293
3294 ret = t4_memory_write(adapter, mtype, maddr,
3295 size, data);
3296 if (ret == 0 && resid != 0) {
3297 union {
3298 __be32 word;
3299 char buf[4];
3300 } last;
3301 int i;
3302
3303 last.word = data[size >> 2];
3304 for (i = resid; i < 4; i++)
3305 last.buf[i] = 0;
3306 ret = t4_memory_write(adapter, mtype,
3307 maddr + size,
3308 4, &last.word);
3309 }
3310 }
3311 }
3312
3313 release_firmware(cf);
3314 if (ret)
3315 goto bye;
3316 }
3317
3318 /*
3319 * Issue a Capability Configuration command to the firmware to get it
3320 * to parse the Configuration File. We don't use t4_fw_config_file()
3321 * because we want the ability to modify various features after we've
3322 * processed the configuration file ...
3323 */
3324 memset(&caps_cmd, 0, sizeof(caps_cmd));
3325 caps_cmd.op_to_write =
3326 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3327 FW_CMD_REQUEST |
3328 FW_CMD_READ);
3329 caps_cmd.retval_len16 =
3330 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
3331 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
3332 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
3333 FW_LEN16(caps_cmd));
3334 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3335 &caps_cmd);
3336 if (ret < 0)
3337 goto bye;
3338
3339 finiver = ntohl(caps_cmd.finiver);
3340 finicsum = ntohl(caps_cmd.finicsum);
3341 cfcsum = ntohl(caps_cmd.cfcsum);
3342 if (finicsum != cfcsum)
3343 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
3344 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
3345 finicsum, cfcsum);
3346
3347 /*
3348 * If we're a pure NIC driver then disable all offloading facilities.
3349 * This will allow the firmware to optimize aspects of the hardware
3350 * configuration which will result in improved performance.
3351 */
3352 caps_cmd.ofldcaps = 0;
3353 caps_cmd.iscsicaps = 0;
3354 caps_cmd.rdmacaps = 0;
3355 caps_cmd.fcoecaps = 0;
3356
3357 /*
3358 * And now tell the firmware to use the configuration we just loaded.
3359 */
3360 caps_cmd.op_to_write =
3361 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3362 FW_CMD_REQUEST |
3363 FW_CMD_WRITE);
3364 caps_cmd.retval_len16 = htonl(FW_LEN16(caps_cmd));
3365 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3366 NULL);
3367 if (ret < 0)
3368 goto bye;
3369
3370 /*
3371 * Tweak configuration based on system architecture, module
3372 * parameters, etc.
3373 */
3374 ret = adap_init0_tweaks(adapter);
3375 if (ret < 0)
3376 goto bye;
3377
3378 /*
3379 * And finally tell the firmware to initialize itself using the
3380 * parameters from the Configuration File.
3381 */
3382 ret = t4_fw_initialize(adapter, adapter->mbox);
3383 if (ret < 0)
3384 goto bye;
3385
3386 /*
3387 * Return successfully and note that we're operating with parameters
3388 * not supplied by the driver, rather than from hard-wired
3389 * initialization constants burried in the driver.
3390 */
3391 adapter->flags |= USING_SOFT_PARAMS;
3392 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
3393 "Configuration File %s, version %#x, computed checksum %#x\n",
3394 (using_flash
3395 ? "in device FLASH"
3396 : "/lib/firmware/" FW_CFNAME),
3397 finiver, cfcsum);
3398 return 0;
3399
3400 /*
3401 * Something bad happened. Return the error ... (If the "error"
3402 * is that there's no Configuration File on the adapter we don't
3403 * want to issue a warning since this is fairly common.)
3404 */
3405bye:
3406 if (ret != -ENOENT)
3407 dev_warn(adapter->pdev_dev, "Configuration file error %d\n",
3408 -ret);
3409 return ret;
3410}
3411
b8ff05a9
DM		 3412/*
3413 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3414 */
3415static int adap_init0(struct adapter *adap)
3416{
3417 int ret;
3418 u32 v, port_vec;
3419 enum dev_state state;
3420 u32 params[7], val[7];
636f9d37 3421 int reset = 1, j;
b8ff05a9 3422
636f9d37
VP		 3423 /*
3424 * Contact FW, advertising Master capability (and potentially forcing
3425 * ourselves as the Master PF if our module parameter force_init is
3426 * set).
3427 */
3428 ret = t4_fw_hello(adap, adap->mbox, adap->fn,
3429 force_init ? MASTER_MUST : MASTER_MAY,
3430 &state);
b8ff05a9
DM		 3431 if (ret < 0) {
3432 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3433 ret);
3434 return ret;
3435 }
636f9d37
VP		 3436 if (ret == adap->mbox)
3437 adap->flags |= MASTER_PF;
3438 if (force_init && state == DEV_STATE_INIT)
3439 state = DEV_STATE_UNINIT;
b8ff05a9 3440
636f9d37
VP		 3441 /*
3442 * If we're the Master PF Driver and the device is uninitialized,
3443 * then let's consider upgrading the firmware ... (We always want
3444 * to check the firmware version number in order to A. get it for
3445 * later reporting and B. to warn if the currently loaded firmware
3446 * is excessively mismatched relative to the driver.)
3447 */
3448 ret = t4_check_fw_version(adap);
3449 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
3450 if (ret == -EINVAL || ret > 0) {
3451 if (upgrade_fw(adap) >= 0) {
3452 /*
3453 * Note that the chip was reset as part of the
3454 * firmware upgrade so we don't reset it again
3455 * below and grab the new firmware version.
3456 */
3457 reset = 0;
3458 ret = t4_check_fw_version(adap);
3459 }
3460 }
3461 if (ret < 0)
3462 return ret;
3463 }
b8ff05a9 3464
636f9d37
VP		 3465 /*
3466 * Grab VPD parameters. This should be done after we establish a
3467 * connection to the firmware since some of the VPD parameters
3468 * (notably the Core Clock frequency) are retrieved via requests to
3469 * the firmware. On the other hand, we need these fairly early on
3470 * so we do this right after getting ahold of the firmware.
3471 */
3472 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 3473 if (ret < 0)
3474 goto bye;
a0881cab 3475
636f9d37
VP		 3476 /*
3477 * Find out what ports are available to us.
3478 */
3479 v =
3480 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3481 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
3482 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 3483 if (ret < 0)
3484 goto bye;
3485
636f9d37
VP		 3486 adap->params.nports = hweight32(port_vec);
3487 adap->params.portvec = port_vec;
3488
3489 /*
3490 * If the firmware is initialized already (and we're not forcing a
3491 * master initialization), note that we're living with existing
3492 * adapter parameters. Otherwise, it's time to try initializing the
3493 * adapter ...
3494 */
3495 if (state == DEV_STATE_INIT) {
3496 dev_info(adap->pdev_dev, "Coming up as %s: "\
3497 "Adapter already initialized\n",
3498 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
3499 adap->flags |= USING_SOFT_PARAMS;
3500 } else {
3501 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
3502 "Initializing adapter\n");
3503 /*
3504 * Find out whether we're dealing with a version of
3505 * the firmware which has configuration file support.
3506 */
3507 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3508 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
3509 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
3510 params, val);
3511
3512 /*
3513 * If the firmware doesn't support Configuration
3514 * Files warn user and exit,
3515 */
3516 if (ret < 0)
3517 dev_warn(adap->pdev_dev, "Firmware doesn't support "\
3518 "configuration file.\n");
3519 else {
3520 /*
3521 * The firmware provides us with a memory
3522 * buffer where we can load a Configuration
3523 * File from the host if we want to override
3524 * the Configuration File in flash.
3525 */
3526
3527 ret = adap_init0_config(adap, reset);
3528 if (ret == -ENOENT) {
3529 dev_info(adap->pdev_dev,
3530 "No Configuration File present "
3531 "on adapter.\n");
3532 }
3533 }
3534 if (ret < 0) {
3535 dev_err(adap->pdev_dev,
3536 "could not initialize adapter, error %d\n",
3537 -ret);
3538 goto bye;
3539 }
3540 }
3541
3542 /*
3543 * If we're living with non-hard-coded parameters (either from a
3544 * Firmware Configuration File or values programmed by a different PF
3545 * Driver), give the SGE code a chance to pull in anything that it
3546 * needs ... Note that this must be called after we retrieve our VPD
3547 * parameters in order to know how to convert core ticks to seconds.
3548 */
3549 if (adap->flags & USING_SOFT_PARAMS) {
3550 ret = t4_sge_init(adap);
3551 if (ret < 0)
3552 goto bye;
3553 }
3554
3555 /*
3556 * Grab some of our basic fundamental operating parameters.
3557 */
3558#define FW_PARAM_DEV(param) \
3559 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3560 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3561
b8ff05a9 3562#define FW_PARAM_PFVF(param) \
636f9d37
VP		 3563 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3564 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
3565 FW_PARAMS_PARAM_Y(0) | \
3566 FW_PARAMS_PARAM_Z(0)
b8ff05a9 3567
636f9d37 3568 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 3569 params[1] = FW_PARAM_PFVF(L2T_START);
3570 params[2] = FW_PARAM_PFVF(L2T_END);
3571 params[3] = FW_PARAM_PFVF(FILTER_START);
3572 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 3573 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 3574 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 3575 if (ret < 0)
3576 goto bye;
636f9d37
VP		 3577 adap->sge.egr_start = val[0];
3578 adap->l2t_start = val[1];
3579 adap->l2t_end = val[2];
b8ff05a9
DM		 3580 adap->tids.ftid_base = val[3];
3581 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 3582 adap->sge.ingr_start = val[5];
b8ff05a9 3583
636f9d37
VP		 3584 /* query params related to active filter region */
3585 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
3586 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
3587 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
3588 /* If Active filter size is set we enable establishing
3589 * offload connection through firmware work request
3590 */
3591 if ((val[0] != val[1]) && (ret >= 0)) {
3592 adap->flags |= FW_OFLD_CONN;
3593 adap->tids.aftid_base = val[0];
3594 adap->tids.aftid_end = val[1];
3595 }
3596
3597#ifdef CONFIG_CHELSIO_T4_OFFLOAD
3598 /*
3599 * Get device capabilities so we can determine what resources we need
3600 * to manage.
3601 */
3602 memset(&caps_cmd, 0, sizeof(caps_cmd));
3603 caps_cmd.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3604 F_FW_CMD_REQUEST | F_FW_CMD_READ);
3605 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
3606 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
3607 &caps_cmd);
3608 if (ret < 0)
3609 goto bye;
3610
3611 if (caps_cmd.toecaps) {
b8ff05a9
DM		 3612 /* query offload-related parameters */
3613 params[0] = FW_PARAM_DEV(NTID);
3614 params[1] = FW_PARAM_PFVF(SERVER_START);
3615 params[2] = FW_PARAM_PFVF(SERVER_END);
3616 params[3] = FW_PARAM_PFVF(TDDP_START);
3617 params[4] = FW_PARAM_PFVF(TDDP_END);
3618 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 3619 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
3620 params, val);
b8ff05a9
DM		 3621 if (ret < 0)
3622 goto bye;
3623 adap->tids.ntids = val[0];
3624 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3625 adap->tids.stid_base = val[1];
3626 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37
VP		 3627 /*
3628 * Setup server filter region. Divide the availble filter
3629 * region into two parts. Regular filters get 1/3rd and server
3630 * filters get 2/3rd part. This is only enabled if workarond
3631 * path is enabled.
3632 * 1. For regular filters.
3633 * 2. Server filter: This are special filters which are used
3634 * to redirect SYN packets to offload queue.
3635 */
3636 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
3637 adap->tids.sftid_base = adap->tids.ftid_base +
3638 DIV_ROUND_UP(adap->tids.nftids, 3);
3639 adap->tids.nsftids = adap->tids.nftids -
3640 DIV_ROUND_UP(adap->tids.nftids, 3);
3641 adap->tids.nftids = adap->tids.sftid_base -
3642 adap->tids.ftid_base;
3643 }
b8ff05a9
DM		 3644 adap->vres.ddp.start = val[3];
3645 adap->vres.ddp.size = val[4] - val[3] + 1;
3646 adap->params.ofldq_wr_cred = val[5];
636f9d37
VP		 3647
3648 params[0] = FW_PARAM_PFVF(ETHOFLD_START);
3649 params[1] = FW_PARAM_PFVF(ETHOFLD_END);
3650 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
3651 params, val);
3652 if ((val[0] != val[1]) && (ret >= 0)) {
3653 adap->tids.uotid_base = val[0];
3654 adap->tids.nuotids = val[1] - val[0] + 1;
3655 }
3656
b8ff05a9
DM		 3657 adap->params.offload = 1;
3658 }
636f9d37 3659 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 3660 params[0] = FW_PARAM_PFVF(STAG_START);
3661 params[1] = FW_PARAM_PFVF(STAG_END);
3662 params[2] = FW_PARAM_PFVF(RQ_START);
3663 params[3] = FW_PARAM_PFVF(RQ_END);
3664 params[4] = FW_PARAM_PFVF(PBL_START);
3665 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 3666 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
3667 params, val);
b8ff05a9
DM		 3668 if (ret < 0)
3669 goto bye;
3670 adap->vres.stag.start = val[0];
3671 adap->vres.stag.size = val[1] - val[0] + 1;
3672 adap->vres.rq.start = val[2];
3673 adap->vres.rq.size = val[3] - val[2] + 1;
3674 adap->vres.pbl.start = val[4];
3675 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 3676
3677 params[0] = FW_PARAM_PFVF(SQRQ_START);
3678 params[1] = FW_PARAM_PFVF(SQRQ_END);
3679 params[2] = FW_PARAM_PFVF(CQ_START);
3680 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 3681 params[4] = FW_PARAM_PFVF(OCQ_START);
3682 params[5] = FW_PARAM_PFVF(OCQ_END);
636f9d37 3683 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
a0881cab
DM		 3684 if (ret < 0)
3685 goto bye;
3686 adap->vres.qp.start = val[0];
3687 adap->vres.qp.size = val[1] - val[0] + 1;
3688 adap->vres.cq.start = val[2];
3689 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 3690 adap->vres.ocq.start = val[4];
3691 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9 3692 }
636f9d37 3693 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 3694 params[0] = FW_PARAM_PFVF(ISCSI_START);
3695 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 3696 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
3697 params, val);
b8ff05a9
DM		 3698 if (ret < 0)
3699 goto bye;
3700 adap->vres.iscsi.start = val[0];
3701 adap->vres.iscsi.size = val[1] - val[0] + 1;
3702 }
3703#undef FW_PARAM_PFVF
3704#undef FW_PARAM_DEV
636f9d37 3705#endif /* CONFIG_CHELSIO_T4_OFFLOAD */
b8ff05a9 3706
636f9d37
VP		 3707 /*
3708 * These are finalized by FW initialization, load their values now.
3709 */
b8ff05a9
DM		 3710 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3711 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
636f9d37 3712 adap->params.tp.dack_re = DELAYEDACKRESOLUTION_GET(v);
b8ff05a9
DM		 3713 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3714 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3715 adap->params.b_wnd);
7ee9ff94 3716
636f9d37
VP		 3717 /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
3718 for (j = 0; j < NCHAN; j++)
3719 adap->params.tp.tx_modq[j] = j;
7ee9ff94 3720
636f9d37 3721 adap->flags |= FW_OK;
b8ff05a9
DM		 3722 return 0;
3723
3724 /*
636f9d37
VP		 3725 * Something bad happened. If a command timed out or failed with EIO
3726 * FW does not operate within its spec or something catastrophic
3727 * happened to HW/FW, stop issuing commands.
b8ff05a9 3728 */
636f9d37
VP		 3729bye:
3730 if (ret != -ETIMEDOUT && ret != -EIO)
3731 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 3732 return ret;
3733}
3734
204dc3c0
DM		 3735/* EEH callbacks */
3736
3737static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3738 pci_channel_state_t state)
3739{
3740 int i;
3741 struct adapter *adap = pci_get_drvdata(pdev);
3742
3743 if (!adap)
3744 goto out;
3745
3746 rtnl_lock();
3747 adap->flags &= ~FW_OK;
3748 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3749 for_each_port(adap, i) {
3750 struct net_device *dev = adap->port[i];
3751
3752 netif_device_detach(dev);
3753 netif_carrier_off(dev);
3754 }
3755 if (adap->flags & FULL_INIT_DONE)
3756 cxgb_down(adap);
3757 rtnl_unlock();
3758 pci_disable_device(pdev);
3759out: return state == pci_channel_io_perm_failure ?
3760 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3761}
3762
3763static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3764{
3765 int i, ret;
3766 struct fw_caps_config_cmd c;
3767 struct adapter *adap = pci_get_drvdata(pdev);
3768
3769 if (!adap) {
3770 pci_restore_state(pdev);
3771 pci_save_state(pdev);
3772 return PCI_ERS_RESULT_RECOVERED;
3773 }
3774
3775 if (pci_enable_device(pdev)) {
3776 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3777 return PCI_ERS_RESULT_DISCONNECT;
3778 }
3779
3780 pci_set_master(pdev);
3781 pci_restore_state(pdev);
3782 pci_save_state(pdev);
3783 pci_cleanup_aer_uncorrect_error_status(pdev);
3784
3785 if (t4_wait_dev_ready(adap) < 0)
3786 return PCI_ERS_RESULT_DISCONNECT;
060e0c75 3787 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
204dc3c0
DM		 3788 return PCI_ERS_RESULT_DISCONNECT;
3789 adap->flags |= FW_OK;
3790 if (adap_init1(adap, &c))
3791 return PCI_ERS_RESULT_DISCONNECT;
3792
3793 for_each_port(adap, i) {
3794 struct port_info *p = adap2pinfo(adap, i);
3795
060e0c75
DM		 3796 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3797 NULL, NULL);
204dc3c0
DM		 3798 if (ret < 0)
3799 return PCI_ERS_RESULT_DISCONNECT;
3800 p->viid = ret;
3801 p->xact_addr_filt = -1;
3802 }
3803
3804 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3805 adap->params.b_wnd);
1ae970e0 3806 setup_memwin(adap);
204dc3c0
DM		 3807 if (cxgb_up(adap))
3808 return PCI_ERS_RESULT_DISCONNECT;
3809 return PCI_ERS_RESULT_RECOVERED;
3810}
3811
3812static void eeh_resume(struct pci_dev *pdev)
3813{
3814 int i;
3815 struct adapter *adap = pci_get_drvdata(pdev);
3816
3817 if (!adap)
3818 return;
3819
3820 rtnl_lock();
3821 for_each_port(adap, i) {
3822 struct net_device *dev = adap->port[i];
3823
3824 if (netif_running(dev)) {
3825 link_start(dev);
3826 cxgb_set_rxmode(dev);
3827 }
3828 netif_device_attach(dev);
3829 }
3830 rtnl_unlock();
3831}
3832
3833static struct pci_error_handlers cxgb4_eeh = {
3834 .error_detected = eeh_err_detected,
3835 .slot_reset = eeh_slot_reset,
3836 .resume = eeh_resume,
3837};
3838
b8ff05a9
DM		 3839static inline bool is_10g_port(const struct link_config *lc)
3840{
3841 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3842}
3843
3844static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3845 unsigned int size, unsigned int iqe_size)
3846{
3847 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3848 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3849 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3850 q->iqe_len = iqe_size;
3851 q->size = size;
3852}
3853
3854/*
3855 * Perform default configuration of DMA queues depending on the number and type
3856 * of ports we found and the number of available CPUs. Most settings can be
3857 * modified by the admin prior to actual use.
3858 */
3859static void __devinit cfg_queues(struct adapter *adap)
3860{
3861 struct sge *s = &adap->sge;
3862 int i, q10g = 0, n10g = 0, qidx = 0;
3863
3864 for_each_port(adap, i)
3865 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3866
3867 /*
3868 * We default to 1 queue per non-10G port and up to # of cores queues
3869 * per 10G port.
3870 */
3871 if (n10g)
3872 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 3873 if (q10g > netif_get_num_default_rss_queues())
3874 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 3875
3876 for_each_port(adap, i) {
3877 struct port_info *pi = adap2pinfo(adap, i);
3878
3879 pi->first_qset = qidx;
3880 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3881 qidx += pi->nqsets;
3882 }
3883
3884 s->ethqsets = qidx;
3885 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3886
3887 if (is_offload(adap)) {
3888 /*
3889 * For offload we use 1 queue/channel if all ports are up to 1G,
3890 * otherwise we divide all available queues amongst the channels
3891 * capped by the number of available cores.
3892 */
3893 if (n10g) {
3894 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3895 num_online_cpus());
3896 s->ofldqsets = roundup(i, adap->params.nports);
3897 } else
3898 s->ofldqsets = adap->params.nports;
3899 /* For RDMA one Rx queue per channel suffices */
3900 s->rdmaqs = adap->params.nports;
3901 }
3902
3903 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3904 struct sge_eth_rxq *r = &s->ethrxq[i];
3905
3906 init_rspq(&r->rspq, 0, 0, 1024, 64);
3907 r->fl.size = 72;
3908 }
3909
3910 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3911 s->ethtxq[i].q.size = 1024;
3912
3913 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3914 s->ctrlq[i].q.size = 512;
3915
3916 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3917 s->ofldtxq[i].q.size = 1024;
3918
3919 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3920 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3921
3922 init_rspq(&r->rspq, 0, 0, 1024, 64);
3923 r->rspq.uld = CXGB4_ULD_ISCSI;
3924 r->fl.size = 72;
3925 }
3926
3927 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3928 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3929
3930 init_rspq(&r->rspq, 0, 0, 511, 64);
3931 r->rspq.uld = CXGB4_ULD_RDMA;
3932 r->fl.size = 72;
3933 }
3934
3935 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3936 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3937}
3938
3939/*
3940 * Reduce the number of Ethernet queues across all ports to at most n.
3941 * n provides at least one queue per port.
3942 */
3943static void __devinit reduce_ethqs(struct adapter *adap, int n)
3944{
3945 int i;
3946 struct port_info *pi;
3947
3948 while (n < adap->sge.ethqsets)
3949 for_each_port(adap, i) {
3950 pi = adap2pinfo(adap, i);
3951 if (pi->nqsets > 1) {
3952 pi->nqsets--;
3953 adap->sge.ethqsets--;
3954 if (adap->sge.ethqsets <= n)
3955 break;
3956 }
3957 }
3958
3959 n = 0;
3960 for_each_port(adap, i) {
3961 pi = adap2pinfo(adap, i);
3962 pi->first_qset = n;
3963 n += pi->nqsets;
3964 }
3965}
3966
3967/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3968#define EXTRA_VECS 2
3969
3970static int __devinit enable_msix(struct adapter *adap)
3971{
3972 int ofld_need = 0;
3973 int i, err, want, need;
3974 struct sge *s = &adap->sge;
3975 unsigned int nchan = adap->params.nports;
3976 struct msix_entry entries[MAX_INGQ + 1];
3977
3978 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3979 entries[i].entry = i;
3980
3981 want = s->max_ethqsets + EXTRA_VECS;
3982 if (is_offload(adap)) {
3983 want += s->rdmaqs + s->ofldqsets;
3984 /* need nchan for each possible ULD */
3985 ofld_need = 2 * nchan;
3986 }
3987 need = adap->params.nports + EXTRA_VECS + ofld_need;
3988
3989 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3990 want = err;
3991
3992 if (!err) {
3993 /*
3994 * Distribute available vectors to the various queue groups.
3995 * Every group gets its minimum requirement and NIC gets top
3996 * priority for leftovers.
3997 */
3998 i = want - EXTRA_VECS - ofld_need;
3999 if (i < s->max_ethqsets) {
4000 s->max_ethqsets = i;
4001 if (i < s->ethqsets)
4002 reduce_ethqs(adap, i);
4003 }
4004 if (is_offload(adap)) {
4005 i = want - EXTRA_VECS - s->max_ethqsets;
4006 i -= ofld_need - nchan;
4007 s->ofldqsets = (i / nchan) * nchan; /* round down */
4008 }
4009 for (i = 0; i < want; ++i)
4010 adap->msix_info[i].vec = entries[i].vector;
4011 } else if (err > 0)
4012 dev_info(adap->pdev_dev,
4013 "only %d MSI-X vectors left, not using MSI-X\n", err);
4014 return err;
4015}
4016
4017#undef EXTRA_VECS
4018
671b0060
DM		 4019static int __devinit init_rss(struct adapter *adap)
4020{
4021 unsigned int i, j;
4022
4023 for_each_port(adap, i) {
4024 struct port_info *pi = adap2pinfo(adap, i);
4025
4026 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
4027 if (!pi->rss)
4028 return -ENOMEM;
4029 for (j = 0; j < pi->rss_size; j++)
278bc429 4030 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 4031 }
4032 return 0;
4033}
4034
118969ed 4035static void __devinit print_port_info(const struct net_device *dev)
b8ff05a9
DM		 4036{
4037 static const char *base[] = {
a0881cab 4038 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
7d5e77aa 4039 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
b8ff05a9
DM		 4040 };
4041
b8ff05a9 4042 char buf[80];
118969ed 4043 char *bufp = buf;
f1a051b9 4044 const char *spd = "";
118969ed
DM		 4045 const struct port_info *pi = netdev_priv(dev);
4046 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 4047
4048 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
4049 spd = " 2.5 GT/s";
4050 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
4051 spd = " 5 GT/s";
b8ff05a9 4052
118969ed
DM		 4053 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
4054 bufp += sprintf(bufp, "100/");
4055 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
4056 bufp += sprintf(bufp, "1000/");
4057 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
4058 bufp += sprintf(bufp, "10G/");
4059 if (bufp != buf)
4060 --bufp;
4061 sprintf(bufp, "BASE-%s", base[pi->port_type]);
4062
4063 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
4064 adap->params.vpd.id, adap->params.rev, buf,
4065 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
4066 (adap->flags & USING_MSIX) ? " MSI-X" :
4067 (adap->flags & USING_MSI) ? " MSI" : "");
4068 netdev_info(dev, "S/N: %s, E/C: %s\n",
4069 adap->params.vpd.sn, adap->params.vpd.ec);
b8ff05a9
DM		 4070}
4071
ef306b50
DM		 4072static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
4073{
4074 u16 v;
4075 int pos;
4076
4077 pos = pci_pcie_cap(dev);
4078 if (pos > 0) {
4079 pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &v);
4080 v |= PCI_EXP_DEVCTL_RELAX_EN;
4081 pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, v);
4082 }
4083}
4084
06546391
DM		 4085/*
4086 * Free the following resources:
4087 * - memory used for tables
4088 * - MSI/MSI-X
4089 * - net devices
4090 * - resources FW is holding for us
4091 */
4092static void free_some_resources(struct adapter *adapter)
4093{
4094 unsigned int i;
4095
4096 t4_free_mem(adapter->l2t);
4097 t4_free_mem(adapter->tids.tid_tab);
4098 disable_msi(adapter);
4099
4100 for_each_port(adapter, i)
671b0060
DM		 4101 if (adapter->port[i]) {
4102 kfree(adap2pinfo(adapter, i)->rss);
06546391 4103 free_netdev(adapter->port[i]);
671b0060 4104 }
06546391 4105 if (adapter->flags & FW_OK)
060e0c75 4106 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 4107}
4108
2ed28baa 4109#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 4110#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9
DM		 4111 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
4112
4113static int __devinit init_one(struct pci_dev *pdev,
4114 const struct pci_device_id *ent)
4115{
4116 int func, i, err;
4117 struct port_info *pi;
c8f44aff 4118 bool highdma = false;
b8ff05a9
DM		 4119 struct adapter *adapter = NULL;
4120
4121 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
4122
4123 err = pci_request_regions(pdev, KBUILD_MODNAME);
4124 if (err) {
4125 /* Just info, some other driver may have claimed the device. */
4126 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
4127 return err;
4128 }
4129
060e0c75 4130 /* We control everything through one PF */
b8ff05a9 4131 func = PCI_FUNC(pdev->devfn);
060e0c75 4132 if (func != ent->driver_data) {
204dc3c0 4133 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 4134 goto sriov;
204dc3c0 4135 }
b8ff05a9
DM		 4136
4137 err = pci_enable_device(pdev);
4138 if (err) {
4139 dev_err(&pdev->dev, "cannot enable PCI device\n");
4140 goto out_release_regions;
4141 }
4142
4143 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 4144 highdma = true;
b8ff05a9
DM		 4145 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
4146 if (err) {
4147 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
4148 "coherent allocations\n");
4149 goto out_disable_device;
4150 }
4151 } else {
4152 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4153 if (err) {
4154 dev_err(&pdev->dev, "no usable DMA configuration\n");
4155 goto out_disable_device;
4156 }
4157 }
4158
4159 pci_enable_pcie_error_reporting(pdev);
ef306b50 4160 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 4161 pci_set_master(pdev);
4162 pci_save_state(pdev);
4163
4164 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
4165 if (!adapter) {
4166 err = -ENOMEM;
4167 goto out_disable_device;
4168 }
4169
4170 adapter->regs = pci_ioremap_bar(pdev, 0);
4171 if (!adapter->regs) {
4172 dev_err(&pdev->dev, "cannot map device registers\n");
4173 err = -ENOMEM;
4174 goto out_free_adapter;
4175 }
4176
4177 adapter->pdev = pdev;
4178 adapter->pdev_dev = &pdev->dev;
3069ee9b 4179 adapter->mbox = func;
060e0c75 4180 adapter->fn = func;
b8ff05a9
DM		 4181 adapter->msg_enable = dflt_msg_enable;
4182 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
4183
4184 spin_lock_init(&adapter->stats_lock);
4185 spin_lock_init(&adapter->tid_release_lock);
4186
4187 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 4188 INIT_WORK(&adapter->db_full_task, process_db_full);
4189 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 4190
4191 err = t4_prep_adapter(adapter);
4192 if (err)
4193 goto out_unmap_bar;
636f9d37 4194 setup_memwin(adapter);
b8ff05a9 4195 err = adap_init0(adapter);
636f9d37 4196 setup_memwin_rdma(adapter);
b8ff05a9
DM		 4197 if (err)
4198 goto out_unmap_bar;
4199
4200 for_each_port(adapter, i) {
4201 struct net_device *netdev;
4202
4203 netdev = alloc_etherdev_mq(sizeof(struct port_info),
4204 MAX_ETH_QSETS);
4205 if (!netdev) {
4206 err = -ENOMEM;
4207 goto out_free_dev;
4208 }
4209
4210 SET_NETDEV_DEV(netdev, &pdev->dev);
4211
4212 adapter->port[i] = netdev;
4213 pi = netdev_priv(netdev);
4214 pi->adapter = adapter;
4215 pi->xact_addr_filt = -1;
b8ff05a9 4216 pi->port_id = i;
b8ff05a9
DM		 4217 netdev->irq = pdev->irq;
4218
2ed28baa
MM		 4219 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
4220 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4221 NETIF_F_RXCSUM | NETIF_F_RXHASH |
4222 NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
c8f44aff
MM		 4223 if (highdma)
4224 netdev->hw_features |= NETIF_F_HIGHDMA;
4225 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 4226 netdev->vlan_features = netdev->features & VLAN_FEAT;
4227
01789349
JP		 4228 netdev->priv_flags |= IFF_UNICAST_FLT;
4229
b8ff05a9
DM		 4230 netdev->netdev_ops = &cxgb4_netdev_ops;
4231 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
4232 }
4233
4234 pci_set_drvdata(pdev, adapter);
4235
4236 if (adapter->flags & FW_OK) {
060e0c75 4237 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 4238 if (err)
4239 goto out_free_dev;
4240 }
4241
4242 /*
4243 * Configure queues and allocate tables now, they can be needed as
4244 * soon as the first register_netdev completes.
4245 */
4246 cfg_queues(adapter);
4247
4248 adapter->l2t = t4_init_l2t();
4249 if (!adapter->l2t) {
4250 /* We tolerate a lack of L2T, giving up some functionality */
4251 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
4252 adapter->params.offload = 0;
4253 }
4254
4255 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
4256 dev_warn(&pdev->dev, "could not allocate TID table, "
4257 "continuing\n");
4258 adapter->params.offload = 0;
4259 }
4260
f7cabcdd
DM		 4261 /* See what interrupts we'll be using */
4262 if (msi > 1 && enable_msix(adapter) == 0)
4263 adapter->flags |= USING_MSIX;
4264 else if (msi > 0 && pci_enable_msi(pdev) == 0)
4265 adapter->flags |= USING_MSI;
4266
671b0060
DM		 4267 err = init_rss(adapter);
4268 if (err)
4269 goto out_free_dev;
4270
b8ff05a9
DM		 4271 /*
4272 * The card is now ready to go. If any errors occur during device
4273 * registration we do not fail the whole card but rather proceed only
4274 * with the ports we manage to register successfully. However we must
4275 * register at least one net device.
4276 */
4277 for_each_port(adapter, i) {
a57cabe0
DM		 4278 pi = adap2pinfo(adapter, i);
4279 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
4280 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
4281
b8ff05a9
DM		 4282 err = register_netdev(adapter->port[i]);
4283 if (err)
b1a3c2b6 4284 break;
b1a3c2b6
DM		 4285 adapter->chan_map[pi->tx_chan] = i;
4286 print_port_info(adapter->port[i]);
b8ff05a9 4287 }
b1a3c2b6 4288 if (i == 0) {
b8ff05a9
DM		 4289 dev_err(&pdev->dev, "could not register any net devices\n");
4290 goto out_free_dev;
4291 }
b1a3c2b6
DM		 4292 if (err) {
4293 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
4294 err = 0;
6403eab1 4295 }
b8ff05a9
DM		 4296
4297 if (cxgb4_debugfs_root) {
4298 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
4299 cxgb4_debugfs_root);
4300 setup_debugfs(adapter);
4301 }
4302
6482aa7c
DLR		 4303 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
4304 pdev->needs_freset = 1;
4305
b8ff05a9
DM		 4306 if (is_offload(adapter))
4307 attach_ulds(adapter);
4308
b8ff05a9
DM		 4309sriov:
4310#ifdef CONFIG_PCI_IOV
4311 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
4312 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
4313 dev_info(&pdev->dev,
4314 "instantiated %u virtual functions\n",
4315 num_vf[func]);
4316#endif
4317 return 0;
4318
4319 out_free_dev:
06546391 4320 free_some_resources(adapter);
b8ff05a9
DM		 4321 out_unmap_bar:
4322 iounmap(adapter->regs);
4323 out_free_adapter:
4324 kfree(adapter);
4325 out_disable_device:
4326 pci_disable_pcie_error_reporting(pdev);
4327 pci_disable_device(pdev);
4328 out_release_regions:
4329 pci_release_regions(pdev);
4330 pci_set_drvdata(pdev, NULL);
4331 return err;
4332}
4333
4334static void __devexit remove_one(struct pci_dev *pdev)
4335{
4336 struct adapter *adapter = pci_get_drvdata(pdev);
4337
636f9d37 4338#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 4339 pci_disable_sriov(pdev);
4340
636f9d37
VP		 4341#endif
4342
b8ff05a9
DM		 4343 if (adapter) {
4344 int i;
4345
4346 if (is_offload(adapter))
4347 detach_ulds(adapter);
4348
4349 for_each_port(adapter, i)
8f3a7676 4350 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 4351 unregister_netdev(adapter->port[i]);
4352
4353 if (adapter->debugfs_root)
4354 debugfs_remove_recursive(adapter->debugfs_root);
4355
aaefae9b
DM		 4356 if (adapter->flags & FULL_INIT_DONE)
4357 cxgb_down(adapter);
b8ff05a9 4358
06546391 4359 free_some_resources(adapter);
b8ff05a9
DM		 4360 iounmap(adapter->regs);
4361 kfree(adapter);
4362 pci_disable_pcie_error_reporting(pdev);
4363 pci_disable_device(pdev);
4364 pci_release_regions(pdev);
4365 pci_set_drvdata(pdev, NULL);
a069ec91 4366 } else
b8ff05a9
DM		 4367 pci_release_regions(pdev);
4368}
4369
4370static struct pci_driver cxgb4_driver = {
4371 .name = KBUILD_MODNAME,
4372 .id_table = cxgb4_pci_tbl,
4373 .probe = init_one,
4374 .remove = __devexit_p(remove_one),
204dc3c0 4375 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 4376};
4377
4378static int __init cxgb4_init_module(void)
4379{
4380 int ret;
4381
3069ee9b
VP		 4382 workq = create_singlethread_workqueue("cxgb4");
4383 if (!workq)
4384 return -ENOMEM;
4385
b8ff05a9
DM		 4386 /* Debugfs support is optional, just warn if this fails */
4387 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
4388 if (!cxgb4_debugfs_root)
4389 pr_warning("could not create debugfs entry, continuing\n");
4390
4391 ret = pci_register_driver(&cxgb4_driver);
4392 if (ret < 0)
4393 debugfs_remove(cxgb4_debugfs_root);
4394 return ret;
4395}
4396
4397static void __exit cxgb4_cleanup_module(void)
4398{
4399 pci_unregister_driver(&cxgb4_driver);
4400 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3069ee9b
VP		 4401 flush_workqueue(workq);
4402 destroy_workqueue(workq);
b8ff05a9
DM		 4403}
4404
4405module_init(cxgb4_init_module);
4406module_exit(cxgb4_cleanup_module);
Linux 6.x block layer and io_uring tree(s)