drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/init.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/netdevice.h>
  41 #include <linux/etherdevice.h>
  42 #include <linux/if_vlan.h>
  43 #include <linux/mdio.h>
  44 #include <linux/sockios.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/proc_fs.h>
  47 #include <linux/rtnetlink.h>
  48 #include <linux/firmware.h>
  49 #include <linux/log2.h>
  50 #include <linux/stringify.h>
  51 #include <linux/sched.h>
  52 #include <linux/slab.h>
  53 #include <asm/uaccess.h>
  54
  55 #include "common.h"
  56 #include "cxgb3_ioctl.h"
  57 #include "regs.h"
  58 #include "cxgb3_offload.h"
  59 #include "version.h"
  60
  61 #include "cxgb3_ctl_defs.h"
  62 #include "t3_cpl.h"
  63 #include "firmware_exports.h"
  64
  65 enum {
  66         MAX_TXQ_ENTRIES = 16384,
  67         MAX_CTRL_TXQ_ENTRIES = 1024,
  68         MAX_RSPQ_ENTRIES = 16384,
  69         MAX_RX_BUFFERS = 16384,
  70         MAX_RX_JUMBO_BUFFERS = 16384,
  71         MIN_TXQ_ENTRIES = 4,
  72         MIN_CTRL_TXQ_ENTRIES = 4,
  73         MIN_RSPQ_ENTRIES = 32,
  74         MIN_FL_ENTRIES = 32
  75 };
  76
  77 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  78
  79 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  80                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  81                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  82
  83 #define EEPROM_MAGIC 0x38E2F10C
  84
  85 #define CH_DEVICE(devid, idx) \
  86         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  87
  88 static const struct pci_device_id cxgb3_pci_tbl[] = {
  89         CH_DEVICE(0x20, 0),     /* PE9000 */
  90         CH_DEVICE(0x21, 1),     /* T302E */
  91         CH_DEVICE(0x22, 2),     /* T310E */
  92         CH_DEVICE(0x23, 3),     /* T320X */
  93         CH_DEVICE(0x24, 1),     /* T302X */
  94         CH_DEVICE(0x25, 3),     /* T320E */
  95         CH_DEVICE(0x26, 2),     /* T310X */
  96         CH_DEVICE(0x30, 2),     /* T3B10 */
  97         CH_DEVICE(0x31, 3),     /* T3B20 */
  98         CH_DEVICE(0x32, 1),     /* T3B02 */
  99         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
 100         CH_DEVICE(0x36, 3),     /* S320E-CR */
 101         CH_DEVICE(0x37, 7),     /* N320E-G2 */
 102         {0,}
 103 };
 104
 105 MODULE_DESCRIPTION(DRV_DESC);
 106 MODULE_AUTHOR("Chelsio Communications");
 107 MODULE_LICENSE("Dual BSD/GPL");
 108 MODULE_VERSION(DRV_VERSION);
 109 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 110
 111 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 112
 113 module_param(dflt_msg_enable, int, 0644);
 114 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 115
 116 /*
 117  * The driver uses the best interrupt scheme available on a platform in the
 118  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 119  * of these schemes the driver may consider as follows:
 120  *
 121  * msi = 2: choose from among all three options
 122  * msi = 1: only consider MSI and pin interrupts
 123  * msi = 0: force pin interrupts
 124  */
 125 static int msi = 2;
 126
 127 module_param(msi, int, 0644);
 128 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 129
 130 /*
 131  * The driver enables offload as a default.
 132  * To disable it, use ofld_disable = 1.
 133  */
 134
 135 static int ofld_disable = 0;
 136
 137 module_param(ofld_disable, int, 0644);
 138 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 139
 140 /*
 141  * We have work elements that we need to cancel when an interface is taken
 142  * down.  Normally the work elements would be executed by keventd but that
 143  * can deadlock because of linkwatch.  If our close method takes the rtnl
 144  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 145  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 146  * for our work to complete.  Get our own work queue to solve this.
 147  */
 148 struct workqueue_struct *cxgb3_wq;
 149
 150 /**
 151  *      link_report - show link status and link speed/duplex
 152  *      @p: the port whose settings are to be reported
 153  *
 154  *      Shows the link status, speed, and duplex of a port.
 155  */
 156 static void link_report(struct net_device *dev)
 157 {
 158         if (!netif_carrier_ok(dev))
 159                 netdev_info(dev, "link down\n");
 160         else {
 161                 const char *s = "10Mbps";
 162                 const struct port_info *p = netdev_priv(dev);
 163
 164                 switch (p->link_config.speed) {
 165                 case SPEED_10000:
 166                         s = "10Gbps";
 167                         break;
 168                 case SPEED_1000:
 169                         s = "1000Mbps";
 170                         break;
 171                 case SPEED_100:
 172                         s = "100Mbps";
 173                         break;
 174                 }
 175
 176                 netdev_info(dev, "link up, %s, %s-duplex\n",
 177                             s, p->link_config.duplex == DUPLEX_FULL
 178                             ? "full" : "half");
 179         }
 180 }
 181
 182 static void enable_tx_fifo_drain(struct adapter *adapter,
 183                                  struct port_info *pi)
 184 {
 185         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 186                          F_ENDROPPKT);
 187         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 188         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 189         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 190 }
 191
 192 static void disable_tx_fifo_drain(struct adapter *adapter,
 193                                   struct port_info *pi)
 194 {
 195         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 196                          F_ENDROPPKT, 0);
 197 }
 198
 199 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 200 {
 201         struct net_device *dev = adap->port[port_id];
 202         struct port_info *pi = netdev_priv(dev);
 203
 204         if (state == netif_carrier_ok(dev))
 205                 return;
 206
 207         if (state) {
 208                 struct cmac *mac = &pi->mac;
 209
 210                 netif_carrier_on(dev);
 211
 212                 disable_tx_fifo_drain(adap, pi);
 213
 214                 /* Clear local faults */
 215                 t3_xgm_intr_disable(adap, pi->port_id);
 216                 t3_read_reg(adap, A_XGM_INT_STATUS +
 217                                     pi->mac.offset);
 218                 t3_write_reg(adap,
 219                              A_XGM_INT_CAUSE + pi->mac.offset,
 220                              F_XGM_INT);
 221
 222                 t3_set_reg_field(adap,
 223                                  A_XGM_INT_ENABLE +
 224                                  pi->mac.offset,
 225                                  F_XGM_INT, F_XGM_INT);
 226                 t3_xgm_intr_enable(adap, pi->port_id);
 227
 228                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 229         } else {
 230                 netif_carrier_off(dev);
 231
 232                 /* Flush TX FIFO */
 233                 enable_tx_fifo_drain(adap, pi);
 234         }
 235         link_report(dev);
 236 }
 237
 238 /**
 239  *      t3_os_link_changed - handle link status changes
 240  *      @adapter: the adapter associated with the link change
 241  *      @port_id: the port index whose limk status has changed
 242  *      @link_stat: the new status of the link
 243  *      @speed: the new speed setting
 244  *      @duplex: the new duplex setting
 245  *      @pause: the new flow-control setting
 246  *
 247  *      This is the OS-dependent handler for link status changes.  The OS
 248  *      neutral handler takes care of most of the processing for these events,
 249  *      then calls this handler for any OS-specific processing.
 250  */
 251 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 252                         int speed, int duplex, int pause)
 253 {
 254         struct net_device *dev = adapter->port[port_id];
 255         struct port_info *pi = netdev_priv(dev);
 256         struct cmac *mac = &pi->mac;
 257
 258         /* Skip changes from disabled ports. */
 259         if (!netif_running(dev))
 260                 return;
 261
 262         if (link_stat != netif_carrier_ok(dev)) {
 263                 if (link_stat) {
 264                         disable_tx_fifo_drain(adapter, pi);
 265
 266                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 267
 268                         /* Clear local faults */
 269                         t3_xgm_intr_disable(adapter, pi->port_id);
 270                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 271                                     pi->mac.offset);
 272                         t3_write_reg(adapter,
 273                                      A_XGM_INT_CAUSE + pi->mac.offset,
 274                                      F_XGM_INT);
 275
 276                         t3_set_reg_field(adapter,
 277                                          A_XGM_INT_ENABLE + pi->mac.offset,
 278                                          F_XGM_INT, F_XGM_INT);
 279                         t3_xgm_intr_enable(adapter, pi->port_id);
 280
 281                         netif_carrier_on(dev);
 282                 } else {
 283                         netif_carrier_off(dev);
 284
 285                         t3_xgm_intr_disable(adapter, pi->port_id);
 286                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 287                         t3_set_reg_field(adapter,
 288                                          A_XGM_INT_ENABLE + pi->mac.offset,
 289                                          F_XGM_INT, 0);
 290
 291                         if (is_10G(adapter))
 292                                 pi->phy.ops->power_down(&pi->phy, 1);
 293
 294                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 295                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 296                         t3_link_start(&pi->phy, mac, &pi->link_config);
 297
 298                         /* Flush TX FIFO */
 299                         enable_tx_fifo_drain(adapter, pi);
 300                 }
 301
 302                 link_report(dev);
 303         }
 304 }
 305
 306 /**
 307  *      t3_os_phymod_changed - handle PHY module changes
 308  *      @phy: the PHY reporting the module change
 309  *      @mod_type: new module type
 310  *
 311  *      This is the OS-dependent handler for PHY module changes.  It is
 312  *      invoked when a PHY module is removed or inserted for any OS-specific
 313  *      processing.
 314  */
 315 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 316 {
 317         static const char *mod_str[] = {
 318                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 319         };
 320
 321         const struct net_device *dev = adap->port[port_id];
 322         const struct port_info *pi = netdev_priv(dev);
 323
 324         if (pi->phy.modtype == phy_modtype_none)
 325                 netdev_info(dev, "PHY module unplugged\n");
 326         else
 327                 netdev_info(dev, "%s PHY module inserted\n",
 328                             mod_str[pi->phy.modtype]);
 329 }
 330
 331 static void cxgb_set_rxmode(struct net_device *dev)
 332 {
 333         struct port_info *pi = netdev_priv(dev);
 334
 335         t3_mac_set_rx_mode(&pi->mac, dev);
 336 }
 337
 338 /**
 339  *      link_start - enable a port
 340  *      @dev: the device to enable
 341  *
 342  *      Performs the MAC and PHY actions needed to enable a port.
 343  */
 344 static void link_start(struct net_device *dev)
 345 {
 346         struct port_info *pi = netdev_priv(dev);
 347         struct cmac *mac = &pi->mac;
 348
 349         t3_mac_reset(mac);
 350         t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
 351         t3_mac_set_mtu(mac, dev->mtu);
 352         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
 353         t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
 354         t3_mac_set_rx_mode(mac, dev);
 355         t3_link_start(&pi->phy, mac, &pi->link_config);
 356         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 357 }
 358
 359 static inline void cxgb_disable_msi(struct adapter *adapter)
 360 {
 361         if (adapter->flags & USING_MSIX) {
 362                 pci_disable_msix(adapter->pdev);
 363                 adapter->flags &= ~USING_MSIX;
 364         } else if (adapter->flags & USING_MSI) {
 365                 pci_disable_msi(adapter->pdev);
 366                 adapter->flags &= ~USING_MSI;
 367         }
 368 }
 369
 370 /*
 371  * Interrupt handler for asynchronous events used with MSI-X.
 372  */
 373 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 374 {
 375         t3_slow_intr_handler(cookie);
 376         return IRQ_HANDLED;
 377 }
 378
 379 /*
 380  * Name the MSI-X interrupts.
 381  */
 382 static void name_msix_vecs(struct adapter *adap)
 383 {
 384         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 385
 386         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 387         adap->msix_info[0].desc[n] = 0;
 388
 389         for_each_port(adap, j) {
 390                 struct net_device *d = adap->port[j];
 391                 const struct port_info *pi = netdev_priv(d);
 392
 393                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 394                         snprintf(adap->msix_info[msi_idx].desc, n,
 395                                  "%s-%d", d->name, pi->first_qset + i);
 396                         adap->msix_info[msi_idx].desc[n] = 0;
 397                 }
 398         }
 399 }
 400
 401 static int request_msix_data_irqs(struct adapter *adap)
 402 {
 403         int i, j, err, qidx = 0;
 404
 405         for_each_port(adap, i) {
 406                 int nqsets = adap2pinfo(adap, i)->nqsets;
 407
 408                 for (j = 0; j < nqsets; ++j) {
 409                         err = request_irq(adap->msix_info[qidx + 1].vec,
 410                                           t3_intr_handler(adap,
 411                                                           adap->sge.qs[qidx].
 412                                                           rspq.polling), 0,
 413                                           adap->msix_info[qidx + 1].desc,
 414                                           &adap->sge.qs[qidx]);
 415                         if (err) {
 416                                 while (--qidx >= 0)
 417                                         free_irq(adap->msix_info[qidx + 1].vec,
 418                                                  &adap->sge.qs[qidx]);
 419                                 return err;
 420                         }
 421                         qidx++;
 422                 }
 423         }
 424         return 0;
 425 }
 426
 427 static void free_irq_resources(struct adapter *adapter)
 428 {
 429         if (adapter->flags & USING_MSIX) {
 430                 int i, n = 0;
 431
 432                 free_irq(adapter->msix_info[0].vec, adapter);
 433                 for_each_port(adapter, i)
 434                         n += adap2pinfo(adapter, i)->nqsets;
 435
 436                 for (i = 0; i < n; ++i)
 437                         free_irq(adapter->msix_info[i + 1].vec,
 438                                  &adapter->sge.qs[i]);
 439         } else
 440                 free_irq(adapter->pdev->irq, adapter);
 441 }
 442
 443 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 444                               unsigned long n)
 445 {
 446         int attempts = 10;
 447
 448         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 449                 if (!--attempts)
 450                         return -ETIMEDOUT;
 451                 msleep(10);
 452         }
 453         return 0;
 454 }
 455
 456 static int init_tp_parity(struct adapter *adap)
 457 {
 458         int i;
 459         struct sk_buff *skb;
 460         struct cpl_set_tcb_field *greq;
 461         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 462
 463         t3_tp_set_offload_mode(adap, 1);
 464
 465         for (i = 0; i < 16; i++) {
 466                 struct cpl_smt_write_req *req;
 467
 468                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 469                 if (!skb)
 470                         skb = adap->nofail_skb;
 471                 if (!skb)
 472                         goto alloc_skb_fail;
 473
 474                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 475                 memset(req, 0, sizeof(*req));
 476                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 477                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 478                 req->mtu_idx = NMTUS - 1;
 479                 req->iff = i;
 480                 t3_mgmt_tx(adap, skb);
 481                 if (skb == adap->nofail_skb) {
 482                         await_mgmt_replies(adap, cnt, i + 1);
 483                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 484                         if (!adap->nofail_skb)
 485                                 goto alloc_skb_fail;
 486                 }
 487         }
 488
 489         for (i = 0; i < 2048; i++) {
 490                 struct cpl_l2t_write_req *req;
 491
 492                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 493                 if (!skb)
 494                         skb = adap->nofail_skb;
 495                 if (!skb)
 496                         goto alloc_skb_fail;
 497
 498                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 499                 memset(req, 0, sizeof(*req));
 500                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 501                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 502                 req->params = htonl(V_L2T_W_IDX(i));
 503                 t3_mgmt_tx(adap, skb);
 504                 if (skb == adap->nofail_skb) {
 505                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 506                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 507                         if (!adap->nofail_skb)
 508                                 goto alloc_skb_fail;
 509                 }
 510         }
 511
 512         for (i = 0; i < 2048; i++) {
 513                 struct cpl_rte_write_req *req;
 514
 515                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 516                 if (!skb)
 517                         skb = adap->nofail_skb;
 518                 if (!skb)
 519                         goto alloc_skb_fail;
 520
 521                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 522                 memset(req, 0, sizeof(*req));
 523                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 524                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 525                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 526                 t3_mgmt_tx(adap, skb);
 527                 if (skb == adap->nofail_skb) {
 528                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 529                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 530                         if (!adap->nofail_skb)
 531                                 goto alloc_skb_fail;
 532                 }
 533         }
 534
 535         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 536         if (!skb)
 537                 skb = adap->nofail_skb;
 538         if (!skb)
 539                 goto alloc_skb_fail;
 540
 541         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 542         memset(greq, 0, sizeof(*greq));
 543         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 544         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 545         greq->mask = cpu_to_be64(1);
 546         t3_mgmt_tx(adap, skb);
 547
 548         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 549         if (skb == adap->nofail_skb) {
 550                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 551                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 552         }
 553
 554         t3_tp_set_offload_mode(adap, 0);
 555         return i;
 556
 557 alloc_skb_fail:
 558         t3_tp_set_offload_mode(adap, 0);
 559         return -ENOMEM;
 560 }
 561
 562 /**
 563  *      setup_rss - configure RSS
 564  *      @adap: the adapter
 565  *
 566  *      Sets up RSS to distribute packets to multiple receive queues.  We
 567  *      configure the RSS CPU lookup table to distribute to the number of HW
 568  *      receive queues, and the response queue lookup table to narrow that
 569  *      down to the response queues actually configured for each port.
 570  *      We always configure the RSS mapping for two ports since the mapping
 571  *      table has plenty of entries.
 572  */
 573 static void setup_rss(struct adapter *adap)
 574 {
 575         int i;
 576         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 577         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 578         u8 cpus[SGE_QSETS + 1];
 579         u16 rspq_map[RSS_TABLE_SIZE + 1];
 580
 581         for (i = 0; i < SGE_QSETS; ++i)
 582                 cpus[i] = i;
 583         cpus[SGE_QSETS] = 0xff; /* terminator */
 584
 585         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 586                 rspq_map[i] = i % nq0;
 587                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 588         }
 589         rspq_map[RSS_TABLE_SIZE] = 0xffff; /* terminator */
 590
 591         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 592                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 593                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 594 }
 595
 596 static void ring_dbs(struct adapter *adap)
 597 {
 598         int i, j;
 599
 600         for (i = 0; i < SGE_QSETS; i++) {
 601                 struct sge_qset *qs = &adap->sge.qs[i];
 602
 603                 if (qs->adap)
 604                         for (j = 0; j < SGE_TXQ_PER_SET; j++)
 605                                 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
 606         }
 607 }
 608
 609 static void init_napi(struct adapter *adap)
 610 {
 611         int i;
 612
 613         for (i = 0; i < SGE_QSETS; i++) {
 614                 struct sge_qset *qs = &adap->sge.qs[i];
 615
 616                 if (qs->adap)
 617                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 618                                        64);
 619         }
 620
 621         /*
 622          * netif_napi_add() can be called only once per napi_struct because it
 623          * adds each new napi_struct to a list.  Be careful not to call it a
 624          * second time, e.g., during EEH recovery, by making a note of it.
 625          */
 626         adap->flags |= NAPI_INIT;
 627 }
 628
 629 /*
 630  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 631  * both netdevices representing interfaces and the dummy ones for the extra
 632  * queues.
 633  */
 634 static void quiesce_rx(struct adapter *adap)
 635 {
 636         int i;
 637
 638         for (i = 0; i < SGE_QSETS; i++)
 639                 if (adap->sge.qs[i].adap)
 640                         napi_disable(&adap->sge.qs[i].napi);
 641 }
 642
 643 static void enable_all_napi(struct adapter *adap)
 644 {
 645         int i;
 646         for (i = 0; i < SGE_QSETS; i++)
 647                 if (adap->sge.qs[i].adap)
 648                         napi_enable(&adap->sge.qs[i].napi);
 649 }
 650
 651 /**
 652  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 653  *      @adap: the adapter
 654  *
 655  *      Determines how many sets of SGE queues to use and initializes them.
 656  *      We support multiple queue sets per port if we have MSI-X, otherwise
 657  *      just one queue set per port.
 658  */
 659 static int setup_sge_qsets(struct adapter *adap)
 660 {
 661         int i, j, err, irq_idx = 0, qset_idx = 0;
 662         unsigned int ntxq = SGE_TXQ_PER_SET;
 663
 664         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 665                 irq_idx = -1;
 666
 667         for_each_port(adap, i) {
 668                 struct net_device *dev = adap->port[i];
 669                 struct port_info *pi = netdev_priv(dev);
 670
 671                 pi->qs = &adap->sge.qs[pi->first_qset];
 672                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 673                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 674                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 675                                                              irq_idx,
 676                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 677                                 netdev_get_tx_queue(dev, j));
 678                         if (err) {
 679                                 t3_free_sge_resources(adap);
 680                                 return err;
 681                         }
 682                 }
 683         }
 684
 685         return 0;
 686 }
 687
 688 static ssize_t attr_show(struct device *d, char *buf,
 689                          ssize_t(*format) (struct net_device *, char *))
 690 {
 691         ssize_t len;
 692
 693         /* Synchronize with ioctls that may shut down the device */
 694         rtnl_lock();
 695         len = (*format) (to_net_dev(d), buf);
 696         rtnl_unlock();
 697         return len;
 698 }
 699
 700 static ssize_t attr_store(struct device *d,
 701                           const char *buf, size_t len,
 702                           ssize_t(*set) (struct net_device *, unsigned int),
 703                           unsigned int min_val, unsigned int max_val)
 704 {
 705         ssize_t ret;
 706         unsigned int val;
 707
 708         if (!capable(CAP_NET_ADMIN))
 709                 return -EPERM;
 710
 711         ret = kstrtouint(buf, 0, &val);
 712         if (ret)
 713                 return ret;
 714         if (val < min_val || val > max_val)
 715                 return -EINVAL;
 716
 717         rtnl_lock();
 718         ret = (*set) (to_net_dev(d), val);
 719         if (!ret)
 720                 ret = len;
 721         rtnl_unlock();
 722         return ret;
 723 }
 724
 725 #define CXGB3_SHOW(name, val_expr) \
 726 static ssize_t format_##name(struct net_device *dev, char *buf) \
 727 { \
 728         struct port_info *pi = netdev_priv(dev); \
 729         struct adapter *adap = pi->adapter; \
 730         return sprintf(buf, "%u\n", val_expr); \
 731 } \
 732 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 733                            char *buf) \
 734 { \
 735         return attr_show(d, buf, format_##name); \
 736 }
 737
 738 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 739 {
 740         struct port_info *pi = netdev_priv(dev);
 741         struct adapter *adap = pi->adapter;
 742         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 743
 744         if (adap->flags & FULL_INIT_DONE)
 745                 return -EBUSY;
 746         if (val && adap->params.rev == 0)
 747                 return -EINVAL;
 748         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 749             min_tids)
 750                 return -EINVAL;
 751         adap->params.mc5.nfilters = val;
 752         return 0;
 753 }
 754
 755 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 756                               const char *buf, size_t len)
 757 {
 758         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 759 }
 760
 761 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 762 {
 763         struct port_info *pi = netdev_priv(dev);
 764         struct adapter *adap = pi->adapter;
 765
 766         if (adap->flags & FULL_INIT_DONE)
 767                 return -EBUSY;
 768         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 769             MC5_MIN_TIDS)
 770                 return -EINVAL;
 771         adap->params.mc5.nservers = val;
 772         return 0;
 773 }
 774
 775 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 776                               const char *buf, size_t len)
 777 {
 778         return attr_store(d, buf, len, set_nservers, 0, ~0);
 779 }
 780
 781 #define CXGB3_ATTR_R(name, val_expr) \
 782 CXGB3_SHOW(name, val_expr) \
 783 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 784
 785 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 786 CXGB3_SHOW(name, val_expr) \
 787 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 788
 789 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 790 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 791 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 792
 793 static struct attribute *cxgb3_attrs[] = {
 794         &dev_attr_cam_size.attr,
 795         &dev_attr_nfilters.attr,
 796         &dev_attr_nservers.attr,
 797         NULL
 798 };
 799
 800 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 801
 802 static ssize_t tm_attr_show(struct device *d,
 803                             char *buf, int sched)
 804 {
 805         struct port_info *pi = netdev_priv(to_net_dev(d));
 806         struct adapter *adap = pi->adapter;
 807         unsigned int v, addr, bpt, cpt;
 808         ssize_t len;
 809
 810         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 811         rtnl_lock();
 812         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 813         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 814         if (sched & 1)
 815                 v >>= 16;
 816         bpt = (v >> 8) & 0xff;
 817         cpt = v & 0xff;
 818         if (!cpt)
 819                 len = sprintf(buf, "disabled\n");
 820         else {
 821                 v = (adap->params.vpd.cclk * 1000) / cpt;
 822                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 823         }
 824         rtnl_unlock();
 825         return len;
 826 }
 827
 828 static ssize_t tm_attr_store(struct device *d,
 829                              const char *buf, size_t len, int sched)
 830 {
 831         struct port_info *pi = netdev_priv(to_net_dev(d));
 832         struct adapter *adap = pi->adapter;
 833         unsigned int val;
 834         ssize_t ret;
 835
 836         if (!capable(CAP_NET_ADMIN))
 837                 return -EPERM;
 838
 839         ret = kstrtouint(buf, 0, &val);
 840         if (ret)
 841                 return ret;
 842         if (val > 10000000)
 843                 return -EINVAL;
 844
 845         rtnl_lock();
 846         ret = t3_config_sched(adap, val, sched);
 847         if (!ret)
 848                 ret = len;
 849         rtnl_unlock();
 850         return ret;
 851 }
 852
 853 #define TM_ATTR(name, sched) \
 854 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 855                            char *buf) \
 856 { \
 857         return tm_attr_show(d, buf, sched); \
 858 } \
 859 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 860                             const char *buf, size_t len) \
 861 { \
 862         return tm_attr_store(d, buf, len, sched); \
 863 } \
 864 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 865
 866 TM_ATTR(sched0, 0);
 867 TM_ATTR(sched1, 1);
 868 TM_ATTR(sched2, 2);
 869 TM_ATTR(sched3, 3);
 870 TM_ATTR(sched4, 4);
 871 TM_ATTR(sched5, 5);
 872 TM_ATTR(sched6, 6);
 873 TM_ATTR(sched7, 7);
 874
 875 static struct attribute *offload_attrs[] = {
 876         &dev_attr_sched0.attr,
 877         &dev_attr_sched1.attr,
 878         &dev_attr_sched2.attr,
 879         &dev_attr_sched3.attr,
 880         &dev_attr_sched4.attr,
 881         &dev_attr_sched5.attr,
 882         &dev_attr_sched6.attr,
 883         &dev_attr_sched7.attr,
 884         NULL
 885 };
 886
 887 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 888
 889 /*
 890  * Sends an sk_buff to an offload queue driver
 891  * after dealing with any active network taps.
 892  */
 893 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 894 {
 895         int ret;
 896
 897         local_bh_disable();
 898         ret = t3_offload_tx(tdev, skb);
 899         local_bh_enable();
 900         return ret;
 901 }
 902
 903 static int write_smt_entry(struct adapter *adapter, int idx)
 904 {
 905         struct cpl_smt_write_req *req;
 906         struct port_info *pi = netdev_priv(adapter->port[idx]);
 907         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 908
 909         if (!skb)
 910                 return -ENOMEM;
 911
 912         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 913         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 914         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 915         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 916         req->iff = idx;
 917         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 918         memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
 919         skb->priority = 1;
 920         offload_tx(&adapter->tdev, skb);
 921         return 0;
 922 }
 923
 924 static int init_smt(struct adapter *adapter)
 925 {
 926         int i;
 927
 928         for_each_port(adapter, i)
 929             write_smt_entry(adapter, i);
 930         return 0;
 931 }
 932
 933 static void init_port_mtus(struct adapter *adapter)
 934 {
 935         unsigned int mtus = adapter->port[0]->mtu;
 936
 937         if (adapter->port[1])
 938                 mtus |= adapter->port[1]->mtu << 16;
 939         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 940 }
 941
 942 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 943                               int hi, int port)
 944 {
 945         struct sk_buff *skb;
 946         struct mngt_pktsched_wr *req;
 947         int ret;
 948
 949         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 950         if (!skb)
 951                 skb = adap->nofail_skb;
 952         if (!skb)
 953                 return -ENOMEM;
 954
 955         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 956         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 957         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 958         req->sched = sched;
 959         req->idx = qidx;
 960         req->min = lo;
 961         req->max = hi;
 962         req->binding = port;
 963         ret = t3_mgmt_tx(adap, skb);
 964         if (skb == adap->nofail_skb) {
 965                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 966                                              GFP_KERNEL);
 967                 if (!adap->nofail_skb)
 968                         ret = -ENOMEM;
 969         }
 970
 971         return ret;
 972 }
 973
 974 static int bind_qsets(struct adapter *adap)
 975 {
 976         int i, j, err = 0;
 977
 978         for_each_port(adap, i) {
 979                 const struct port_info *pi = adap2pinfo(adap, i);
 980
 981                 for (j = 0; j < pi->nqsets; ++j) {
 982                         int ret = send_pktsched_cmd(adap, 1,
 983                                                     pi->first_qset + j, -1,
 984                                                     -1, i);
 985                         if (ret)
 986                                 err = ret;
 987                 }
 988         }
 989
 990         return err;
 991 }
 992
 993 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."                    \
 994         __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
 995 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
 996 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."                \
 997         __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
 998 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
 999 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
1000 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
1001 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
1002 MODULE_FIRMWARE(FW_FNAME);
1003 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1004 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1005 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1006 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1007 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1008
1009 static inline const char *get_edc_fw_name(int edc_idx)
1010 {
1011         const char *fw_name = NULL;
1012
1013         switch (edc_idx) {
1014         case EDC_OPT_AEL2005:
1015                 fw_name = AEL2005_OPT_EDC_NAME;
1016                 break;
1017         case EDC_TWX_AEL2005:
1018                 fw_name = AEL2005_TWX_EDC_NAME;
1019                 break;
1020         case EDC_TWX_AEL2020:
1021                 fw_name = AEL2020_TWX_EDC_NAME;
1022                 break;
1023         }
1024         return fw_name;
1025 }
1026
1027 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1028 {
1029         struct adapter *adapter = phy->adapter;
1030         const struct firmware *fw;
1031         const char *fw_name;
1032         u32 csum;
1033         const __be32 *p;
1034         u16 *cache = phy->phy_cache;
1035         int i, ret = -EINVAL;
1036
1037         fw_name = get_edc_fw_name(edc_idx);
1038         if (fw_name)
1039                 ret = request_firmware(&fw, fw_name, &adapter->pdev->dev);
1040         if (ret < 0) {
1041                 dev_err(&adapter->pdev->dev,
1042                         "could not upgrade firmware: unable to load %s\n",
1043                         fw_name);
1044                 return ret;
1045         }
1046
1047         /* check size, take checksum in account */
1048         if (fw->size > size + 4) {
1049                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1050                        (unsigned int)fw->size, size + 4);
1051                 ret = -EINVAL;
1052         }
1053
1054         /* compute checksum */
1055         p = (const __be32 *)fw->data;
1056         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1057                 csum += ntohl(p[i]);
1058
1059         if (csum != 0xffffffff) {
1060                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1061                        csum);
1062                 ret = -EINVAL;
1063         }
1064
1065         for (i = 0; i < size / 4 ; i++) {
1066                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1067                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1068         }
1069
1070         release_firmware(fw);
1071
1072         return ret;
1073 }
1074
1075 static int upgrade_fw(struct adapter *adap)
1076 {
1077         int ret;
1078         const struct firmware *fw;
1079         struct device *dev = &adap->pdev->dev;
1080
1081         ret = request_firmware(&fw, FW_FNAME, dev);
1082         if (ret < 0) {
1083                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1084                         FW_FNAME);
1085                 return ret;
1086         }
1087         ret = t3_load_fw(adap, fw->data, fw->size);
1088         release_firmware(fw);
1089
1090         if (ret == 0)
1091                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1092                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1093         else
1094                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1095                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1096
1097         return ret;
1098 }
1099
1100 static inline char t3rev2char(struct adapter *adapter)
1101 {
1102         char rev = 0;
1103
1104         switch(adapter->params.rev) {
1105         case T3_REV_B:
1106         case T3_REV_B2:
1107                 rev = 'b';
1108                 break;
1109         case T3_REV_C:
1110                 rev = 'c';
1111                 break;
1112         }
1113         return rev;
1114 }
1115
1116 static int update_tpsram(struct adapter *adap)
1117 {
1118         const struct firmware *tpsram;
1119         char buf[64];
1120         struct device *dev = &adap->pdev->dev;
1121         int ret;
1122         char rev;
1123
1124         rev = t3rev2char(adap);
1125         if (!rev)
1126                 return 0;
1127
1128         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1129
1130         ret = request_firmware(&tpsram, buf, dev);
1131         if (ret < 0) {
1132                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1133                         buf);
1134                 return ret;
1135         }
1136
1137         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1138         if (ret)
1139                 goto release_tpsram;
1140
1141         ret = t3_set_proto_sram(adap, tpsram->data);
1142         if (ret == 0)
1143                 dev_info(dev,
1144                          "successful update of protocol engine "
1145                          "to %d.%d.%d\n",
1146                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1147         else
1148                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1149                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1150         if (ret)
1151                 dev_err(dev, "loading protocol SRAM failed\n");
1152
1153 release_tpsram:
1154         release_firmware(tpsram);
1155
1156         return ret;
1157 }
1158
1159 /**
1160  * t3_synchronize_rx - wait for current Rx processing on a port to complete
1161  * @adap: the adapter
1162  * @p: the port
1163  *
1164  * Ensures that current Rx processing on any of the queues associated with
1165  * the given port completes before returning.  We do this by acquiring and
1166  * releasing the locks of the response queues associated with the port.
1167  */
1168 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1169 {
1170         int i;
1171
1172         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1173                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
1174
1175                 spin_lock_irq(&q->lock);
1176                 spin_unlock_irq(&q->lock);
1177         }
1178 }
1179
1180 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1181 {
1182         struct port_info *pi = netdev_priv(dev);
1183         struct adapter *adapter = pi->adapter;
1184
1185         if (adapter->params.rev > 0) {
1186                 t3_set_vlan_accel(adapter, 1 << pi->port_id,
1187                                   features & NETIF_F_HW_VLAN_CTAG_RX);
1188         } else {
1189                 /* single control for all ports */
1190                 unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX;
1191
1192                 for_each_port(adapter, i)
1193                         have_vlans |=
1194                                 adapter->port[i]->features &
1195                                 NETIF_F_HW_VLAN_CTAG_RX;
1196
1197                 t3_set_vlan_accel(adapter, 1, have_vlans);
1198         }
1199         t3_synchronize_rx(adapter, pi);
1200 }
1201
1202 /**
1203  *      cxgb_up - enable the adapter
1204  *      @adapter: adapter being enabled
1205  *
1206  *      Called when the first port is enabled, this function performs the
1207  *      actions necessary to make an adapter operational, such as completing
1208  *      the initialization of HW modules, and enabling interrupts.
1209  *
1210  *      Must be called with the rtnl lock held.
1211  */
1212 static int cxgb_up(struct adapter *adap)
1213 {
1214         int i, err;
1215
1216         if (!(adap->flags & FULL_INIT_DONE)) {
1217                 err = t3_check_fw_version(adap);
1218                 if (err == -EINVAL) {
1219                         err = upgrade_fw(adap);
1220                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1221                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1222                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1223                 }
1224
1225                 err = t3_check_tpsram_version(adap);
1226                 if (err == -EINVAL) {
1227                         err = update_tpsram(adap);
1228                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1229                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1230                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1231                 }
1232
1233                 /*
1234                  * Clear interrupts now to catch errors if t3_init_hw fails.
1235                  * We clear them again later as initialization may trigger
1236                  * conditions that can interrupt.
1237                  */
1238                 t3_intr_clear(adap);
1239
1240                 err = t3_init_hw(adap, 0);
1241                 if (err)
1242                         goto out;
1243
1244                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1245                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1246
1247                 err = setup_sge_qsets(adap);
1248                 if (err)
1249                         goto out;
1250
1251                 for_each_port(adap, i)
1252                         cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1253
1254                 setup_rss(adap);
1255                 if (!(adap->flags & NAPI_INIT))
1256                         init_napi(adap);
1257
1258                 t3_start_sge_timers(adap);
1259                 adap->flags |= FULL_INIT_DONE;
1260         }
1261
1262         t3_intr_clear(adap);
1263
1264         if (adap->flags & USING_MSIX) {
1265                 name_msix_vecs(adap);
1266                 err = request_irq(adap->msix_info[0].vec,
1267                                   t3_async_intr_handler, 0,
1268                                   adap->msix_info[0].desc, adap);
1269                 if (err)
1270                         goto irq_err;
1271
1272                 err = request_msix_data_irqs(adap);
1273                 if (err) {
1274                         free_irq(adap->msix_info[0].vec, adap);
1275                         goto irq_err;
1276                 }
1277         } else if ((err = request_irq(adap->pdev->irq,
1278                                       t3_intr_handler(adap,
1279                                                       adap->sge.qs[0].rspq.
1280                                                       polling),
1281                                       (adap->flags & USING_MSI) ?
1282                                        0 : IRQF_SHARED,
1283                                       adap->name, adap)))
1284                 goto irq_err;
1285
1286         enable_all_napi(adap);
1287         t3_sge_start(adap);
1288         t3_intr_enable(adap);
1289
1290         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1291             is_offload(adap) && init_tp_parity(adap) == 0)
1292                 adap->flags |= TP_PARITY_INIT;
1293
1294         if (adap->flags & TP_PARITY_INIT) {
1295                 t3_write_reg(adap, A_TP_INT_CAUSE,
1296                              F_CMCACHEPERR | F_ARPLUTPERR);
1297                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1298         }
1299
1300         if (!(adap->flags & QUEUES_BOUND)) {
1301                 int ret = bind_qsets(adap);
1302
1303                 if (ret < 0) {
1304                         CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1305                         t3_intr_disable(adap);
1306                         free_irq_resources(adap);
1307                         err = ret;
1308                         goto out;
1309                 }
1310                 adap->flags |= QUEUES_BOUND;
1311         }
1312
1313 out:
1314         return err;
1315 irq_err:
1316         CH_ERR(adap, "request_irq failed, err %d\n", err);
1317         goto out;
1318 }
1319
1320 /*
1321  * Release resources when all the ports and offloading have been stopped.
1322  */
1323 static void cxgb_down(struct adapter *adapter, int on_wq)
1324 {
1325         t3_sge_stop(adapter);
1326         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1327         t3_intr_disable(adapter);
1328         spin_unlock_irq(&adapter->work_lock);
1329
1330         free_irq_resources(adapter);
1331         quiesce_rx(adapter);
1332         t3_sge_stop(adapter);
1333         if (!on_wq)
1334                 flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1335 }
1336
1337 static void schedule_chk_task(struct adapter *adap)
1338 {
1339         unsigned int timeo;
1340
1341         timeo = adap->params.linkpoll_period ?
1342             (HZ * adap->params.linkpoll_period) / 10 :
1343             adap->params.stats_update_period * HZ;
1344         if (timeo)
1345                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1346 }
1347
1348 static int offload_open(struct net_device *dev)
1349 {
1350         struct port_info *pi = netdev_priv(dev);
1351         struct adapter *adapter = pi->adapter;
1352         struct t3cdev *tdev = dev2t3cdev(dev);
1353         int adap_up = adapter->open_device_map & PORT_MASK;
1354         int err;
1355
1356         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1357                 return 0;
1358
1359         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1360                 goto out;
1361
1362         t3_tp_set_offload_mode(adapter, 1);
1363         tdev->lldev = adapter->port[0];
1364         err = cxgb3_offload_activate(adapter);
1365         if (err)
1366                 goto out;
1367
1368         init_port_mtus(adapter);
1369         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1370                      adapter->params.b_wnd,
1371                      adapter->params.rev == 0 ?
1372                      adapter->port[0]->mtu : 0xffff);
1373         init_smt(adapter);
1374
1375         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1376                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1377
1378         /* Call back all registered clients */
1379         cxgb3_add_clients(tdev);
1380
1381 out:
1382         /* restore them in case the offload module has changed them */
1383         if (err) {
1384                 t3_tp_set_offload_mode(adapter, 0);
1385                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1386                 cxgb3_set_dummy_ops(tdev);
1387         }
1388         return err;
1389 }
1390
1391 static int offload_close(struct t3cdev *tdev)
1392 {
1393         struct adapter *adapter = tdev2adap(tdev);
1394         struct t3c_data *td = T3C_DATA(tdev);
1395
1396         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1397                 return 0;
1398
1399         /* Call back all registered clients */
1400         cxgb3_remove_clients(tdev);
1401
1402         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1403
1404         /* Flush work scheduled while releasing TIDs */
1405         flush_work(&td->tid_release_task);
1406
1407         tdev->lldev = NULL;
1408         cxgb3_set_dummy_ops(tdev);
1409         t3_tp_set_offload_mode(adapter, 0);
1410         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1411
1412         if (!adapter->open_device_map)
1413                 cxgb_down(adapter, 0);
1414
1415         cxgb3_offload_deactivate(adapter);
1416         return 0;
1417 }
1418
1419 static int cxgb_open(struct net_device *dev)
1420 {
1421         struct port_info *pi = netdev_priv(dev);
1422         struct adapter *adapter = pi->adapter;
1423         int other_ports = adapter->open_device_map & PORT_MASK;
1424         int err;
1425
1426         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1427                 return err;
1428
1429         set_bit(pi->port_id, &adapter->open_device_map);
1430         if (is_offload(adapter) && !ofld_disable) {
1431                 err = offload_open(dev);
1432                 if (err)
1433                         pr_warn("Could not initialize offload capabilities\n");
1434         }
1435
1436         netif_set_real_num_tx_queues(dev, pi->nqsets);
1437         err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1438         if (err)
1439                 return err;
1440         link_start(dev);
1441         t3_port_intr_enable(adapter, pi->port_id);
1442         netif_tx_start_all_queues(dev);
1443         if (!other_ports)
1444                 schedule_chk_task(adapter);
1445
1446         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1447         return 0;
1448 }
1449
1450 static int __cxgb_close(struct net_device *dev, int on_wq)
1451 {
1452         struct port_info *pi = netdev_priv(dev);
1453         struct adapter *adapter = pi->adapter;
1454
1455
1456         if (!adapter->open_device_map)
1457                 return 0;
1458
1459         /* Stop link fault interrupts */
1460         t3_xgm_intr_disable(adapter, pi->port_id);
1461         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1462
1463         t3_port_intr_disable(adapter, pi->port_id);
1464         netif_tx_stop_all_queues(dev);
1465         pi->phy.ops->power_down(&pi->phy, 1);
1466         netif_carrier_off(dev);
1467         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1468
1469         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1470         clear_bit(pi->port_id, &adapter->open_device_map);
1471         spin_unlock_irq(&adapter->work_lock);
1472
1473         if (!(adapter->open_device_map & PORT_MASK))
1474                 cancel_delayed_work_sync(&adapter->adap_check_task);
1475
1476         if (!adapter->open_device_map)
1477                 cxgb_down(adapter, on_wq);
1478
1479         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1480         return 0;
1481 }
1482
1483 static int cxgb_close(struct net_device *dev)
1484 {
1485         return __cxgb_close(dev, 0);
1486 }
1487
1488 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1489 {
1490         struct port_info *pi = netdev_priv(dev);
1491         struct adapter *adapter = pi->adapter;
1492         struct net_device_stats *ns = &pi->netstats;
1493         const struct mac_stats *pstats;
1494
1495         spin_lock(&adapter->stats_lock);
1496         pstats = t3_mac_update_stats(&pi->mac);
1497         spin_unlock(&adapter->stats_lock);
1498
1499         ns->tx_bytes = pstats->tx_octets;
1500         ns->tx_packets = pstats->tx_frames;
1501         ns->rx_bytes = pstats->rx_octets;
1502         ns->rx_packets = pstats->rx_frames;
1503         ns->multicast = pstats->rx_mcast_frames;
1504
1505         ns->tx_errors = pstats->tx_underrun;
1506         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1507             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1508             pstats->rx_fifo_ovfl;
1509
1510         /* detailed rx_errors */
1511         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1512         ns->rx_over_errors = 0;
1513         ns->rx_crc_errors = pstats->rx_fcs_errs;
1514         ns->rx_frame_errors = pstats->rx_symbol_errs;
1515         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1516         ns->rx_missed_errors = pstats->rx_cong_drops;
1517
1518         /* detailed tx_errors */
1519         ns->tx_aborted_errors = 0;
1520         ns->tx_carrier_errors = 0;
1521         ns->tx_fifo_errors = pstats->tx_underrun;
1522         ns->tx_heartbeat_errors = 0;
1523         ns->tx_window_errors = 0;
1524         return ns;
1525 }
1526
1527 static u32 get_msglevel(struct net_device *dev)
1528 {
1529         struct port_info *pi = netdev_priv(dev);
1530         struct adapter *adapter = pi->adapter;
1531
1532         return adapter->msg_enable;
1533 }
1534
1535 static void set_msglevel(struct net_device *dev, u32 val)
1536 {
1537         struct port_info *pi = netdev_priv(dev);
1538         struct adapter *adapter = pi->adapter;
1539
1540         adapter->msg_enable = val;
1541 }
1542
1543 static const char stats_strings[][ETH_GSTRING_LEN] = {
1544         "TxOctetsOK         ",
1545         "TxFramesOK         ",
1546         "TxMulticastFramesOK",
1547         "TxBroadcastFramesOK",
1548         "TxPauseFrames      ",
1549         "TxUnderrun         ",
1550         "TxExtUnderrun      ",
1551
1552         "TxFrames64         ",
1553         "TxFrames65To127    ",
1554         "TxFrames128To255   ",
1555         "TxFrames256To511   ",
1556         "TxFrames512To1023  ",
1557         "TxFrames1024To1518 ",
1558         "TxFrames1519ToMax  ",
1559
1560         "RxOctetsOK         ",
1561         "RxFramesOK         ",
1562         "RxMulticastFramesOK",
1563         "RxBroadcastFramesOK",
1564         "RxPauseFrames      ",
1565         "RxFCSErrors        ",
1566         "RxSymbolErrors     ",
1567         "RxShortErrors      ",
1568         "RxJabberErrors     ",
1569         "RxLengthErrors     ",
1570         "RxFIFOoverflow     ",
1571
1572         "RxFrames64         ",
1573         "RxFrames65To127    ",
1574         "RxFrames128To255   ",
1575         "RxFrames256To511   ",
1576         "RxFrames512To1023  ",
1577         "RxFrames1024To1518 ",
1578         "RxFrames1519ToMax  ",
1579
1580         "PhyFIFOErrors      ",
1581         "TSO                ",
1582         "VLANextractions    ",
1583         "VLANinsertions     ",
1584         "TxCsumOffload      ",
1585         "RxCsumGood         ",
1586         "LroAggregated      ",
1587         "LroFlushed         ",
1588         "LroNoDesc          ",
1589         "RxDrops            ",
1590
1591         "CheckTXEnToggled   ",
1592         "CheckResets        ",
1593
1594         "LinkFaults         ",
1595 };
1596
1597 static int get_sset_count(struct net_device *dev, int sset)
1598 {
1599         switch (sset) {
1600         case ETH_SS_STATS:
1601                 return ARRAY_SIZE(stats_strings);
1602         default:
1603                 return -EOPNOTSUPP;
1604         }
1605 }
1606
1607 #define T3_REGMAP_SIZE (3 * 1024)
1608
1609 static int get_regs_len(struct net_device *dev)
1610 {
1611         return T3_REGMAP_SIZE;
1612 }
1613
1614 static int get_eeprom_len(struct net_device *dev)
1615 {
1616         return EEPROMSIZE;
1617 }
1618
1619 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1620 {
1621         struct port_info *pi = netdev_priv(dev);
1622         struct adapter *adapter = pi->adapter;
1623         u32 fw_vers = 0;
1624         u32 tp_vers = 0;
1625
1626         spin_lock(&adapter->stats_lock);
1627         t3_get_fw_version(adapter, &fw_vers);
1628         t3_get_tp_version(adapter, &tp_vers);
1629         spin_unlock(&adapter->stats_lock);
1630
1631         strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1632         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1633         strlcpy(info->bus_info, pci_name(adapter->pdev),
1634                 sizeof(info->bus_info));
1635         if (fw_vers)
1636                 snprintf(info->fw_version, sizeof(info->fw_version),
1637                          "%s %u.%u.%u TP %u.%u.%u",
1638                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1639                          G_FW_VERSION_MAJOR(fw_vers),
1640                          G_FW_VERSION_MINOR(fw_vers),
1641                          G_FW_VERSION_MICRO(fw_vers),
1642                          G_TP_VERSION_MAJOR(tp_vers),
1643                          G_TP_VERSION_MINOR(tp_vers),
1644                          G_TP_VERSION_MICRO(tp_vers));
1645 }
1646
1647 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1648 {
1649         if (stringset == ETH_SS_STATS)
1650                 memcpy(data, stats_strings, sizeof(stats_strings));
1651 }
1652
1653 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1654                                             struct port_info *p, int idx)
1655 {
1656         int i;
1657         unsigned long tot = 0;
1658
1659         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1660                 tot += adapter->sge.qs[i].port_stats[idx];
1661         return tot;
1662 }
1663
1664 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1665                       u64 *data)
1666 {
1667         struct port_info *pi = netdev_priv(dev);
1668         struct adapter *adapter = pi->adapter;
1669         const struct mac_stats *s;
1670
1671         spin_lock(&adapter->stats_lock);
1672         s = t3_mac_update_stats(&pi->mac);
1673         spin_unlock(&adapter->stats_lock);
1674
1675         *data++ = s->tx_octets;
1676         *data++ = s->tx_frames;
1677         *data++ = s->tx_mcast_frames;
1678         *data++ = s->tx_bcast_frames;
1679         *data++ = s->tx_pause;
1680         *data++ = s->tx_underrun;
1681         *data++ = s->tx_fifo_urun;
1682
1683         *data++ = s->tx_frames_64;
1684         *data++ = s->tx_frames_65_127;
1685         *data++ = s->tx_frames_128_255;
1686         *data++ = s->tx_frames_256_511;
1687         *data++ = s->tx_frames_512_1023;
1688         *data++ = s->tx_frames_1024_1518;
1689         *data++ = s->tx_frames_1519_max;
1690
1691         *data++ = s->rx_octets;
1692         *data++ = s->rx_frames;
1693         *data++ = s->rx_mcast_frames;
1694         *data++ = s->rx_bcast_frames;
1695         *data++ = s->rx_pause;
1696         *data++ = s->rx_fcs_errs;
1697         *data++ = s->rx_symbol_errs;
1698         *data++ = s->rx_short;
1699         *data++ = s->rx_jabber;
1700         *data++ = s->rx_too_long;
1701         *data++ = s->rx_fifo_ovfl;
1702
1703         *data++ = s->rx_frames_64;
1704         *data++ = s->rx_frames_65_127;
1705         *data++ = s->rx_frames_128_255;
1706         *data++ = s->rx_frames_256_511;
1707         *data++ = s->rx_frames_512_1023;
1708         *data++ = s->rx_frames_1024_1518;
1709         *data++ = s->rx_frames_1519_max;
1710
1711         *data++ = pi->phy.fifo_errors;
1712
1713         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1714         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1715         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1716         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1717         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1718         *data++ = 0;
1719         *data++ = 0;
1720         *data++ = 0;
1721         *data++ = s->rx_cong_drops;
1722
1723         *data++ = s->num_toggled;
1724         *data++ = s->num_resets;
1725
1726         *data++ = s->link_faults;
1727 }
1728
1729 static inline void reg_block_dump(struct adapter *ap, void *buf,
1730                                   unsigned int start, unsigned int end)
1731 {
1732         u32 *p = buf + start;
1733
1734         for (; start <= end; start += sizeof(u32))
1735                 *p++ = t3_read_reg(ap, start);
1736 }
1737
1738 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1739                      void *buf)
1740 {
1741         struct port_info *pi = netdev_priv(dev);
1742         struct adapter *ap = pi->adapter;
1743
1744         /*
1745          * Version scheme:
1746          * bits 0..9: chip version
1747          * bits 10..15: chip revision
1748          * bit 31: set for PCIe cards
1749          */
1750         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1751
1752         /*
1753          * We skip the MAC statistics registers because they are clear-on-read.
1754          * Also reading multi-register stats would need to synchronize with the
1755          * periodic mac stats accumulation.  Hard to justify the complexity.
1756          */
1757         memset(buf, 0, T3_REGMAP_SIZE);
1758         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1759         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1760         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1761         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1762         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1763         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1764                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1765         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1766                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1767 }
1768
1769 static int restart_autoneg(struct net_device *dev)
1770 {
1771         struct port_info *p = netdev_priv(dev);
1772
1773         if (!netif_running(dev))
1774                 return -EAGAIN;
1775         if (p->link_config.autoneg != AUTONEG_ENABLE)
1776                 return -EINVAL;
1777         p->phy.ops->autoneg_restart(&p->phy);
1778         return 0;
1779 }
1780
1781 static int set_phys_id(struct net_device *dev,
1782                        enum ethtool_phys_id_state state)
1783 {
1784         struct port_info *pi = netdev_priv(dev);
1785         struct adapter *adapter = pi->adapter;
1786
1787         switch (state) {
1788         case ETHTOOL_ID_ACTIVE:
1789                 return 1;       /* cycle on/off once per second */
1790
1791         case ETHTOOL_ID_OFF:
1792                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1793                 break;
1794
1795         case ETHTOOL_ID_ON:
1796         case ETHTOOL_ID_INACTIVE:
1797                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1798                          F_GPIO0_OUT_VAL);
1799         }
1800
1801         return 0;
1802 }
1803
1804 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1805 {
1806         struct port_info *p = netdev_priv(dev);
1807
1808         cmd->supported = p->link_config.supported;
1809         cmd->advertising = p->link_config.advertising;
1810
1811         if (netif_carrier_ok(dev)) {
1812                 ethtool_cmd_speed_set(cmd, p->link_config.speed);
1813                 cmd->duplex = p->link_config.duplex;
1814         } else {
1815                 ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
1816                 cmd->duplex = DUPLEX_UNKNOWN;
1817         }
1818
1819         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1820         cmd->phy_address = p->phy.mdio.prtad;
1821         cmd->transceiver = XCVR_EXTERNAL;
1822         cmd->autoneg = p->link_config.autoneg;
1823         cmd->maxtxpkt = 0;
1824         cmd->maxrxpkt = 0;
1825         return 0;
1826 }
1827
1828 static int speed_duplex_to_caps(int speed, int duplex)
1829 {
1830         int cap = 0;
1831
1832         switch (speed) {
1833         case SPEED_10:
1834                 if (duplex == DUPLEX_FULL)
1835                         cap = SUPPORTED_10baseT_Full;
1836                 else
1837                         cap = SUPPORTED_10baseT_Half;
1838                 break;
1839         case SPEED_100:
1840                 if (duplex == DUPLEX_FULL)
1841                         cap = SUPPORTED_100baseT_Full;
1842                 else
1843                         cap = SUPPORTED_100baseT_Half;
1844                 break;
1845         case SPEED_1000:
1846                 if (duplex == DUPLEX_FULL)
1847                         cap = SUPPORTED_1000baseT_Full;
1848                 else
1849                         cap = SUPPORTED_1000baseT_Half;
1850                 break;
1851         case SPEED_10000:
1852                 if (duplex == DUPLEX_FULL)
1853                         cap = SUPPORTED_10000baseT_Full;
1854         }
1855         return cap;
1856 }
1857
1858 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1859                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1860                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1861                       ADVERTISED_10000baseT_Full)
1862
1863 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1864 {
1865         struct port_info *p = netdev_priv(dev);
1866         struct link_config *lc = &p->link_config;
1867
1868         if (!(lc->supported & SUPPORTED_Autoneg)) {
1869                 /*
1870                  * PHY offers a single speed/duplex.  See if that's what's
1871                  * being requested.
1872                  */
1873                 if (cmd->autoneg == AUTONEG_DISABLE) {
1874                         u32 speed = ethtool_cmd_speed(cmd);
1875                         int cap = speed_duplex_to_caps(speed, cmd->duplex);
1876                         if (lc->supported & cap)
1877                                 return 0;
1878                 }
1879                 return -EINVAL;
1880         }
1881
1882         if (cmd->autoneg == AUTONEG_DISABLE) {
1883                 u32 speed = ethtool_cmd_speed(cmd);
1884                 int cap = speed_duplex_to_caps(speed, cmd->duplex);
1885
1886                 if (!(lc->supported & cap) || (speed == SPEED_1000))
1887                         return -EINVAL;
1888                 lc->requested_speed = speed;
1889                 lc->requested_duplex = cmd->duplex;
1890                 lc->advertising = 0;
1891         } else {
1892                 cmd->advertising &= ADVERTISED_MASK;
1893                 cmd->advertising &= lc->supported;
1894                 if (!cmd->advertising)
1895                         return -EINVAL;
1896                 lc->requested_speed = SPEED_INVALID;
1897                 lc->requested_duplex = DUPLEX_INVALID;
1898                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1899         }
1900         lc->autoneg = cmd->autoneg;
1901         if (netif_running(dev))
1902                 t3_link_start(&p->phy, &p->mac, lc);
1903         return 0;
1904 }
1905
1906 static void get_pauseparam(struct net_device *dev,
1907                            struct ethtool_pauseparam *epause)
1908 {
1909         struct port_info *p = netdev_priv(dev);
1910
1911         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1912         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1913         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1914 }
1915
1916 static int set_pauseparam(struct net_device *dev,
1917                           struct ethtool_pauseparam *epause)
1918 {
1919         struct port_info *p = netdev_priv(dev);
1920         struct link_config *lc = &p->link_config;
1921
1922         if (epause->autoneg == AUTONEG_DISABLE)
1923                 lc->requested_fc = 0;
1924         else if (lc->supported & SUPPORTED_Autoneg)
1925                 lc->requested_fc = PAUSE_AUTONEG;
1926         else
1927                 return -EINVAL;
1928
1929         if (epause->rx_pause)
1930                 lc->requested_fc |= PAUSE_RX;
1931         if (epause->tx_pause)
1932                 lc->requested_fc |= PAUSE_TX;
1933         if (lc->autoneg == AUTONEG_ENABLE) {
1934                 if (netif_running(dev))
1935                         t3_link_start(&p->phy, &p->mac, lc);
1936         } else {
1937                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1938                 if (netif_running(dev))
1939                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1940         }
1941         return 0;
1942 }
1943
1944 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1945 {
1946         struct port_info *pi = netdev_priv(dev);
1947         struct adapter *adapter = pi->adapter;
1948         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1949
1950         e->rx_max_pending = MAX_RX_BUFFERS;
1951         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1952         e->tx_max_pending = MAX_TXQ_ENTRIES;
1953
1954         e->rx_pending = q->fl_size;
1955         e->rx_mini_pending = q->rspq_size;
1956         e->rx_jumbo_pending = q->jumbo_size;
1957         e->tx_pending = q->txq_size[0];
1958 }
1959
1960 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1961 {
1962         struct port_info *pi = netdev_priv(dev);
1963         struct adapter *adapter = pi->adapter;
1964         struct qset_params *q;
1965         int i;
1966
1967         if (e->rx_pending > MAX_RX_BUFFERS ||
1968             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1969             e->tx_pending > MAX_TXQ_ENTRIES ||
1970             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1971             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1972             e->rx_pending < MIN_FL_ENTRIES ||
1973             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1974             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1975                 return -EINVAL;
1976
1977         if (adapter->flags & FULL_INIT_DONE)
1978                 return -EBUSY;
1979
1980         q = &adapter->params.sge.qset[pi->first_qset];
1981         for (i = 0; i < pi->nqsets; ++i, ++q) {
1982                 q->rspq_size = e->rx_mini_pending;
1983                 q->fl_size = e->rx_pending;
1984                 q->jumbo_size = e->rx_jumbo_pending;
1985                 q->txq_size[0] = e->tx_pending;
1986                 q->txq_size[1] = e->tx_pending;
1987                 q->txq_size[2] = e->tx_pending;
1988         }
1989         return 0;
1990 }
1991
1992 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1993 {
1994         struct port_info *pi = netdev_priv(dev);
1995         struct adapter *adapter = pi->adapter;
1996         struct qset_params *qsp;
1997         struct sge_qset *qs;
1998         int i;
1999
2000         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
2001                 return -EINVAL;
2002
2003         for (i = 0; i < pi->nqsets; i++) {
2004                 qsp = &adapter->params.sge.qset[i];
2005                 qs = &adapter->sge.qs[i];
2006                 qsp->coalesce_usecs = c->rx_coalesce_usecs;
2007                 t3_update_qset_coalesce(qs, qsp);
2008         }
2009
2010         return 0;
2011 }
2012
2013 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2014 {
2015         struct port_info *pi = netdev_priv(dev);
2016         struct adapter *adapter = pi->adapter;
2017         struct qset_params *q = adapter->params.sge.qset;
2018
2019         c->rx_coalesce_usecs = q->coalesce_usecs;
2020         return 0;
2021 }
2022
2023 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2024                       u8 * data)
2025 {
2026         struct port_info *pi = netdev_priv(dev);
2027         struct adapter *adapter = pi->adapter;
2028         int i, err = 0;
2029
2030         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2031         if (!buf)
2032                 return -ENOMEM;
2033
2034         e->magic = EEPROM_MAGIC;
2035         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2036                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2037
2038         if (!err)
2039                 memcpy(data, buf + e->offset, e->len);
2040         kfree(buf);
2041         return err;
2042 }
2043
2044 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2045                       u8 * data)
2046 {
2047         struct port_info *pi = netdev_priv(dev);
2048         struct adapter *adapter = pi->adapter;
2049         u32 aligned_offset, aligned_len;
2050         __le32 *p;
2051         u8 *buf;
2052         int err;
2053
2054         if (eeprom->magic != EEPROM_MAGIC)
2055                 return -EINVAL;
2056
2057         aligned_offset = eeprom->offset & ~3;
2058         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2059
2060         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2061                 buf = kmalloc(aligned_len, GFP_KERNEL);
2062                 if (!buf)
2063                         return -ENOMEM;
2064                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2065                 if (!err && aligned_len > 4)
2066                         err = t3_seeprom_read(adapter,
2067                                               aligned_offset + aligned_len - 4,
2068                                               (__le32 *) & buf[aligned_len - 4]);
2069                 if (err)
2070                         goto out;
2071                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2072         } else
2073                 buf = data;
2074
2075         err = t3_seeprom_wp(adapter, 0);
2076         if (err)
2077                 goto out;
2078
2079         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2080                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2081                 aligned_offset += 4;
2082         }
2083
2084         if (!err)
2085                 err = t3_seeprom_wp(adapter, 1);
2086 out:
2087         if (buf != data)
2088                 kfree(buf);
2089         return err;
2090 }
2091
2092 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2093 {
2094         wol->supported = 0;
2095         wol->wolopts = 0;
2096         memset(&wol->sopass, 0, sizeof(wol->sopass));
2097 }
2098
2099 static const struct ethtool_ops cxgb_ethtool_ops = {
2100         .get_settings = get_settings,
2101         .set_settings = set_settings,
2102         .get_drvinfo = get_drvinfo,
2103         .get_msglevel = get_msglevel,
2104         .set_msglevel = set_msglevel,
2105         .get_ringparam = get_sge_param,
2106         .set_ringparam = set_sge_param,
2107         .get_coalesce = get_coalesce,
2108         .set_coalesce = set_coalesce,
2109         .get_eeprom_len = get_eeprom_len,
2110         .get_eeprom = get_eeprom,
2111         .set_eeprom = set_eeprom,
2112         .get_pauseparam = get_pauseparam,
2113         .set_pauseparam = set_pauseparam,
2114         .get_link = ethtool_op_get_link,
2115         .get_strings = get_strings,
2116         .set_phys_id = set_phys_id,
2117         .nway_reset = restart_autoneg,
2118         .get_sset_count = get_sset_count,
2119         .get_ethtool_stats = get_stats,
2120         .get_regs_len = get_regs_len,
2121         .get_regs = get_regs,
2122         .get_wol = get_wol,
2123 };
2124
2125 static int in_range(int val, int lo, int hi)
2126 {
2127         return val < 0 || (val <= hi && val >= lo);
2128 }
2129
2130 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2131 {
2132         struct port_info *pi = netdev_priv(dev);
2133         struct adapter *adapter = pi->adapter;
2134         u32 cmd;
2135         int ret;
2136
2137         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2138                 return -EFAULT;
2139
2140         switch (cmd) {
2141         case CHELSIO_SET_QSET_PARAMS:{
2142                 int i;
2143                 struct qset_params *q;
2144                 struct ch_qset_params t;
2145                 int q1 = pi->first_qset;
2146                 int nqsets = pi->nqsets;
2147
2148                 if (!capable(CAP_NET_ADMIN))
2149                         return -EPERM;
2150                 if (copy_from_user(&t, useraddr, sizeof(t)))
2151                         return -EFAULT;
2152                 if (t.qset_idx >= SGE_QSETS)
2153                         return -EINVAL;
2154                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2155                     !in_range(t.cong_thres, 0, 255) ||
2156                     !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2157                               MAX_TXQ_ENTRIES) ||
2158                     !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2159                               MAX_TXQ_ENTRIES) ||
2160                     !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2161                               MAX_CTRL_TXQ_ENTRIES) ||
2162                     !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2163                               MAX_RX_BUFFERS) ||
2164                     !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2165                               MAX_RX_JUMBO_BUFFERS) ||
2166                     !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2167                               MAX_RSPQ_ENTRIES))
2168                         return -EINVAL;
2169
2170                 if ((adapter->flags & FULL_INIT_DONE) &&
2171                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2172                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2173                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2174                         t.polling >= 0 || t.cong_thres >= 0))
2175                         return -EBUSY;
2176
2177                 /* Allow setting of any available qset when offload enabled */
2178                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2179                         q1 = 0;
2180                         for_each_port(adapter, i) {
2181                                 pi = adap2pinfo(adapter, i);
2182                                 nqsets += pi->first_qset + pi->nqsets;
2183                         }
2184                 }
2185
2186                 if (t.qset_idx < q1)
2187                         return -EINVAL;
2188                 if (t.qset_idx > q1 + nqsets - 1)
2189                         return -EINVAL;
2190
2191                 q = &adapter->params.sge.qset[t.qset_idx];
2192
2193                 if (t.rspq_size >= 0)
2194                         q->rspq_size = t.rspq_size;
2195                 if (t.fl_size[0] >= 0)
2196                         q->fl_size = t.fl_size[0];
2197                 if (t.fl_size[1] >= 0)
2198                         q->jumbo_size = t.fl_size[1];
2199                 if (t.txq_size[0] >= 0)
2200                         q->txq_size[0] = t.txq_size[0];
2201                 if (t.txq_size[1] >= 0)
2202                         q->txq_size[1] = t.txq_size[1];
2203                 if (t.txq_size[2] >= 0)
2204                         q->txq_size[2] = t.txq_size[2];
2205                 if (t.cong_thres >= 0)
2206                         q->cong_thres = t.cong_thres;
2207                 if (t.intr_lat >= 0) {
2208                         struct sge_qset *qs =
2209                                 &adapter->sge.qs[t.qset_idx];
2210
2211                         q->coalesce_usecs = t.intr_lat;
2212                         t3_update_qset_coalesce(qs, q);
2213                 }
2214                 if (t.polling >= 0) {
2215                         if (adapter->flags & USING_MSIX)
2216                                 q->polling = t.polling;
2217                         else {
2218                                 /* No polling with INTx for T3A */
2219                                 if (adapter->params.rev == 0 &&
2220                                         !(adapter->flags & USING_MSI))
2221                                         t.polling = 0;
2222
2223                                 for (i = 0; i < SGE_QSETS; i++) {
2224                                         q = &adapter->params.sge.
2225                                                 qset[i];
2226                                         q->polling = t.polling;
2227                                 }
2228                         }
2229                 }
2230
2231                 if (t.lro >= 0) {
2232                         if (t.lro)
2233                                 dev->wanted_features |= NETIF_F_GRO;
2234                         else
2235                                 dev->wanted_features &= ~NETIF_F_GRO;
2236                         netdev_update_features(dev);
2237                 }
2238
2239                 break;
2240         }
2241         case CHELSIO_GET_QSET_PARAMS:{
2242                 struct qset_params *q;
2243                 struct ch_qset_params t;
2244                 int q1 = pi->first_qset;
2245                 int nqsets = pi->nqsets;
2246                 int i;
2247
2248                 if (copy_from_user(&t, useraddr, sizeof(t)))
2249                         return -EFAULT;
2250
2251                 /* Display qsets for all ports when offload enabled */
2252                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2253                         q1 = 0;
2254                         for_each_port(adapter, i) {
2255                                 pi = adap2pinfo(adapter, i);
2256                                 nqsets = pi->first_qset + pi->nqsets;
2257                         }
2258                 }
2259
2260                 if (t.qset_idx >= nqsets)
2261                         return -EINVAL;
2262
2263                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2264                 t.rspq_size = q->rspq_size;
2265                 t.txq_size[0] = q->txq_size[0];
2266                 t.txq_size[1] = q->txq_size[1];
2267                 t.txq_size[2] = q->txq_size[2];
2268                 t.fl_size[0] = q->fl_size;
2269                 t.fl_size[1] = q->jumbo_size;
2270                 t.polling = q->polling;
2271                 t.lro = !!(dev->features & NETIF_F_GRO);
2272                 t.intr_lat = q->coalesce_usecs;
2273                 t.cong_thres = q->cong_thres;
2274                 t.qnum = q1;
2275
2276                 if (adapter->flags & USING_MSIX)
2277                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2278                 else
2279                         t.vector = adapter->pdev->irq;
2280
2281                 if (copy_to_user(useraddr, &t, sizeof(t)))
2282                         return -EFAULT;
2283                 break;
2284         }
2285         case CHELSIO_SET_QSET_NUM:{
2286                 struct ch_reg edata;
2287                 unsigned int i, first_qset = 0, other_qsets = 0;
2288
2289                 if (!capable(CAP_NET_ADMIN))
2290                         return -EPERM;
2291                 if (adapter->flags & FULL_INIT_DONE)
2292                         return -EBUSY;
2293                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2294                         return -EFAULT;
2295                 if (edata.val < 1 ||
2296                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2297                         return -EINVAL;
2298
2299                 for_each_port(adapter, i)
2300                         if (adapter->port[i] && adapter->port[i] != dev)
2301                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2302
2303                 if (edata.val + other_qsets > SGE_QSETS)
2304                         return -EINVAL;
2305
2306                 pi->nqsets = edata.val;
2307
2308                 for_each_port(adapter, i)
2309                         if (adapter->port[i]) {
2310                                 pi = adap2pinfo(adapter, i);
2311                                 pi->first_qset = first_qset;
2312                                 first_qset += pi->nqsets;
2313                         }
2314                 break;
2315         }
2316         case CHELSIO_GET_QSET_NUM:{
2317                 struct ch_reg edata;
2318
2319                 memset(&edata, 0, sizeof(struct ch_reg));
2320
2321                 edata.cmd = CHELSIO_GET_QSET_NUM;
2322                 edata.val = pi->nqsets;
2323                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2324                         return -EFAULT;
2325                 break;
2326         }
2327         case CHELSIO_LOAD_FW:{
2328                 u8 *fw_data;
2329                 struct ch_mem_range t;
2330
2331                 if (!capable(CAP_SYS_RAWIO))
2332                         return -EPERM;
2333                 if (copy_from_user(&t, useraddr, sizeof(t)))
2334                         return -EFAULT;
2335                 /* Check t.len sanity ? */
2336                 fw_data = memdup_user(useraddr + sizeof(t), t.len);
2337                 if (IS_ERR(fw_data))
2338                         return PTR_ERR(fw_data);
2339
2340                 ret = t3_load_fw(adapter, fw_data, t.len);
2341                 kfree(fw_data);
2342                 if (ret)
2343                         return ret;
2344                 break;
2345         }
2346         case CHELSIO_SETMTUTAB:{
2347                 struct ch_mtus m;
2348                 int i;
2349
2350                 if (!is_offload(adapter))
2351                         return -EOPNOTSUPP;
2352                 if (!capable(CAP_NET_ADMIN))
2353                         return -EPERM;
2354                 if (offload_running(adapter))
2355                         return -EBUSY;
2356                 if (copy_from_user(&m, useraddr, sizeof(m)))
2357                         return -EFAULT;
2358                 if (m.nmtus != NMTUS)
2359                         return -EINVAL;
2360                 if (m.mtus[0] < 81)     /* accommodate SACK */
2361                         return -EINVAL;
2362
2363                 /* MTUs must be in ascending order */
2364                 for (i = 1; i < NMTUS; ++i)
2365                         if (m.mtus[i] < m.mtus[i - 1])
2366                                 return -EINVAL;
2367
2368                 memcpy(adapter->params.mtus, m.mtus,
2369                         sizeof(adapter->params.mtus));
2370                 break;
2371         }
2372         case CHELSIO_GET_PM:{
2373                 struct tp_params *p = &adapter->params.tp;
2374                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2375
2376                 if (!is_offload(adapter))
2377                         return -EOPNOTSUPP;
2378                 m.tx_pg_sz = p->tx_pg_size;
2379                 m.tx_num_pg = p->tx_num_pgs;
2380                 m.rx_pg_sz = p->rx_pg_size;
2381                 m.rx_num_pg = p->rx_num_pgs;
2382                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2383                 if (copy_to_user(useraddr, &m, sizeof(m)))
2384                         return -EFAULT;
2385                 break;
2386         }
2387         case CHELSIO_SET_PM:{
2388                 struct ch_pm m;
2389                 struct tp_params *p = &adapter->params.tp;
2390
2391                 if (!is_offload(adapter))
2392                         return -EOPNOTSUPP;
2393                 if (!capable(CAP_NET_ADMIN))
2394                         return -EPERM;
2395                 if (adapter->flags & FULL_INIT_DONE)
2396                         return -EBUSY;
2397                 if (copy_from_user(&m, useraddr, sizeof(m)))
2398                         return -EFAULT;
2399                 if (!is_power_of_2(m.rx_pg_sz) ||
2400                         !is_power_of_2(m.tx_pg_sz))
2401                         return -EINVAL; /* not power of 2 */
2402                 if (!(m.rx_pg_sz & 0x14000))
2403                         return -EINVAL; /* not 16KB or 64KB */
2404                 if (!(m.tx_pg_sz & 0x1554000))
2405                         return -EINVAL;
2406                 if (m.tx_num_pg == -1)
2407                         m.tx_num_pg = p->tx_num_pgs;
2408                 if (m.rx_num_pg == -1)
2409                         m.rx_num_pg = p->rx_num_pgs;
2410                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2411                         return -EINVAL;
2412                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2413                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2414                         return -EINVAL;
2415                 p->rx_pg_size = m.rx_pg_sz;
2416                 p->tx_pg_size = m.tx_pg_sz;
2417                 p->rx_num_pgs = m.rx_num_pg;
2418                 p->tx_num_pgs = m.tx_num_pg;
2419                 break;
2420         }
2421         case CHELSIO_GET_MEM:{
2422                 struct ch_mem_range t;
2423                 struct mc7 *mem;
2424                 u64 buf[32];
2425
2426                 if (!is_offload(adapter))
2427                         return -EOPNOTSUPP;
2428                 if (!(adapter->flags & FULL_INIT_DONE))
2429                         return -EIO;    /* need the memory controllers */
2430                 if (copy_from_user(&t, useraddr, sizeof(t)))
2431                         return -EFAULT;
2432                 if ((t.addr & 7) || (t.len & 7))
2433                         return -EINVAL;
2434                 if (t.mem_id == MEM_CM)
2435                         mem = &adapter->cm;
2436                 else if (t.mem_id == MEM_PMRX)
2437                         mem = &adapter->pmrx;
2438                 else if (t.mem_id == MEM_PMTX)
2439                         mem = &adapter->pmtx;
2440                 else
2441                         return -EINVAL;
2442
2443                 /*
2444                  * Version scheme:
2445                  * bits 0..9: chip version
2446                  * bits 10..15: chip revision
2447                  */
2448                 t.version = 3 | (adapter->params.rev << 10);
2449                 if (copy_to_user(useraddr, &t, sizeof(t)))
2450                         return -EFAULT;
2451
2452                 /*
2453                  * Read 256 bytes at a time as len can be large and we don't
2454                  * want to use huge intermediate buffers.
2455                  */
2456                 useraddr += sizeof(t);  /* advance to start of buffer */
2457                 while (t.len) {
2458                         unsigned int chunk =
2459                                 min_t(unsigned int, t.len, sizeof(buf));
2460
2461                         ret =
2462                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2463                                                 buf);
2464                         if (ret)
2465                                 return ret;
2466                         if (copy_to_user(useraddr, buf, chunk))
2467                                 return -EFAULT;
2468                         useraddr += chunk;
2469                         t.addr += chunk;
2470                         t.len -= chunk;
2471                 }
2472                 break;
2473         }
2474         case CHELSIO_SET_TRACE_FILTER:{
2475                 struct ch_trace t;
2476                 const struct trace_params *tp;
2477
2478                 if (!capable(CAP_NET_ADMIN))
2479                         return -EPERM;
2480                 if (!offload_running(adapter))
2481                         return -EAGAIN;
2482                 if (copy_from_user(&t, useraddr, sizeof(t)))
2483                         return -EFAULT;
2484
2485                 tp = (const struct trace_params *)&t.sip;
2486                 if (t.config_tx)
2487                         t3_config_trace_filter(adapter, tp, 0,
2488                                                 t.invert_match,
2489                                                 t.trace_tx);
2490                 if (t.config_rx)
2491                         t3_config_trace_filter(adapter, tp, 1,
2492                                                 t.invert_match,
2493                                                 t.trace_rx);
2494                 break;
2495         }
2496         default:
2497                 return -EOPNOTSUPP;
2498         }
2499         return 0;
2500 }
2501
2502 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2503 {
2504         struct mii_ioctl_data *data = if_mii(req);
2505         struct port_info *pi = netdev_priv(dev);
2506         struct adapter *adapter = pi->adapter;
2507
2508         switch (cmd) {
2509         case SIOCGMIIREG:
2510         case SIOCSMIIREG:
2511                 /* Convert phy_id from older PRTAD/DEVAD format */
2512                 if (is_10G(adapter) &&
2513                     !mdio_phy_id_is_c45(data->phy_id) &&
2514                     (data->phy_id & 0x1f00) &&
2515                     !(data->phy_id & 0xe0e0))
2516                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2517                                                        data->phy_id & 0x1f);
2518                 /* FALLTHRU */
2519         case SIOCGMIIPHY:
2520                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2521         case SIOCCHIOCTL:
2522                 return cxgb_extension_ioctl(dev, req->ifr_data);
2523         default:
2524                 return -EOPNOTSUPP;
2525         }
2526 }
2527
2528 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2529 {
2530         struct port_info *pi = netdev_priv(dev);
2531         struct adapter *adapter = pi->adapter;
2532         int ret;
2533
2534         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2535                 return ret;
2536         dev->mtu = new_mtu;
2537         init_port_mtus(adapter);
2538         if (adapter->params.rev == 0 && offload_running(adapter))
2539                 t3_load_mtus(adapter, adapter->params.mtus,
2540                              adapter->params.a_wnd, adapter->params.b_wnd,
2541                              adapter->port[0]->mtu);
2542         return 0;
2543 }
2544
2545 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2546 {
2547         struct port_info *pi = netdev_priv(dev);
2548         struct adapter *adapter = pi->adapter;
2549         struct sockaddr *addr = p;
2550
2551         if (!is_valid_ether_addr(addr->sa_data))
2552                 return -EADDRNOTAVAIL;
2553
2554         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2555         t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2556         if (offload_running(adapter))
2557                 write_smt_entry(adapter, pi->port_id);
2558         return 0;
2559 }
2560
2561 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2562         netdev_features_t features)
2563 {
2564         /*
2565          * Since there is no support for separate rx/tx vlan accel
2566          * enable/disable make sure tx flag is always in same state as rx.
2567          */
2568         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2569                 features |= NETIF_F_HW_VLAN_CTAG_TX;
2570         else
2571                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2572
2573         return features;
2574 }
2575
2576 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2577 {
2578         netdev_features_t changed = dev->features ^ features;
2579
2580         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2581                 cxgb_vlan_mode(dev, features);
2582
2583         return 0;
2584 }
2585
2586 #ifdef CONFIG_NET_POLL_CONTROLLER
2587 static void cxgb_netpoll(struct net_device *dev)
2588 {
2589         struct port_info *pi = netdev_priv(dev);
2590         struct adapter *adapter = pi->adapter;
2591         int qidx;
2592
2593         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2594                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2595                 void *source;
2596
2597                 if (adapter->flags & USING_MSIX)
2598                         source = qs;
2599                 else
2600                         source = adapter;
2601
2602                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2603         }
2604 }
2605 #endif
2606
2607 /*
2608  * Periodic accumulation of MAC statistics.
2609  */
2610 static void mac_stats_update(struct adapter *adapter)
2611 {
2612         int i;
2613
2614         for_each_port(adapter, i) {
2615                 struct net_device *dev = adapter->port[i];
2616                 struct port_info *p = netdev_priv(dev);
2617
2618                 if (netif_running(dev)) {
2619                         spin_lock(&adapter->stats_lock);
2620                         t3_mac_update_stats(&p->mac);
2621                         spin_unlock(&adapter->stats_lock);
2622                 }
2623         }
2624 }
2625
2626 static void check_link_status(struct adapter *adapter)
2627 {
2628         int i;
2629
2630         for_each_port(adapter, i) {
2631                 struct net_device *dev = adapter->port[i];
2632                 struct port_info *p = netdev_priv(dev);
2633                 int link_fault;
2634
2635                 spin_lock_irq(&adapter->work_lock);
2636                 link_fault = p->link_fault;
2637                 spin_unlock_irq(&adapter->work_lock);
2638
2639                 if (link_fault) {
2640                         t3_link_fault(adapter, i);
2641                         continue;
2642                 }
2643
2644                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2645                         t3_xgm_intr_disable(adapter, i);
2646                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2647
2648                         t3_link_changed(adapter, i);
2649                         t3_xgm_intr_enable(adapter, i);
2650                 }
2651         }
2652 }
2653
2654 static void check_t3b2_mac(struct adapter *adapter)
2655 {
2656         int i;
2657
2658         if (!rtnl_trylock())    /* synchronize with ifdown */
2659                 return;
2660
2661         for_each_port(adapter, i) {
2662                 struct net_device *dev = adapter->port[i];
2663                 struct port_info *p = netdev_priv(dev);
2664                 int status;
2665
2666                 if (!netif_running(dev))
2667                         continue;
2668
2669                 status = 0;
2670                 if (netif_running(dev) && netif_carrier_ok(dev))
2671                         status = t3b2_mac_watchdog_task(&p->mac);
2672                 if (status == 1)
2673                         p->mac.stats.num_toggled++;
2674                 else if (status == 2) {
2675                         struct cmac *mac = &p->mac;
2676
2677                         t3_mac_set_mtu(mac, dev->mtu);
2678                         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2679                         cxgb_set_rxmode(dev);
2680                         t3_link_start(&p->phy, mac, &p->link_config);
2681                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2682                         t3_port_intr_enable(adapter, p->port_id);
2683                         p->mac.stats.num_resets++;
2684                 }
2685         }
2686         rtnl_unlock();
2687 }
2688
2689
2690 static void t3_adap_check_task(struct work_struct *work)
2691 {
2692         struct adapter *adapter = container_of(work, struct adapter,
2693                                                adap_check_task.work);
2694         const struct adapter_params *p = &adapter->params;
2695         int port;
2696         unsigned int v, status, reset;
2697
2698         adapter->check_task_cnt++;
2699
2700         check_link_status(adapter);
2701
2702         /* Accumulate MAC stats if needed */
2703         if (!p->linkpoll_period ||
2704             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2705             p->stats_update_period) {
2706                 mac_stats_update(adapter);
2707                 adapter->check_task_cnt = 0;
2708         }
2709
2710         if (p->rev == T3_REV_B2)
2711                 check_t3b2_mac(adapter);
2712
2713         /*
2714          * Scan the XGMAC's to check for various conditions which we want to
2715          * monitor in a periodic polling manner rather than via an interrupt
2716          * condition.  This is used for conditions which would otherwise flood
2717          * the system with interrupts and we only really need to know that the
2718          * conditions are "happening" ...  For each condition we count the
2719          * detection of the condition and reset it for the next polling loop.
2720          */
2721         for_each_port(adapter, port) {
2722                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2723                 u32 cause;
2724
2725                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2726                 reset = 0;
2727                 if (cause & F_RXFIFO_OVERFLOW) {
2728                         mac->stats.rx_fifo_ovfl++;
2729                         reset |= F_RXFIFO_OVERFLOW;
2730                 }
2731
2732                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2733         }
2734
2735         /*
2736          * We do the same as above for FL_EMPTY interrupts.
2737          */
2738         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2739         reset = 0;
2740
2741         if (status & F_FLEMPTY) {
2742                 struct sge_qset *qs = &adapter->sge.qs[0];
2743                 int i = 0;
2744
2745                 reset |= F_FLEMPTY;
2746
2747                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2748                     0xffff;
2749
2750                 while (v) {
2751                         qs->fl[i].empty += (v & 1);
2752                         if (i)
2753                                 qs++;
2754                         i ^= 1;
2755                         v >>= 1;
2756                 }
2757         }
2758
2759         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2760
2761         /* Schedule the next check update if any port is active. */
2762         spin_lock_irq(&adapter->work_lock);
2763         if (adapter->open_device_map & PORT_MASK)
2764                 schedule_chk_task(adapter);
2765         spin_unlock_irq(&adapter->work_lock);
2766 }
2767
2768 static void db_full_task(struct work_struct *work)
2769 {
2770         struct adapter *adapter = container_of(work, struct adapter,
2771                                                db_full_task);
2772
2773         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2774 }
2775
2776 static void db_empty_task(struct work_struct *work)
2777 {
2778         struct adapter *adapter = container_of(work, struct adapter,
2779                                                db_empty_task);
2780
2781         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2782 }
2783
2784 static void db_drop_task(struct work_struct *work)
2785 {
2786         struct adapter *adapter = container_of(work, struct adapter,
2787                                                db_drop_task);
2788         unsigned long delay = 1000;
2789         unsigned short r;
2790
2791         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2792
2793         /*
2794          * Sleep a while before ringing the driver qset dbs.
2795          * The delay is between 1000-2023 usecs.
2796          */
2797         get_random_bytes(&r, 2);
2798         delay += r & 1023;
2799         set_current_state(TASK_UNINTERRUPTIBLE);
2800         schedule_timeout(usecs_to_jiffies(delay));
2801         ring_dbs(adapter);
2802 }
2803
2804 /*
2805  * Processes external (PHY) interrupts in process context.
2806  */
2807 static void ext_intr_task(struct work_struct *work)
2808 {
2809         struct adapter *adapter = container_of(work, struct adapter,
2810                                                ext_intr_handler_task);
2811         int i;
2812
2813         /* Disable link fault interrupts */
2814         for_each_port(adapter, i) {
2815                 struct net_device *dev = adapter->port[i];
2816                 struct port_info *p = netdev_priv(dev);
2817
2818                 t3_xgm_intr_disable(adapter, i);
2819                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2820         }
2821
2822         /* Re-enable link fault interrupts */
2823         t3_phy_intr_handler(adapter);
2824
2825         for_each_port(adapter, i)
2826                 t3_xgm_intr_enable(adapter, i);
2827
2828         /* Now reenable external interrupts */
2829         spin_lock_irq(&adapter->work_lock);
2830         if (adapter->slow_intr_mask) {
2831                 adapter->slow_intr_mask |= F_T3DBG;
2832                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2833                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2834                              adapter->slow_intr_mask);
2835         }
2836         spin_unlock_irq(&adapter->work_lock);
2837 }
2838
2839 /*
2840  * Interrupt-context handler for external (PHY) interrupts.
2841  */
2842 void t3_os_ext_intr_handler(struct adapter *adapter)
2843 {
2844         /*
2845          * Schedule a task to handle external interrupts as they may be slow
2846          * and we use a mutex to protect MDIO registers.  We disable PHY
2847          * interrupts in the meantime and let the task reenable them when
2848          * it's done.
2849          */
2850         spin_lock(&adapter->work_lock);
2851         if (adapter->slow_intr_mask) {
2852                 adapter->slow_intr_mask &= ~F_T3DBG;
2853                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2854                              adapter->slow_intr_mask);
2855                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2856         }
2857         spin_unlock(&adapter->work_lock);
2858 }
2859
2860 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2861 {
2862         struct net_device *netdev = adapter->port[port_id];
2863         struct port_info *pi = netdev_priv(netdev);
2864
2865         spin_lock(&adapter->work_lock);
2866         pi->link_fault = 1;
2867         spin_unlock(&adapter->work_lock);
2868 }
2869
2870 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2871 {
2872         int i, ret = 0;
2873
2874         if (is_offload(adapter) &&
2875             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2876                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2877                 offload_close(&adapter->tdev);
2878         }
2879
2880         /* Stop all ports */
2881         for_each_port(adapter, i) {
2882                 struct net_device *netdev = adapter->port[i];
2883
2884                 if (netif_running(netdev))
2885                         __cxgb_close(netdev, on_wq);
2886         }
2887
2888         /* Stop SGE timers */
2889         t3_stop_sge_timers(adapter);
2890
2891         adapter->flags &= ~FULL_INIT_DONE;
2892
2893         if (reset)
2894                 ret = t3_reset_adapter(adapter);
2895
2896         pci_disable_device(adapter->pdev);
2897
2898         return ret;
2899 }
2900
2901 static int t3_reenable_adapter(struct adapter *adapter)
2902 {
2903         if (pci_enable_device(adapter->pdev)) {
2904                 dev_err(&adapter->pdev->dev,
2905                         "Cannot re-enable PCI device after reset.\n");
2906                 goto err;
2907         }
2908         pci_set_master(adapter->pdev);
2909         pci_restore_state(adapter->pdev);
2910         pci_save_state(adapter->pdev);
2911
2912         /* Free sge resources */
2913         t3_free_sge_resources(adapter);
2914
2915         if (t3_replay_prep_adapter(adapter))
2916                 goto err;
2917
2918         return 0;
2919 err:
2920         return -1;
2921 }
2922
2923 static void t3_resume_ports(struct adapter *adapter)
2924 {
2925         int i;
2926
2927         /* Restart the ports */
2928         for_each_port(adapter, i) {
2929                 struct net_device *netdev = adapter->port[i];
2930
2931                 if (netif_running(netdev)) {
2932                         if (cxgb_open(netdev)) {
2933                                 dev_err(&adapter->pdev->dev,
2934                                         "can't bring device back up"
2935                                         " after reset\n");
2936                                 continue;
2937                         }
2938                 }
2939         }
2940
2941         if (is_offload(adapter) && !ofld_disable)
2942                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2943 }
2944
2945 /*
2946  * processes a fatal error.
2947  * Bring the ports down, reset the chip, bring the ports back up.
2948  */
2949 static void fatal_error_task(struct work_struct *work)
2950 {
2951         struct adapter *adapter = container_of(work, struct adapter,
2952                                                fatal_error_handler_task);
2953         int err = 0;
2954
2955         rtnl_lock();
2956         err = t3_adapter_error(adapter, 1, 1);
2957         if (!err)
2958                 err = t3_reenable_adapter(adapter);
2959         if (!err)
2960                 t3_resume_ports(adapter);
2961
2962         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2963         rtnl_unlock();
2964 }
2965
2966 void t3_fatal_err(struct adapter *adapter)
2967 {
2968         unsigned int fw_status[4];
2969
2970         if (adapter->flags & FULL_INIT_DONE) {
2971                 t3_sge_stop(adapter);
2972                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2973                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2974                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2975                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2976
2977                 spin_lock(&adapter->work_lock);
2978                 t3_intr_disable(adapter);
2979                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2980                 spin_unlock(&adapter->work_lock);
2981         }
2982         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2983         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2984                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2985                          fw_status[0], fw_status[1],
2986                          fw_status[2], fw_status[3]);
2987 }
2988
2989 /**
2990  * t3_io_error_detected - called when PCI error is detected
2991  * @pdev: Pointer to PCI device
2992  * @state: The current pci connection state
2993  *
2994  * This function is called after a PCI bus error affecting
2995  * this device has been detected.
2996  */
2997 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2998                                              pci_channel_state_t state)
2999 {
3000         struct adapter *adapter = pci_get_drvdata(pdev);
3001
3002         if (state == pci_channel_io_perm_failure)
3003                 return PCI_ERS_RESULT_DISCONNECT;
3004
3005         t3_adapter_error(adapter, 0, 0);
3006
3007         /* Request a slot reset. */
3008         return PCI_ERS_RESULT_NEED_RESET;
3009 }
3010
3011 /**
3012  * t3_io_slot_reset - called after the pci bus has been reset.
3013  * @pdev: Pointer to PCI device
3014  *
3015  * Restart the card from scratch, as if from a cold-boot.
3016  */
3017 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3018 {
3019         struct adapter *adapter = pci_get_drvdata(pdev);
3020
3021         if (!t3_reenable_adapter(adapter))
3022                 return PCI_ERS_RESULT_RECOVERED;
3023
3024         return PCI_ERS_RESULT_DISCONNECT;
3025 }
3026
3027 /**
3028  * t3_io_resume - called when traffic can start flowing again.
3029  * @pdev: Pointer to PCI device
3030  *
3031  * This callback is called when the error recovery driver tells us that
3032  * its OK to resume normal operation.
3033  */
3034 static void t3_io_resume(struct pci_dev *pdev)
3035 {
3036         struct adapter *adapter = pci_get_drvdata(pdev);
3037
3038         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3039                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
3040
3041         rtnl_lock();
3042         t3_resume_ports(adapter);
3043         rtnl_unlock();
3044 }
3045
3046 static const struct pci_error_handlers t3_err_handler = {
3047         .error_detected = t3_io_error_detected,
3048         .slot_reset = t3_io_slot_reset,
3049         .resume = t3_io_resume,
3050 };
3051
3052 /*
3053  * Set the number of qsets based on the number of CPUs and the number of ports,
3054  * not to exceed the number of available qsets, assuming there are enough qsets
3055  * per port in HW.
3056  */
3057 static void set_nqsets(struct adapter *adap)
3058 {
3059         int i, j = 0;
3060         int num_cpus = netif_get_num_default_rss_queues();
3061         int hwports = adap->params.nports;
3062         int nqsets = adap->msix_nvectors - 1;
3063
3064         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3065                 if (hwports == 2 &&
3066                     (hwports * nqsets > SGE_QSETS ||
3067                      num_cpus >= nqsets / hwports))
3068                         nqsets /= hwports;
3069                 if (nqsets > num_cpus)
3070                         nqsets = num_cpus;
3071                 if (nqsets < 1 || hwports == 4)
3072                         nqsets = 1;
3073         } else
3074                 nqsets = 1;
3075
3076         for_each_port(adap, i) {
3077                 struct port_info *pi = adap2pinfo(adap, i);
3078
3079                 pi->first_qset = j;
3080                 pi->nqsets = nqsets;
3081                 j = pi->first_qset + nqsets;
3082
3083                 dev_info(&adap->pdev->dev,
3084                          "Port %d using %d queue sets.\n", i, nqsets);
3085         }
3086 }
3087
3088 static int cxgb_enable_msix(struct adapter *adap)
3089 {
3090         struct msix_entry entries[SGE_QSETS + 1];
3091         int vectors;
3092         int i;
3093
3094         vectors = ARRAY_SIZE(entries);
3095         for (i = 0; i < vectors; ++i)
3096                 entries[i].entry = i;
3097
3098         vectors = pci_enable_msix_range(adap->pdev, entries,
3099                                         adap->params.nports + 1, vectors);
3100         if (vectors < 0)
3101                 return vectors;
3102
3103         for (i = 0; i < vectors; ++i)
3104                 adap->msix_info[i].vec = entries[i].vector;
3105         adap->msix_nvectors = vectors;
3106
3107         return 0;
3108 }
3109
3110 static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
3111 {
3112         static const char *pci_variant[] = {
3113                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3114         };
3115
3116         int i;
3117         char buf[80];
3118
3119         if (is_pcie(adap))
3120                 snprintf(buf, sizeof(buf), "%s x%d",
3121                          pci_variant[adap->params.pci.variant],
3122                          adap->params.pci.width);
3123         else
3124                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3125                          pci_variant[adap->params.pci.variant],
3126                          adap->params.pci.speed, adap->params.pci.width);
3127
3128         for_each_port(adap, i) {
3129                 struct net_device *dev = adap->port[i];
3130                 const struct port_info *pi = netdev_priv(dev);
3131
3132                 if (!test_bit(i, &adap->registered_device_map))
3133                         continue;
3134                 netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3135                             ai->desc, pi->phy.desc,
3136                             is_offload(adap) ? "R" : "", adap->params.rev, buf,
3137                             (adap->flags & USING_MSIX) ? " MSI-X" :
3138                             (adap->flags & USING_MSI) ? " MSI" : "");
3139                 if (adap->name == dev->name && adap->params.vpd.mclk)
3140                         pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3141                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3142                                t3_mc7_size(&adap->pmtx) >> 20,
3143                                t3_mc7_size(&adap->pmrx) >> 20,
3144                                adap->params.vpd.sn);
3145         }
3146 }
3147
3148 static const struct net_device_ops cxgb_netdev_ops = {
3149         .ndo_open               = cxgb_open,
3150         .ndo_stop               = cxgb_close,
3151         .ndo_start_xmit         = t3_eth_xmit,
3152         .ndo_get_stats          = cxgb_get_stats,
3153         .ndo_validate_addr      = eth_validate_addr,
3154         .ndo_set_rx_mode        = cxgb_set_rxmode,
3155         .ndo_do_ioctl           = cxgb_ioctl,
3156         .ndo_change_mtu         = cxgb_change_mtu,
3157         .ndo_set_mac_address    = cxgb_set_mac_addr,
3158         .ndo_fix_features       = cxgb_fix_features,
3159         .ndo_set_features       = cxgb_set_features,
3160 #ifdef CONFIG_NET_POLL_CONTROLLER
3161         .ndo_poll_controller    = cxgb_netpoll,
3162 #endif
3163 };
3164
3165 static void cxgb3_init_iscsi_mac(struct net_device *dev)
3166 {
3167         struct port_info *pi = netdev_priv(dev);
3168
3169         memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3170         pi->iscsic.mac_addr[3] |= 0x80;
3171 }
3172
3173 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3174 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3175                         NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3176 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
3177 {
3178         int i, err, pci_using_dac = 0;
3179         resource_size_t mmio_start, mmio_len;
3180         const struct adapter_info *ai;
3181         struct adapter *adapter = NULL;
3182         struct port_info *pi;
3183
3184         pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
3185
3186         if (!cxgb3_wq) {
3187                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3188                 if (!cxgb3_wq) {
3189                         pr_err("cannot initialize work queue\n");
3190                         return -ENOMEM;
3191                 }
3192         }
3193
3194         err = pci_enable_device(pdev);
3195         if (err) {
3196                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3197                 goto out;
3198         }
3199
3200         err = pci_request_regions(pdev, DRV_NAME);
3201         if (err) {
3202                 /* Just info, some other driver may have claimed the device. */
3203                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3204                 goto out_disable_device;
3205         }
3206
3207         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3208                 pci_using_dac = 1;
3209                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3210                 if (err) {
3211                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3212                                "coherent allocations\n");
3213                         goto out_release_regions;
3214                 }
3215         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3216                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3217                 goto out_release_regions;
3218         }
3219
3220         pci_set_master(pdev);
3221         pci_save_state(pdev);
3222
3223         mmio_start = pci_resource_start(pdev, 0);
3224         mmio_len = pci_resource_len(pdev, 0);
3225         ai = t3_get_adapter_info(ent->driver_data);
3226
3227         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3228         if (!adapter) {
3229                 err = -ENOMEM;
3230                 goto out_release_regions;
3231         }
3232
3233         adapter->nofail_skb =
3234                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3235         if (!adapter->nofail_skb) {
3236                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3237                 err = -ENOMEM;
3238                 goto out_free_adapter;
3239         }
3240
3241         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3242         if (!adapter->regs) {
3243                 dev_err(&pdev->dev, "cannot map device registers\n");
3244                 err = -ENOMEM;
3245                 goto out_free_adapter;
3246         }
3247
3248         adapter->pdev = pdev;
3249         adapter->name = pci_name(pdev);
3250         adapter->msg_enable = dflt_msg_enable;
3251         adapter->mmio_len = mmio_len;
3252
3253         mutex_init(&adapter->mdio_lock);
3254         spin_lock_init(&adapter->work_lock);
3255         spin_lock_init(&adapter->stats_lock);
3256
3257         INIT_LIST_HEAD(&adapter->adapter_list);
3258         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3259         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3260
3261         INIT_WORK(&adapter->db_full_task, db_full_task);
3262         INIT_WORK(&adapter->db_empty_task, db_empty_task);
3263         INIT_WORK(&adapter->db_drop_task, db_drop_task);
3264
3265         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3266
3267         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3268                 struct net_device *netdev;
3269
3270                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3271                 if (!netdev) {
3272                         err = -ENOMEM;
3273                         goto out_free_dev;
3274                 }
3275
3276                 SET_NETDEV_DEV(netdev, &pdev->dev);
3277
3278                 adapter->port[i] = netdev;
3279                 pi = netdev_priv(netdev);
3280                 pi->adapter = adapter;
3281                 pi->port_id = i;
3282                 netif_carrier_off(netdev);
3283                 netdev->irq = pdev->irq;
3284                 netdev->mem_start = mmio_start;
3285                 netdev->mem_end = mmio_start + mmio_len - 1;
3286                 netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3287                         NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
3288                 netdev->features |= netdev->hw_features |
3289                                     NETIF_F_HW_VLAN_CTAG_TX;
3290                 netdev->vlan_features |= netdev->features & VLAN_FEAT;
3291                 if (pci_using_dac)
3292                         netdev->features |= NETIF_F_HIGHDMA;
3293
3294                 netdev->netdev_ops = &cxgb_netdev_ops;
3295                 netdev->ethtool_ops = &cxgb_ethtool_ops;
3296                 netdev->min_mtu = 81;
3297                 netdev->max_mtu = ETH_MAX_MTU;
3298         }
3299
3300         pci_set_drvdata(pdev, adapter);
3301         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3302                 err = -ENODEV;
3303                 goto out_free_dev;
3304         }
3305
3306         /*
3307          * The card is now ready to go.  If any errors occur during device
3308          * registration we do not fail the whole card but rather proceed only
3309          * with the ports we manage to register successfully.  However we must
3310          * register at least one net device.
3311          */
3312         for_each_port(adapter, i) {
3313                 err = register_netdev(adapter->port[i]);
3314                 if (err)
3315                         dev_warn(&pdev->dev,
3316                                  "cannot register net device %s, skipping\n",
3317                                  adapter->port[i]->name);
3318                 else {
3319                         /*
3320                          * Change the name we use for messages to the name of
3321                          * the first successfully registered interface.
3322                          */
3323                         if (!adapter->registered_device_map)
3324                                 adapter->name = adapter->port[i]->name;
3325
3326                         __set_bit(i, &adapter->registered_device_map);
3327                 }
3328         }
3329         if (!adapter->registered_device_map) {
3330                 dev_err(&pdev->dev, "could not register any net devices\n");
3331                 goto out_free_dev;
3332         }
3333
3334         for_each_port(adapter, i)
3335                 cxgb3_init_iscsi_mac(adapter->port[i]);
3336
3337         /* Driver's ready. Reflect it on LEDs */
3338         t3_led_ready(adapter);
3339
3340         if (is_offload(adapter)) {
3341                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3342                 cxgb3_adapter_ofld(adapter);
3343         }
3344
3345         /* See what interrupts we'll be using */
3346         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3347                 adapter->flags |= USING_MSIX;
3348         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3349                 adapter->flags |= USING_MSI;
3350
3351         set_nqsets(adapter);
3352
3353         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3354                                  &cxgb3_attr_group);
3355
3356         print_port_info(adapter, ai);
3357         return 0;
3358
3359 out_free_dev:
3360         iounmap(adapter->regs);
3361         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3362                 if (adapter->port[i])
3363                         free_netdev(adapter->port[i]);
3364
3365 out_free_adapter:
3366         kfree(adapter);
3367
3368 out_release_regions:
3369         pci_release_regions(pdev);
3370 out_disable_device:
3371         pci_disable_device(pdev);
3372 out:
3373         return err;
3374 }
3375
3376 static void remove_one(struct pci_dev *pdev)
3377 {
3378         struct adapter *adapter = pci_get_drvdata(pdev);
3379
3380         if (adapter) {
3381                 int i;
3382
3383                 t3_sge_stop(adapter);
3384                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3385                                    &cxgb3_attr_group);
3386
3387                 if (is_offload(adapter)) {
3388                         cxgb3_adapter_unofld(adapter);
3389                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3390                                      &adapter->open_device_map))
3391                                 offload_close(&adapter->tdev);
3392                 }
3393
3394                 for_each_port(adapter, i)
3395                     if (test_bit(i, &adapter->registered_device_map))
3396                         unregister_netdev(adapter->port[i]);
3397
3398                 t3_stop_sge_timers(adapter);
3399                 t3_free_sge_resources(adapter);
3400                 cxgb_disable_msi(adapter);
3401
3402                 for_each_port(adapter, i)
3403                         if (adapter->port[i])
3404                                 free_netdev(adapter->port[i]);
3405
3406                 iounmap(adapter->regs);
3407                 if (adapter->nofail_skb)
3408                         kfree_skb(adapter->nofail_skb);
3409                 kfree(adapter);
3410                 pci_release_regions(pdev);
3411                 pci_disable_device(pdev);
3412         }
3413 }
3414
3415 static struct pci_driver driver = {
3416         .name = DRV_NAME,
3417         .id_table = cxgb3_pci_tbl,
3418         .probe = init_one,
3419         .remove = remove_one,
3420         .err_handler = &t3_err_handler,
3421 };
3422
3423 static int __init cxgb3_init_module(void)
3424 {
3425         int ret;
3426
3427         cxgb3_offload_init();
3428
3429         ret = pci_register_driver(&driver);
3430         return ret;
3431 }
3432
3433 static void __exit cxgb3_cleanup_module(void)
3434 {
3435         pci_unregister_driver(&driver);
3436         if (cxgb3_wq)
3437                 destroy_workqueue(cxgb3_wq);
3438 }
3439
3440 module_init(cxgb3_init_module);
3441 module_exit(cxgb3_cleanup_module);