drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mdio.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <linux/stringify.h>
  48 #include <linux/sched.h>
  49 #include <asm/uaccess.h>
  50
  51 #include "common.h"
  52 #include "cxgb3_ioctl.h"
  53 #include "regs.h"
  54 #include "cxgb3_offload.h"
  55 #include "version.h"
  56
  57 #include "cxgb3_ctl_defs.h"
  58 #include "t3_cpl.h"
  59 #include "firmware_exports.h"
  60
  61 enum {
  62         MAX_TXQ_ENTRIES = 16384,
  63         MAX_CTRL_TXQ_ENTRIES = 1024,
  64         MAX_RSPQ_ENTRIES = 16384,
  65         MAX_RX_BUFFERS = 16384,
  66         MAX_RX_JUMBO_BUFFERS = 16384,
  67         MIN_TXQ_ENTRIES = 4,
  68         MIN_CTRL_TXQ_ENTRIES = 4,
  69         MIN_RSPQ_ENTRIES = 32,
  70         MIN_FL_ENTRIES = 32
  71 };
  72
  73 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  74
  75 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  76                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  77                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  78
  79 #define EEPROM_MAGIC 0x38E2F10C
  80
  81 #define CH_DEVICE(devid, idx) \
  82         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  83
  84 static DEFINE_PCI_DEVICE_TABLE(cxgb3_pci_tbl) = {
  85         CH_DEVICE(0x20, 0),     /* PE9000 */
  86         CH_DEVICE(0x21, 1),     /* T302E */
  87         CH_DEVICE(0x22, 2),     /* T310E */
  88         CH_DEVICE(0x23, 3),     /* T320X */
  89         CH_DEVICE(0x24, 1),     /* T302X */
  90         CH_DEVICE(0x25, 3),     /* T320E */
  91         CH_DEVICE(0x26, 2),     /* T310X */
  92         CH_DEVICE(0x30, 2),     /* T3B10 */
  93         CH_DEVICE(0x31, 3),     /* T3B20 */
  94         CH_DEVICE(0x32, 1),     /* T3B02 */
  95         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
  96         CH_DEVICE(0x36, 3),     /* S320E-CR */
  97         CH_DEVICE(0x37, 7),     /* N320E-G2 */
  98         {0,}
  99 };
 100
 101 MODULE_DESCRIPTION(DRV_DESC);
 102 MODULE_AUTHOR("Chelsio Communications");
 103 MODULE_LICENSE("Dual BSD/GPL");
 104 MODULE_VERSION(DRV_VERSION);
 105 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 106
 107 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 108
 109 module_param(dflt_msg_enable, int, 0644);
 110 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 111
 112 /*
 113  * The driver uses the best interrupt scheme available on a platform in the
 114  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 115  * of these schemes the driver may consider as follows:
 116  *
 117  * msi = 2: choose from among all three options
 118  * msi = 1: only consider MSI and pin interrupts
 119  * msi = 0: force pin interrupts
 120  */
 121 static int msi = 2;
 122
 123 module_param(msi, int, 0644);
 124 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 125
 126 /*
 127  * The driver enables offload as a default.
 128  * To disable it, use ofld_disable = 1.
 129  */
 130
 131 static int ofld_disable = 0;
 132
 133 module_param(ofld_disable, int, 0644);
 134 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 135
 136 /*
 137  * We have work elements that we need to cancel when an interface is taken
 138  * down.  Normally the work elements would be executed by keventd but that
 139  * can deadlock because of linkwatch.  If our close method takes the rtnl
 140  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 141  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 142  * for our work to complete.  Get our own work queue to solve this.
 143  */
 144 struct workqueue_struct *cxgb3_wq;
 145
 146 /**
 147  *      link_report - show link status and link speed/duplex
 148  *      @p: the port whose settings are to be reported
 149  *
 150  *      Shows the link status, speed, and duplex of a port.
 151  */
 152 static void link_report(struct net_device *dev)
 153 {
 154         if (!netif_carrier_ok(dev))
 155                 printk(KERN_INFO "%s: link down\n", dev->name);
 156         else {
 157                 const char *s = "10Mbps";
 158                 const struct port_info *p = netdev_priv(dev);
 159
 160                 switch (p->link_config.speed) {
 161                 case SPEED_10000:
 162                         s = "10Gbps";
 163                         break;
 164                 case SPEED_1000:
 165                         s = "1000Mbps";
 166                         break;
 167                 case SPEED_100:
 168                         s = "100Mbps";
 169                         break;
 170                 }
 171
 172                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 173                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 174         }
 175 }
 176
 177 static void enable_tx_fifo_drain(struct adapter *adapter,
 178                                  struct port_info *pi)
 179 {
 180         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 181                          F_ENDROPPKT);
 182         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 183         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 184         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 185 }
 186
 187 static void disable_tx_fifo_drain(struct adapter *adapter,
 188                                   struct port_info *pi)
 189 {
 190         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 191                          F_ENDROPPKT, 0);
 192 }
 193
 194 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 195 {
 196         struct net_device *dev = adap->port[port_id];
 197         struct port_info *pi = netdev_priv(dev);
 198
 199         if (state == netif_carrier_ok(dev))
 200                 return;
 201
 202         if (state) {
 203                 struct cmac *mac = &pi->mac;
 204
 205                 netif_carrier_on(dev);
 206
 207                 disable_tx_fifo_drain(adap, pi);
 208
 209                 /* Clear local faults */
 210                 t3_xgm_intr_disable(adap, pi->port_id);
 211                 t3_read_reg(adap, A_XGM_INT_STATUS +
 212                                     pi->mac.offset);
 213                 t3_write_reg(adap,
 214                              A_XGM_INT_CAUSE + pi->mac.offset,
 215                              F_XGM_INT);
 216
 217                 t3_set_reg_field(adap,
 218                                  A_XGM_INT_ENABLE +
 219                                  pi->mac.offset,
 220                                  F_XGM_INT, F_XGM_INT);
 221                 t3_xgm_intr_enable(adap, pi->port_id);
 222
 223                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 224         } else {
 225                 netif_carrier_off(dev);
 226
 227                 /* Flush TX FIFO */
 228                 enable_tx_fifo_drain(adap, pi);
 229         }
 230         link_report(dev);
 231 }
 232
 233 /**
 234  *      t3_os_link_changed - handle link status changes
 235  *      @adapter: the adapter associated with the link change
 236  *      @port_id: the port index whose limk status has changed
 237  *      @link_stat: the new status of the link
 238  *      @speed: the new speed setting
 239  *      @duplex: the new duplex setting
 240  *      @pause: the new flow-control setting
 241  *
 242  *      This is the OS-dependent handler for link status changes.  The OS
 243  *      neutral handler takes care of most of the processing for these events,
 244  *      then calls this handler for any OS-specific processing.
 245  */
 246 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 247                         int speed, int duplex, int pause)
 248 {
 249         struct net_device *dev = adapter->port[port_id];
 250         struct port_info *pi = netdev_priv(dev);
 251         struct cmac *mac = &pi->mac;
 252
 253         /* Skip changes from disabled ports. */
 254         if (!netif_running(dev))
 255                 return;
 256
 257         if (link_stat != netif_carrier_ok(dev)) {
 258                 if (link_stat) {
 259                         disable_tx_fifo_drain(adapter, pi);
 260
 261                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 262
 263                         /* Clear local faults */
 264                         t3_xgm_intr_disable(adapter, pi->port_id);
 265                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 266                                     pi->mac.offset);
 267                         t3_write_reg(adapter,
 268                                      A_XGM_INT_CAUSE + pi->mac.offset,
 269                                      F_XGM_INT);
 270
 271                         t3_set_reg_field(adapter,
 272                                          A_XGM_INT_ENABLE + pi->mac.offset,
 273                                          F_XGM_INT, F_XGM_INT);
 274                         t3_xgm_intr_enable(adapter, pi->port_id);
 275
 276                         netif_carrier_on(dev);
 277                 } else {
 278                         netif_carrier_off(dev);
 279
 280                         t3_xgm_intr_disable(adapter, pi->port_id);
 281                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 282                         t3_set_reg_field(adapter,
 283                                          A_XGM_INT_ENABLE + pi->mac.offset,
 284                                          F_XGM_INT, 0);
 285
 286                         if (is_10G(adapter))
 287                                 pi->phy.ops->power_down(&pi->phy, 1);
 288
 289                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 290                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 291                         t3_link_start(&pi->phy, mac, &pi->link_config);
 292
 293                         /* Flush TX FIFO */
 294                         enable_tx_fifo_drain(adapter, pi);
 295                 }
 296
 297                 link_report(dev);
 298         }
 299 }
 300
 301 /**
 302  *      t3_os_phymod_changed - handle PHY module changes
 303  *      @phy: the PHY reporting the module change
 304  *      @mod_type: new module type
 305  *
 306  *      This is the OS-dependent handler for PHY module changes.  It is
 307  *      invoked when a PHY module is removed or inserted for any OS-specific
 308  *      processing.
 309  */
 310 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 311 {
 312         static const char *mod_str[] = {
 313                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 314         };
 315
 316         const struct net_device *dev = adap->port[port_id];
 317         const struct port_info *pi = netdev_priv(dev);
 318
 319         if (pi->phy.modtype == phy_modtype_none)
 320                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 321         else
 322                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 323                        mod_str[pi->phy.modtype]);
 324 }
 325
 326 static void cxgb_set_rxmode(struct net_device *dev)
 327 {
 328         struct port_info *pi = netdev_priv(dev);
 329
 330         t3_mac_set_rx_mode(&pi->mac, dev);
 331 }
 332
 333 /**
 334  *      link_start - enable a port
 335  *      @dev: the device to enable
 336  *
 337  *      Performs the MAC and PHY actions needed to enable a port.
 338  */
 339 static void link_start(struct net_device *dev)
 340 {
 341         struct port_info *pi = netdev_priv(dev);
 342         struct cmac *mac = &pi->mac;
 343
 344         t3_mac_reset(mac);
 345         t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
 346         t3_mac_set_mtu(mac, dev->mtu);
 347         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
 348         t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
 349         t3_mac_set_rx_mode(mac, dev);
 350         t3_link_start(&pi->phy, mac, &pi->link_config);
 351         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 352 }
 353
 354 static inline void cxgb_disable_msi(struct adapter *adapter)
 355 {
 356         if (adapter->flags & USING_MSIX) {
 357                 pci_disable_msix(adapter->pdev);
 358                 adapter->flags &= ~USING_MSIX;
 359         } else if (adapter->flags & USING_MSI) {
 360                 pci_disable_msi(adapter->pdev);
 361                 adapter->flags &= ~USING_MSI;
 362         }
 363 }
 364
 365 /*
 366  * Interrupt handler for asynchronous events used with MSI-X.
 367  */
 368 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 369 {
 370         t3_slow_intr_handler(cookie);
 371         return IRQ_HANDLED;
 372 }
 373
 374 /*
 375  * Name the MSI-X interrupts.
 376  */
 377 static void name_msix_vecs(struct adapter *adap)
 378 {
 379         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 380
 381         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 382         adap->msix_info[0].desc[n] = 0;
 383
 384         for_each_port(adap, j) {
 385                 struct net_device *d = adap->port[j];
 386                 const struct port_info *pi = netdev_priv(d);
 387
 388                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 389                         snprintf(adap->msix_info[msi_idx].desc, n,
 390                                  "%s-%d", d->name, pi->first_qset + i);
 391                         adap->msix_info[msi_idx].desc[n] = 0;
 392                 }
 393         }
 394 }
 395
 396 static int request_msix_data_irqs(struct adapter *adap)
 397 {
 398         int i, j, err, qidx = 0;
 399
 400         for_each_port(adap, i) {
 401                 int nqsets = adap2pinfo(adap, i)->nqsets;
 402
 403                 for (j = 0; j < nqsets; ++j) {
 404                         err = request_irq(adap->msix_info[qidx + 1].vec,
 405                                           t3_intr_handler(adap,
 406                                                           adap->sge.qs[qidx].
 407                                                           rspq.polling), 0,
 408                                           adap->msix_info[qidx + 1].desc,
 409                                           &adap->sge.qs[qidx]);
 410                         if (err) {
 411                                 while (--qidx >= 0)
 412                                         free_irq(adap->msix_info[qidx + 1].vec,
 413                                                  &adap->sge.qs[qidx]);
 414                                 return err;
 415                         }
 416                         qidx++;
 417                 }
 418         }
 419         return 0;
 420 }
 421
 422 static void free_irq_resources(struct adapter *adapter)
 423 {
 424         if (adapter->flags & USING_MSIX) {
 425                 int i, n = 0;
 426
 427                 free_irq(adapter->msix_info[0].vec, adapter);
 428                 for_each_port(adapter, i)
 429                         n += adap2pinfo(adapter, i)->nqsets;
 430
 431                 for (i = 0; i < n; ++i)
 432                         free_irq(adapter->msix_info[i + 1].vec,
 433                                  &adapter->sge.qs[i]);
 434         } else
 435                 free_irq(adapter->pdev->irq, adapter);
 436 }
 437
 438 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 439                               unsigned long n)
 440 {
 441         int attempts = 5;
 442
 443         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 444                 if (!--attempts)
 445                         return -ETIMEDOUT;
 446                 msleep(10);
 447         }
 448         return 0;
 449 }
 450
 451 static int init_tp_parity(struct adapter *adap)
 452 {
 453         int i;
 454         struct sk_buff *skb;
 455         struct cpl_set_tcb_field *greq;
 456         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 457
 458         t3_tp_set_offload_mode(adap, 1);
 459
 460         for (i = 0; i < 16; i++) {
 461                 struct cpl_smt_write_req *req;
 462
 463                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 464                 if (!skb)
 465                         skb = adap->nofail_skb;
 466                 if (!skb)
 467                         goto alloc_skb_fail;
 468
 469                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 470                 memset(req, 0, sizeof(*req));
 471                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 472                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 473                 req->mtu_idx = NMTUS - 1;
 474                 req->iff = i;
 475                 t3_mgmt_tx(adap, skb);
 476                 if (skb == adap->nofail_skb) {
 477                         await_mgmt_replies(adap, cnt, i + 1);
 478                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 479                         if (!adap->nofail_skb)
 480                                 goto alloc_skb_fail;
 481                 }
 482         }
 483
 484         for (i = 0; i < 2048; i++) {
 485                 struct cpl_l2t_write_req *req;
 486
 487                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 488                 if (!skb)
 489                         skb = adap->nofail_skb;
 490                 if (!skb)
 491                         goto alloc_skb_fail;
 492
 493                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 494                 memset(req, 0, sizeof(*req));
 495                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 496                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 497                 req->params = htonl(V_L2T_W_IDX(i));
 498                 t3_mgmt_tx(adap, skb);
 499                 if (skb == adap->nofail_skb) {
 500                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 501                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 502                         if (!adap->nofail_skb)
 503                                 goto alloc_skb_fail;
 504                 }
 505         }
 506
 507         for (i = 0; i < 2048; i++) {
 508                 struct cpl_rte_write_req *req;
 509
 510                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 511                 if (!skb)
 512                         skb = adap->nofail_skb;
 513                 if (!skb)
 514                         goto alloc_skb_fail;
 515
 516                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 517                 memset(req, 0, sizeof(*req));
 518                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 519                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 520                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 521                 t3_mgmt_tx(adap, skb);
 522                 if (skb == adap->nofail_skb) {
 523                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 524                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 525                         if (!adap->nofail_skb)
 526                                 goto alloc_skb_fail;
 527                 }
 528         }
 529
 530         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 531         if (!skb)
 532                 skb = adap->nofail_skb;
 533         if (!skb)
 534                 goto alloc_skb_fail;
 535
 536         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 537         memset(greq, 0, sizeof(*greq));
 538         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 539         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 540         greq->mask = cpu_to_be64(1);
 541         t3_mgmt_tx(adap, skb);
 542
 543         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 544         if (skb == adap->nofail_skb) {
 545                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 546                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 547         }
 548
 549         t3_tp_set_offload_mode(adap, 0);
 550         return i;
 551
 552 alloc_skb_fail:
 553         t3_tp_set_offload_mode(adap, 0);
 554         return -ENOMEM;
 555 }
 556
 557 /**
 558  *      setup_rss - configure RSS
 559  *      @adap: the adapter
 560  *
 561  *      Sets up RSS to distribute packets to multiple receive queues.  We
 562  *      configure the RSS CPU lookup table to distribute to the number of HW
 563  *      receive queues, and the response queue lookup table to narrow that
 564  *      down to the response queues actually configured for each port.
 565  *      We always configure the RSS mapping for two ports since the mapping
 566  *      table has plenty of entries.
 567  */
 568 static void setup_rss(struct adapter *adap)
 569 {
 570         int i;
 571         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 572         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 573         u8 cpus[SGE_QSETS + 1];
 574         u16 rspq_map[RSS_TABLE_SIZE];
 575
 576         for (i = 0; i < SGE_QSETS; ++i)
 577                 cpus[i] = i;
 578         cpus[SGE_QSETS] = 0xff; /* terminator */
 579
 580         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 581                 rspq_map[i] = i % nq0;
 582                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 583         }
 584
 585         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 586                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 587                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 588 }
 589
 590 static void ring_dbs(struct adapter *adap)
 591 {
 592         int i, j;
 593
 594         for (i = 0; i < SGE_QSETS; i++) {
 595                 struct sge_qset *qs = &adap->sge.qs[i];
 596
 597                 if (qs->adap)
 598                         for (j = 0; j < SGE_TXQ_PER_SET; j++)
 599                                 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
 600         }
 601 }
 602
 603 static void init_napi(struct adapter *adap)
 604 {
 605         int i;
 606
 607         for (i = 0; i < SGE_QSETS; i++) {
 608                 struct sge_qset *qs = &adap->sge.qs[i];
 609
 610                 if (qs->adap)
 611                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 612                                        64);
 613         }
 614
 615         /*
 616          * netif_napi_add() can be called only once per napi_struct because it
 617          * adds each new napi_struct to a list.  Be careful not to call it a
 618          * second time, e.g., during EEH recovery, by making a note of it.
 619          */
 620         adap->flags |= NAPI_INIT;
 621 }
 622
 623 /*
 624  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 625  * both netdevices representing interfaces and the dummy ones for the extra
 626  * queues.
 627  */
 628 static void quiesce_rx(struct adapter *adap)
 629 {
 630         int i;
 631
 632         for (i = 0; i < SGE_QSETS; i++)
 633                 if (adap->sge.qs[i].adap)
 634                         napi_disable(&adap->sge.qs[i].napi);
 635 }
 636
 637 static void enable_all_napi(struct adapter *adap)
 638 {
 639         int i;
 640         for (i = 0; i < SGE_QSETS; i++)
 641                 if (adap->sge.qs[i].adap)
 642                         napi_enable(&adap->sge.qs[i].napi);
 643 }
 644
 645 /**
 646  *      set_qset_lro - Turn a queue set's LRO capability on and off
 647  *      @dev: the device the qset is attached to
 648  *      @qset_idx: the queue set index
 649  *      @val: the LRO switch
 650  *
 651  *      Sets LRO on or off for a particular queue set.
 652  *      the device's features flag is updated to reflect the LRO
 653  *      capability when all queues belonging to the device are
 654  *      in the same state.
 655  */
 656 static void set_qset_lro(struct net_device *dev, int qset_idx, int val)
 657 {
 658         struct port_info *pi = netdev_priv(dev);
 659         struct adapter *adapter = pi->adapter;
 660
 661         adapter->params.sge.qset[qset_idx].lro = !!val;
 662         adapter->sge.qs[qset_idx].lro_enabled = !!val;
 663 }
 664
 665 /**
 666  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 667  *      @adap: the adapter
 668  *
 669  *      Determines how many sets of SGE queues to use and initializes them.
 670  *      We support multiple queue sets per port if we have MSI-X, otherwise
 671  *      just one queue set per port.
 672  */
 673 static int setup_sge_qsets(struct adapter *adap)
 674 {
 675         int i, j, err, irq_idx = 0, qset_idx = 0;
 676         unsigned int ntxq = SGE_TXQ_PER_SET;
 677
 678         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 679                 irq_idx = -1;
 680
 681         for_each_port(adap, i) {
 682                 struct net_device *dev = adap->port[i];
 683                 struct port_info *pi = netdev_priv(dev);
 684
 685                 pi->qs = &adap->sge.qs[pi->first_qset];
 686                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 687                         set_qset_lro(dev, qset_idx, pi->rx_offload & T3_LRO);
 688                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 689                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 690                                                              irq_idx,
 691                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 692                                 netdev_get_tx_queue(dev, j));
 693                         if (err) {
 694                                 t3_free_sge_resources(adap);
 695                                 return err;
 696                         }
 697                 }
 698         }
 699
 700         return 0;
 701 }
 702
 703 static ssize_t attr_show(struct device *d, char *buf,
 704                          ssize_t(*format) (struct net_device *, char *))
 705 {
 706         ssize_t len;
 707
 708         /* Synchronize with ioctls that may shut down the device */
 709         rtnl_lock();
 710         len = (*format) (to_net_dev(d), buf);
 711         rtnl_unlock();
 712         return len;
 713 }
 714
 715 static ssize_t attr_store(struct device *d,
 716                           const char *buf, size_t len,
 717                           ssize_t(*set) (struct net_device *, unsigned int),
 718                           unsigned int min_val, unsigned int max_val)
 719 {
 720         char *endp;
 721         ssize_t ret;
 722         unsigned int val;
 723
 724         if (!capable(CAP_NET_ADMIN))
 725                 return -EPERM;
 726
 727         val = simple_strtoul(buf, &endp, 0);
 728         if (endp == buf || val < min_val || val > max_val)
 729                 return -EINVAL;
 730
 731         rtnl_lock();
 732         ret = (*set) (to_net_dev(d), val);
 733         if (!ret)
 734                 ret = len;
 735         rtnl_unlock();
 736         return ret;
 737 }
 738
 739 #define CXGB3_SHOW(name, val_expr) \
 740 static ssize_t format_##name(struct net_device *dev, char *buf) \
 741 { \
 742         struct port_info *pi = netdev_priv(dev); \
 743         struct adapter *adap = pi->adapter; \
 744         return sprintf(buf, "%u\n", val_expr); \
 745 } \
 746 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 747                            char *buf) \
 748 { \
 749         return attr_show(d, buf, format_##name); \
 750 }
 751
 752 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 753 {
 754         struct port_info *pi = netdev_priv(dev);
 755         struct adapter *adap = pi->adapter;
 756         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 757
 758         if (adap->flags & FULL_INIT_DONE)
 759                 return -EBUSY;
 760         if (val && adap->params.rev == 0)
 761                 return -EINVAL;
 762         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 763             min_tids)
 764                 return -EINVAL;
 765         adap->params.mc5.nfilters = val;
 766         return 0;
 767 }
 768
 769 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 770                               const char *buf, size_t len)
 771 {
 772         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 773 }
 774
 775 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 776 {
 777         struct port_info *pi = netdev_priv(dev);
 778         struct adapter *adap = pi->adapter;
 779
 780         if (adap->flags & FULL_INIT_DONE)
 781                 return -EBUSY;
 782         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 783             MC5_MIN_TIDS)
 784                 return -EINVAL;
 785         adap->params.mc5.nservers = val;
 786         return 0;
 787 }
 788
 789 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 790                               const char *buf, size_t len)
 791 {
 792         return attr_store(d, buf, len, set_nservers, 0, ~0);
 793 }
 794
 795 #define CXGB3_ATTR_R(name, val_expr) \
 796 CXGB3_SHOW(name, val_expr) \
 797 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 798
 799 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 800 CXGB3_SHOW(name, val_expr) \
 801 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 802
 803 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 804 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 805 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 806
 807 static struct attribute *cxgb3_attrs[] = {
 808         &dev_attr_cam_size.attr,
 809         &dev_attr_nfilters.attr,
 810         &dev_attr_nservers.attr,
 811         NULL
 812 };
 813
 814 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 815
 816 static ssize_t tm_attr_show(struct device *d,
 817                             char *buf, int sched)
 818 {
 819         struct port_info *pi = netdev_priv(to_net_dev(d));
 820         struct adapter *adap = pi->adapter;
 821         unsigned int v, addr, bpt, cpt;
 822         ssize_t len;
 823
 824         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 825         rtnl_lock();
 826         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 827         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 828         if (sched & 1)
 829                 v >>= 16;
 830         bpt = (v >> 8) & 0xff;
 831         cpt = v & 0xff;
 832         if (!cpt)
 833                 len = sprintf(buf, "disabled\n");
 834         else {
 835                 v = (adap->params.vpd.cclk * 1000) / cpt;
 836                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 837         }
 838         rtnl_unlock();
 839         return len;
 840 }
 841
 842 static ssize_t tm_attr_store(struct device *d,
 843                              const char *buf, size_t len, int sched)
 844 {
 845         struct port_info *pi = netdev_priv(to_net_dev(d));
 846         struct adapter *adap = pi->adapter;
 847         unsigned int val;
 848         char *endp;
 849         ssize_t ret;
 850
 851         if (!capable(CAP_NET_ADMIN))
 852                 return -EPERM;
 853
 854         val = simple_strtoul(buf, &endp, 0);
 855         if (endp == buf || val > 10000000)
 856                 return -EINVAL;
 857
 858         rtnl_lock();
 859         ret = t3_config_sched(adap, val, sched);
 860         if (!ret)
 861                 ret = len;
 862         rtnl_unlock();
 863         return ret;
 864 }
 865
 866 #define TM_ATTR(name, sched) \
 867 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 868                            char *buf) \
 869 { \
 870         return tm_attr_show(d, buf, sched); \
 871 } \
 872 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 873                             const char *buf, size_t len) \
 874 { \
 875         return tm_attr_store(d, buf, len, sched); \
 876 } \
 877 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 878
 879 TM_ATTR(sched0, 0);
 880 TM_ATTR(sched1, 1);
 881 TM_ATTR(sched2, 2);
 882 TM_ATTR(sched3, 3);
 883 TM_ATTR(sched4, 4);
 884 TM_ATTR(sched5, 5);
 885 TM_ATTR(sched6, 6);
 886 TM_ATTR(sched7, 7);
 887
 888 static struct attribute *offload_attrs[] = {
 889         &dev_attr_sched0.attr,
 890         &dev_attr_sched1.attr,
 891         &dev_attr_sched2.attr,
 892         &dev_attr_sched3.attr,
 893         &dev_attr_sched4.attr,
 894         &dev_attr_sched5.attr,
 895         &dev_attr_sched6.attr,
 896         &dev_attr_sched7.attr,
 897         NULL
 898 };
 899
 900 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 901
 902 /*
 903  * Sends an sk_buff to an offload queue driver
 904  * after dealing with any active network taps.
 905  */
 906 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 907 {
 908         int ret;
 909
 910         local_bh_disable();
 911         ret = t3_offload_tx(tdev, skb);
 912         local_bh_enable();
 913         return ret;
 914 }
 915
 916 static int write_smt_entry(struct adapter *adapter, int idx)
 917 {
 918         struct cpl_smt_write_req *req;
 919         struct port_info *pi = netdev_priv(adapter->port[idx]);
 920         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 921
 922         if (!skb)
 923                 return -ENOMEM;
 924
 925         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 926         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 927         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 928         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 929         req->iff = idx;
 930         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 931         memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
 932         skb->priority = 1;
 933         offload_tx(&adapter->tdev, skb);
 934         return 0;
 935 }
 936
 937 static int init_smt(struct adapter *adapter)
 938 {
 939         int i;
 940
 941         for_each_port(adapter, i)
 942             write_smt_entry(adapter, i);
 943         return 0;
 944 }
 945
 946 static void init_port_mtus(struct adapter *adapter)
 947 {
 948         unsigned int mtus = adapter->port[0]->mtu;
 949
 950         if (adapter->port[1])
 951                 mtus |= adapter->port[1]->mtu << 16;
 952         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 953 }
 954
 955 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 956                               int hi, int port)
 957 {
 958         struct sk_buff *skb;
 959         struct mngt_pktsched_wr *req;
 960         int ret;
 961
 962         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 963         if (!skb)
 964                 skb = adap->nofail_skb;
 965         if (!skb)
 966                 return -ENOMEM;
 967
 968         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 969         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 970         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 971         req->sched = sched;
 972         req->idx = qidx;
 973         req->min = lo;
 974         req->max = hi;
 975         req->binding = port;
 976         ret = t3_mgmt_tx(adap, skb);
 977         if (skb == adap->nofail_skb) {
 978                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 979                                              GFP_KERNEL);
 980                 if (!adap->nofail_skb)
 981                         ret = -ENOMEM;
 982         }
 983
 984         return ret;
 985 }
 986
 987 static int bind_qsets(struct adapter *adap)
 988 {
 989         int i, j, err = 0;
 990
 991         for_each_port(adap, i) {
 992                 const struct port_info *pi = adap2pinfo(adap, i);
 993
 994                 for (j = 0; j < pi->nqsets; ++j) {
 995                         int ret = send_pktsched_cmd(adap, 1,
 996                                                     pi->first_qset + j, -1,
 997                                                     -1, i);
 998                         if (ret)
 999                                 err = ret;
1000                 }
1001         }
1002
1003         return err;
1004 }
1005
1006 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."                    \
1007         __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
1008 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
1009 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."                \
1010         __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
1011 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
1012 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
1013 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
1014 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
1015 MODULE_FIRMWARE(FW_FNAME);
1016 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1017 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1018 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1019 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1020 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1021
1022 static inline const char *get_edc_fw_name(int edc_idx)
1023 {
1024         const char *fw_name = NULL;
1025
1026         switch (edc_idx) {
1027         case EDC_OPT_AEL2005:
1028                 fw_name = AEL2005_OPT_EDC_NAME;
1029                 break;
1030         case EDC_TWX_AEL2005:
1031                 fw_name = AEL2005_TWX_EDC_NAME;
1032                 break;
1033         case EDC_TWX_AEL2020:
1034                 fw_name = AEL2020_TWX_EDC_NAME;
1035                 break;
1036         }
1037         return fw_name;
1038 }
1039
1040 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1041 {
1042         struct adapter *adapter = phy->adapter;
1043         const struct firmware *fw;
1044         char buf[64];
1045         u32 csum;
1046         const __be32 *p;
1047         u16 *cache = phy->phy_cache;
1048         int i, ret;
1049
1050         snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1051
1052         ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1053         if (ret < 0) {
1054                 dev_err(&adapter->pdev->dev,
1055                         "could not upgrade firmware: unable to load %s\n",
1056                         buf);
1057                 return ret;
1058         }
1059
1060         /* check size, take checksum in account */
1061         if (fw->size > size + 4) {
1062                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1063                        (unsigned int)fw->size, size + 4);
1064                 ret = -EINVAL;
1065         }
1066
1067         /* compute checksum */
1068         p = (const __be32 *)fw->data;
1069         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1070                 csum += ntohl(p[i]);
1071
1072         if (csum != 0xffffffff) {
1073                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1074                        csum);
1075                 ret = -EINVAL;
1076         }
1077
1078         for (i = 0; i < size / 4 ; i++) {
1079                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1080                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1081         }
1082
1083         release_firmware(fw);
1084
1085         return ret;
1086 }
1087
1088 static int upgrade_fw(struct adapter *adap)
1089 {
1090         int ret;
1091         const struct firmware *fw;
1092         struct device *dev = &adap->pdev->dev;
1093
1094         ret = request_firmware(&fw, FW_FNAME, dev);
1095         if (ret < 0) {
1096                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1097                         FW_FNAME);
1098                 return ret;
1099         }
1100         ret = t3_load_fw(adap, fw->data, fw->size);
1101         release_firmware(fw);
1102
1103         if (ret == 0)
1104                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1105                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1106         else
1107                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1108                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1109
1110         return ret;
1111 }
1112
1113 static inline char t3rev2char(struct adapter *adapter)
1114 {
1115         char rev = 0;
1116
1117         switch(adapter->params.rev) {
1118         case T3_REV_B:
1119         case T3_REV_B2:
1120                 rev = 'b';
1121                 break;
1122         case T3_REV_C:
1123                 rev = 'c';
1124                 break;
1125         }
1126         return rev;
1127 }
1128
1129 static int update_tpsram(struct adapter *adap)
1130 {
1131         const struct firmware *tpsram;
1132         char buf[64];
1133         struct device *dev = &adap->pdev->dev;
1134         int ret;
1135         char rev;
1136
1137         rev = t3rev2char(adap);
1138         if (!rev)
1139                 return 0;
1140
1141         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1142
1143         ret = request_firmware(&tpsram, buf, dev);
1144         if (ret < 0) {
1145                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1146                         buf);
1147                 return ret;
1148         }
1149
1150         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1151         if (ret)
1152                 goto release_tpsram;
1153
1154         ret = t3_set_proto_sram(adap, tpsram->data);
1155         if (ret == 0)
1156                 dev_info(dev,
1157                          "successful update of protocol engine "
1158                          "to %d.%d.%d\n",
1159                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1160         else
1161                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1162                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1163         if (ret)
1164                 dev_err(dev, "loading protocol SRAM failed\n");
1165
1166 release_tpsram:
1167         release_firmware(tpsram);
1168
1169         return ret;
1170 }
1171
1172 /**
1173  *      cxgb_up - enable the adapter
1174  *      @adapter: adapter being enabled
1175  *
1176  *      Called when the first port is enabled, this function performs the
1177  *      actions necessary to make an adapter operational, such as completing
1178  *      the initialization of HW modules, and enabling interrupts.
1179  *
1180  *      Must be called with the rtnl lock held.
1181  */
1182 static int cxgb_up(struct adapter *adap)
1183 {
1184         int err;
1185
1186         if (!(adap->flags & FULL_INIT_DONE)) {
1187                 err = t3_check_fw_version(adap);
1188                 if (err == -EINVAL) {
1189                         err = upgrade_fw(adap);
1190                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1191                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1192                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1193                 }
1194
1195                 err = t3_check_tpsram_version(adap);
1196                 if (err == -EINVAL) {
1197                         err = update_tpsram(adap);
1198                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1199                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1200                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1201                 }
1202
1203                 /*
1204                  * Clear interrupts now to catch errors if t3_init_hw fails.
1205                  * We clear them again later as initialization may trigger
1206                  * conditions that can interrupt.
1207                  */
1208                 t3_intr_clear(adap);
1209
1210                 err = t3_init_hw(adap, 0);
1211                 if (err)
1212                         goto out;
1213
1214                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1215                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1216
1217                 err = setup_sge_qsets(adap);
1218                 if (err)
1219                         goto out;
1220
1221                 setup_rss(adap);
1222                 if (!(adap->flags & NAPI_INIT))
1223                         init_napi(adap);
1224
1225                 t3_start_sge_timers(adap);
1226                 adap->flags |= FULL_INIT_DONE;
1227         }
1228
1229         t3_intr_clear(adap);
1230
1231         if (adap->flags & USING_MSIX) {
1232                 name_msix_vecs(adap);
1233                 err = request_irq(adap->msix_info[0].vec,
1234                                   t3_async_intr_handler, 0,
1235                                   adap->msix_info[0].desc, adap);
1236                 if (err)
1237                         goto irq_err;
1238
1239                 err = request_msix_data_irqs(adap);
1240                 if (err) {
1241                         free_irq(adap->msix_info[0].vec, adap);
1242                         goto irq_err;
1243                 }
1244         } else if ((err = request_irq(adap->pdev->irq,
1245                                       t3_intr_handler(adap,
1246                                                       adap->sge.qs[0].rspq.
1247                                                       polling),
1248                                       (adap->flags & USING_MSI) ?
1249                                        0 : IRQF_SHARED,
1250                                       adap->name, adap)))
1251                 goto irq_err;
1252
1253         enable_all_napi(adap);
1254         t3_sge_start(adap);
1255         t3_intr_enable(adap);
1256
1257         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1258             is_offload(adap) && init_tp_parity(adap) == 0)
1259                 adap->flags |= TP_PARITY_INIT;
1260
1261         if (adap->flags & TP_PARITY_INIT) {
1262                 t3_write_reg(adap, A_TP_INT_CAUSE,
1263                              F_CMCACHEPERR | F_ARPLUTPERR);
1264                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1265         }
1266
1267         if (!(adap->flags & QUEUES_BOUND)) {
1268                 err = bind_qsets(adap);
1269                 if (err) {
1270                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1271                         t3_intr_disable(adap);
1272                         free_irq_resources(adap);
1273                         goto out;
1274                 }
1275                 adap->flags |= QUEUES_BOUND;
1276         }
1277
1278 out:
1279         return err;
1280 irq_err:
1281         CH_ERR(adap, "request_irq failed, err %d\n", err);
1282         goto out;
1283 }
1284
1285 /*
1286  * Release resources when all the ports and offloading have been stopped.
1287  */
1288 static void cxgb_down(struct adapter *adapter)
1289 {
1290         t3_sge_stop(adapter);
1291         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1292         t3_intr_disable(adapter);
1293         spin_unlock_irq(&adapter->work_lock);
1294
1295         free_irq_resources(adapter);
1296         quiesce_rx(adapter);
1297         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1298 }
1299
1300 static void schedule_chk_task(struct adapter *adap)
1301 {
1302         unsigned int timeo;
1303
1304         timeo = adap->params.linkpoll_period ?
1305             (HZ * adap->params.linkpoll_period) / 10 :
1306             adap->params.stats_update_period * HZ;
1307         if (timeo)
1308                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1309 }
1310
1311 static int offload_open(struct net_device *dev)
1312 {
1313         struct port_info *pi = netdev_priv(dev);
1314         struct adapter *adapter = pi->adapter;
1315         struct t3cdev *tdev = dev2t3cdev(dev);
1316         int adap_up = adapter->open_device_map & PORT_MASK;
1317         int err;
1318
1319         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1320                 return 0;
1321
1322         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1323                 goto out;
1324
1325         t3_tp_set_offload_mode(adapter, 1);
1326         tdev->lldev = adapter->port[0];
1327         err = cxgb3_offload_activate(adapter);
1328         if (err)
1329                 goto out;
1330
1331         init_port_mtus(adapter);
1332         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1333                      adapter->params.b_wnd,
1334                      adapter->params.rev == 0 ?
1335                      adapter->port[0]->mtu : 0xffff);
1336         init_smt(adapter);
1337
1338         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1339                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1340
1341         /* Call back all registered clients */
1342         cxgb3_add_clients(tdev);
1343
1344 out:
1345         /* restore them in case the offload module has changed them */
1346         if (err) {
1347                 t3_tp_set_offload_mode(adapter, 0);
1348                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1349                 cxgb3_set_dummy_ops(tdev);
1350         }
1351         return err;
1352 }
1353
1354 static int offload_close(struct t3cdev *tdev)
1355 {
1356         struct adapter *adapter = tdev2adap(tdev);
1357
1358         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1359                 return 0;
1360
1361         /* Call back all registered clients */
1362         cxgb3_remove_clients(tdev);
1363
1364         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1365
1366         /* Flush work scheduled while releasing TIDs */
1367         flush_scheduled_work();
1368
1369         tdev->lldev = NULL;
1370         cxgb3_set_dummy_ops(tdev);
1371         t3_tp_set_offload_mode(adapter, 0);
1372         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1373
1374         if (!adapter->open_device_map)
1375                 cxgb_down(adapter);
1376
1377         cxgb3_offload_deactivate(adapter);
1378         return 0;
1379 }
1380
1381 static int cxgb_open(struct net_device *dev)
1382 {
1383         struct port_info *pi = netdev_priv(dev);
1384         struct adapter *adapter = pi->adapter;
1385         int other_ports = adapter->open_device_map & PORT_MASK;
1386         int err;
1387
1388         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1389                 return err;
1390
1391         set_bit(pi->port_id, &adapter->open_device_map);
1392         if (is_offload(adapter) && !ofld_disable) {
1393                 err = offload_open(dev);
1394                 if (err)
1395                         printk(KERN_WARNING
1396                                "Could not initialize offload capabilities\n");
1397         }
1398
1399         dev->real_num_tx_queues = pi->nqsets;
1400         link_start(dev);
1401         t3_port_intr_enable(adapter, pi->port_id);
1402         netif_tx_start_all_queues(dev);
1403         if (!other_ports)
1404                 schedule_chk_task(adapter);
1405
1406         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1407         return 0;
1408 }
1409
1410 static int cxgb_close(struct net_device *dev)
1411 {
1412         struct port_info *pi = netdev_priv(dev);
1413         struct adapter *adapter = pi->adapter;
1414
1415
1416         if (!adapter->open_device_map)
1417                 return 0;
1418
1419         /* Stop link fault interrupts */
1420         t3_xgm_intr_disable(adapter, pi->port_id);
1421         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1422
1423         t3_port_intr_disable(adapter, pi->port_id);
1424         netif_tx_stop_all_queues(dev);
1425         pi->phy.ops->power_down(&pi->phy, 1);
1426         netif_carrier_off(dev);
1427         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1428
1429         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1430         clear_bit(pi->port_id, &adapter->open_device_map);
1431         spin_unlock_irq(&adapter->work_lock);
1432
1433         if (!(adapter->open_device_map & PORT_MASK))
1434                 cancel_delayed_work_sync(&adapter->adap_check_task);
1435
1436         if (!adapter->open_device_map)
1437                 cxgb_down(adapter);
1438
1439         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1440         return 0;
1441 }
1442
1443 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1444 {
1445         struct port_info *pi = netdev_priv(dev);
1446         struct adapter *adapter = pi->adapter;
1447         struct net_device_stats *ns = &pi->netstats;
1448         const struct mac_stats *pstats;
1449
1450         spin_lock(&adapter->stats_lock);
1451         pstats = t3_mac_update_stats(&pi->mac);
1452         spin_unlock(&adapter->stats_lock);
1453
1454         ns->tx_bytes = pstats->tx_octets;
1455         ns->tx_packets = pstats->tx_frames;
1456         ns->rx_bytes = pstats->rx_octets;
1457         ns->rx_packets = pstats->rx_frames;
1458         ns->multicast = pstats->rx_mcast_frames;
1459
1460         ns->tx_errors = pstats->tx_underrun;
1461         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1462             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1463             pstats->rx_fifo_ovfl;
1464
1465         /* detailed rx_errors */
1466         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1467         ns->rx_over_errors = 0;
1468         ns->rx_crc_errors = pstats->rx_fcs_errs;
1469         ns->rx_frame_errors = pstats->rx_symbol_errs;
1470         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1471         ns->rx_missed_errors = pstats->rx_cong_drops;
1472
1473         /* detailed tx_errors */
1474         ns->tx_aborted_errors = 0;
1475         ns->tx_carrier_errors = 0;
1476         ns->tx_fifo_errors = pstats->tx_underrun;
1477         ns->tx_heartbeat_errors = 0;
1478         ns->tx_window_errors = 0;
1479         return ns;
1480 }
1481
1482 static u32 get_msglevel(struct net_device *dev)
1483 {
1484         struct port_info *pi = netdev_priv(dev);
1485         struct adapter *adapter = pi->adapter;
1486
1487         return adapter->msg_enable;
1488 }
1489
1490 static void set_msglevel(struct net_device *dev, u32 val)
1491 {
1492         struct port_info *pi = netdev_priv(dev);
1493         struct adapter *adapter = pi->adapter;
1494
1495         adapter->msg_enable = val;
1496 }
1497
1498 static char stats_strings[][ETH_GSTRING_LEN] = {
1499         "TxOctetsOK         ",
1500         "TxFramesOK         ",
1501         "TxMulticastFramesOK",
1502         "TxBroadcastFramesOK",
1503         "TxPauseFrames      ",
1504         "TxUnderrun         ",
1505         "TxExtUnderrun      ",
1506
1507         "TxFrames64         ",
1508         "TxFrames65To127    ",
1509         "TxFrames128To255   ",
1510         "TxFrames256To511   ",
1511         "TxFrames512To1023  ",
1512         "TxFrames1024To1518 ",
1513         "TxFrames1519ToMax  ",
1514
1515         "RxOctetsOK         ",
1516         "RxFramesOK         ",
1517         "RxMulticastFramesOK",
1518         "RxBroadcastFramesOK",
1519         "RxPauseFrames      ",
1520         "RxFCSErrors        ",
1521         "RxSymbolErrors     ",
1522         "RxShortErrors      ",
1523         "RxJabberErrors     ",
1524         "RxLengthErrors     ",
1525         "RxFIFOoverflow     ",
1526
1527         "RxFrames64         ",
1528         "RxFrames65To127    ",
1529         "RxFrames128To255   ",
1530         "RxFrames256To511   ",
1531         "RxFrames512To1023  ",
1532         "RxFrames1024To1518 ",
1533         "RxFrames1519ToMax  ",
1534
1535         "PhyFIFOErrors      ",
1536         "TSO                ",
1537         "VLANextractions    ",
1538         "VLANinsertions     ",
1539         "TxCsumOffload      ",
1540         "RxCsumGood         ",
1541         "LroAggregated      ",
1542         "LroFlushed         ",
1543         "LroNoDesc          ",
1544         "RxDrops            ",
1545
1546         "CheckTXEnToggled   ",
1547         "CheckResets        ",
1548
1549         "LinkFaults         ",
1550 };
1551
1552 static int get_sset_count(struct net_device *dev, int sset)
1553 {
1554         switch (sset) {
1555         case ETH_SS_STATS:
1556                 return ARRAY_SIZE(stats_strings);
1557         default:
1558                 return -EOPNOTSUPP;
1559         }
1560 }
1561
1562 #define T3_REGMAP_SIZE (3 * 1024)
1563
1564 static int get_regs_len(struct net_device *dev)
1565 {
1566         return T3_REGMAP_SIZE;
1567 }
1568
1569 static int get_eeprom_len(struct net_device *dev)
1570 {
1571         return EEPROMSIZE;
1572 }
1573
1574 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1575 {
1576         struct port_info *pi = netdev_priv(dev);
1577         struct adapter *adapter = pi->adapter;
1578         u32 fw_vers = 0;
1579         u32 tp_vers = 0;
1580
1581         spin_lock(&adapter->stats_lock);
1582         t3_get_fw_version(adapter, &fw_vers);
1583         t3_get_tp_version(adapter, &tp_vers);
1584         spin_unlock(&adapter->stats_lock);
1585
1586         strcpy(info->driver, DRV_NAME);
1587         strcpy(info->version, DRV_VERSION);
1588         strcpy(info->bus_info, pci_name(adapter->pdev));
1589         if (!fw_vers)
1590                 strcpy(info->fw_version, "N/A");
1591         else {
1592                 snprintf(info->fw_version, sizeof(info->fw_version),
1593                          "%s %u.%u.%u TP %u.%u.%u",
1594                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1595                          G_FW_VERSION_MAJOR(fw_vers),
1596                          G_FW_VERSION_MINOR(fw_vers),
1597                          G_FW_VERSION_MICRO(fw_vers),
1598                          G_TP_VERSION_MAJOR(tp_vers),
1599                          G_TP_VERSION_MINOR(tp_vers),
1600                          G_TP_VERSION_MICRO(tp_vers));
1601         }
1602 }
1603
1604 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1605 {
1606         if (stringset == ETH_SS_STATS)
1607                 memcpy(data, stats_strings, sizeof(stats_strings));
1608 }
1609
1610 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1611                                             struct port_info *p, int idx)
1612 {
1613         int i;
1614         unsigned long tot = 0;
1615
1616         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1617                 tot += adapter->sge.qs[i].port_stats[idx];
1618         return tot;
1619 }
1620
1621 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1622                       u64 *data)
1623 {
1624         struct port_info *pi = netdev_priv(dev);
1625         struct adapter *adapter = pi->adapter;
1626         const struct mac_stats *s;
1627
1628         spin_lock(&adapter->stats_lock);
1629         s = t3_mac_update_stats(&pi->mac);
1630         spin_unlock(&adapter->stats_lock);
1631
1632         *data++ = s->tx_octets;
1633         *data++ = s->tx_frames;
1634         *data++ = s->tx_mcast_frames;
1635         *data++ = s->tx_bcast_frames;
1636         *data++ = s->tx_pause;
1637         *data++ = s->tx_underrun;
1638         *data++ = s->tx_fifo_urun;
1639
1640         *data++ = s->tx_frames_64;
1641         *data++ = s->tx_frames_65_127;
1642         *data++ = s->tx_frames_128_255;
1643         *data++ = s->tx_frames_256_511;
1644         *data++ = s->tx_frames_512_1023;
1645         *data++ = s->tx_frames_1024_1518;
1646         *data++ = s->tx_frames_1519_max;
1647
1648         *data++ = s->rx_octets;
1649         *data++ = s->rx_frames;
1650         *data++ = s->rx_mcast_frames;
1651         *data++ = s->rx_bcast_frames;
1652         *data++ = s->rx_pause;
1653         *data++ = s->rx_fcs_errs;
1654         *data++ = s->rx_symbol_errs;
1655         *data++ = s->rx_short;
1656         *data++ = s->rx_jabber;
1657         *data++ = s->rx_too_long;
1658         *data++ = s->rx_fifo_ovfl;
1659
1660         *data++ = s->rx_frames_64;
1661         *data++ = s->rx_frames_65_127;
1662         *data++ = s->rx_frames_128_255;
1663         *data++ = s->rx_frames_256_511;
1664         *data++ = s->rx_frames_512_1023;
1665         *data++ = s->rx_frames_1024_1518;
1666         *data++ = s->rx_frames_1519_max;
1667
1668         *data++ = pi->phy.fifo_errors;
1669
1670         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1671         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1672         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1673         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1674         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1675         *data++ = 0;
1676         *data++ = 0;
1677         *data++ = 0;
1678         *data++ = s->rx_cong_drops;
1679
1680         *data++ = s->num_toggled;
1681         *data++ = s->num_resets;
1682
1683         *data++ = s->link_faults;
1684 }
1685
1686 static inline void reg_block_dump(struct adapter *ap, void *buf,
1687                                   unsigned int start, unsigned int end)
1688 {
1689         u32 *p = buf + start;
1690
1691         for (; start <= end; start += sizeof(u32))
1692                 *p++ = t3_read_reg(ap, start);
1693 }
1694
1695 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1696                      void *buf)
1697 {
1698         struct port_info *pi = netdev_priv(dev);
1699         struct adapter *ap = pi->adapter;
1700
1701         /*
1702          * Version scheme:
1703          * bits 0..9: chip version
1704          * bits 10..15: chip revision
1705          * bit 31: set for PCIe cards
1706          */
1707         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1708
1709         /*
1710          * We skip the MAC statistics registers because they are clear-on-read.
1711          * Also reading multi-register stats would need to synchronize with the
1712          * periodic mac stats accumulation.  Hard to justify the complexity.
1713          */
1714         memset(buf, 0, T3_REGMAP_SIZE);
1715         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1716         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1717         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1718         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1719         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1720         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1721                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1722         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1723                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1724 }
1725
1726 static int restart_autoneg(struct net_device *dev)
1727 {
1728         struct port_info *p = netdev_priv(dev);
1729
1730         if (!netif_running(dev))
1731                 return -EAGAIN;
1732         if (p->link_config.autoneg != AUTONEG_ENABLE)
1733                 return -EINVAL;
1734         p->phy.ops->autoneg_restart(&p->phy);
1735         return 0;
1736 }
1737
1738 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1739 {
1740         struct port_info *pi = netdev_priv(dev);
1741         struct adapter *adapter = pi->adapter;
1742         int i;
1743
1744         if (data == 0)
1745                 data = 2;
1746
1747         for (i = 0; i < data * 2; i++) {
1748                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1749                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1750                 if (msleep_interruptible(500))
1751                         break;
1752         }
1753         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1754                          F_GPIO0_OUT_VAL);
1755         return 0;
1756 }
1757
1758 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1759 {
1760         struct port_info *p = netdev_priv(dev);
1761
1762         cmd->supported = p->link_config.supported;
1763         cmd->advertising = p->link_config.advertising;
1764
1765         if (netif_carrier_ok(dev)) {
1766                 cmd->speed = p->link_config.speed;
1767                 cmd->duplex = p->link_config.duplex;
1768         } else {
1769                 cmd->speed = -1;
1770                 cmd->duplex = -1;
1771         }
1772
1773         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1774         cmd->phy_address = p->phy.mdio.prtad;
1775         cmd->transceiver = XCVR_EXTERNAL;
1776         cmd->autoneg = p->link_config.autoneg;
1777         cmd->maxtxpkt = 0;
1778         cmd->maxrxpkt = 0;
1779         return 0;
1780 }
1781
1782 static int speed_duplex_to_caps(int speed, int duplex)
1783 {
1784         int cap = 0;
1785
1786         switch (speed) {
1787         case SPEED_10:
1788                 if (duplex == DUPLEX_FULL)
1789                         cap = SUPPORTED_10baseT_Full;
1790                 else
1791                         cap = SUPPORTED_10baseT_Half;
1792                 break;
1793         case SPEED_100:
1794                 if (duplex == DUPLEX_FULL)
1795                         cap = SUPPORTED_100baseT_Full;
1796                 else
1797                         cap = SUPPORTED_100baseT_Half;
1798                 break;
1799         case SPEED_1000:
1800                 if (duplex == DUPLEX_FULL)
1801                         cap = SUPPORTED_1000baseT_Full;
1802                 else
1803                         cap = SUPPORTED_1000baseT_Half;
1804                 break;
1805         case SPEED_10000:
1806                 if (duplex == DUPLEX_FULL)
1807                         cap = SUPPORTED_10000baseT_Full;
1808         }
1809         return cap;
1810 }
1811
1812 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1813                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1814                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1815                       ADVERTISED_10000baseT_Full)
1816
1817 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1818 {
1819         struct port_info *p = netdev_priv(dev);
1820         struct link_config *lc = &p->link_config;
1821
1822         if (!(lc->supported & SUPPORTED_Autoneg)) {
1823                 /*
1824                  * PHY offers a single speed/duplex.  See if that's what's
1825                  * being requested.
1826                  */
1827                 if (cmd->autoneg == AUTONEG_DISABLE) {
1828                         int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1829                         if (lc->supported & cap)
1830                                 return 0;
1831                 }
1832                 return -EINVAL;
1833         }
1834
1835         if (cmd->autoneg == AUTONEG_DISABLE) {
1836                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1837
1838                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1839                         return -EINVAL;
1840                 lc->requested_speed = cmd->speed;
1841                 lc->requested_duplex = cmd->duplex;
1842                 lc->advertising = 0;
1843         } else {
1844                 cmd->advertising &= ADVERTISED_MASK;
1845                 cmd->advertising &= lc->supported;
1846                 if (!cmd->advertising)
1847                         return -EINVAL;
1848                 lc->requested_speed = SPEED_INVALID;
1849                 lc->requested_duplex = DUPLEX_INVALID;
1850                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1851         }
1852         lc->autoneg = cmd->autoneg;
1853         if (netif_running(dev))
1854                 t3_link_start(&p->phy, &p->mac, lc);
1855         return 0;
1856 }
1857
1858 static void get_pauseparam(struct net_device *dev,
1859                            struct ethtool_pauseparam *epause)
1860 {
1861         struct port_info *p = netdev_priv(dev);
1862
1863         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1864         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1865         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1866 }
1867
1868 static int set_pauseparam(struct net_device *dev,
1869                           struct ethtool_pauseparam *epause)
1870 {
1871         struct port_info *p = netdev_priv(dev);
1872         struct link_config *lc = &p->link_config;
1873
1874         if (epause->autoneg == AUTONEG_DISABLE)
1875                 lc->requested_fc = 0;
1876         else if (lc->supported & SUPPORTED_Autoneg)
1877                 lc->requested_fc = PAUSE_AUTONEG;
1878         else
1879                 return -EINVAL;
1880
1881         if (epause->rx_pause)
1882                 lc->requested_fc |= PAUSE_RX;
1883         if (epause->tx_pause)
1884                 lc->requested_fc |= PAUSE_TX;
1885         if (lc->autoneg == AUTONEG_ENABLE) {
1886                 if (netif_running(dev))
1887                         t3_link_start(&p->phy, &p->mac, lc);
1888         } else {
1889                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1890                 if (netif_running(dev))
1891                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1892         }
1893         return 0;
1894 }
1895
1896 static u32 get_rx_csum(struct net_device *dev)
1897 {
1898         struct port_info *p = netdev_priv(dev);
1899
1900         return p->rx_offload & T3_RX_CSUM;
1901 }
1902
1903 static int set_rx_csum(struct net_device *dev, u32 data)
1904 {
1905         struct port_info *p = netdev_priv(dev);
1906
1907         if (data) {
1908                 p->rx_offload |= T3_RX_CSUM;
1909         } else {
1910                 int i;
1911
1912                 p->rx_offload &= ~(T3_RX_CSUM | T3_LRO);
1913                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
1914                         set_qset_lro(dev, i, 0);
1915         }
1916         return 0;
1917 }
1918
1919 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1920 {
1921         struct port_info *pi = netdev_priv(dev);
1922         struct adapter *adapter = pi->adapter;
1923         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1924
1925         e->rx_max_pending = MAX_RX_BUFFERS;
1926         e->rx_mini_max_pending = 0;
1927         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1928         e->tx_max_pending = MAX_TXQ_ENTRIES;
1929
1930         e->rx_pending = q->fl_size;
1931         e->rx_mini_pending = q->rspq_size;
1932         e->rx_jumbo_pending = q->jumbo_size;
1933         e->tx_pending = q->txq_size[0];
1934 }
1935
1936 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1937 {
1938         struct port_info *pi = netdev_priv(dev);
1939         struct adapter *adapter = pi->adapter;
1940         struct qset_params *q;
1941         int i;
1942
1943         if (e->rx_pending > MAX_RX_BUFFERS ||
1944             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1945             e->tx_pending > MAX_TXQ_ENTRIES ||
1946             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1947             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1948             e->rx_pending < MIN_FL_ENTRIES ||
1949             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1950             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1951                 return -EINVAL;
1952
1953         if (adapter->flags & FULL_INIT_DONE)
1954                 return -EBUSY;
1955
1956         q = &adapter->params.sge.qset[pi->first_qset];
1957         for (i = 0; i < pi->nqsets; ++i, ++q) {
1958                 q->rspq_size = e->rx_mini_pending;
1959                 q->fl_size = e->rx_pending;
1960                 q->jumbo_size = e->rx_jumbo_pending;
1961                 q->txq_size[0] = e->tx_pending;
1962                 q->txq_size[1] = e->tx_pending;
1963                 q->txq_size[2] = e->tx_pending;
1964         }
1965         return 0;
1966 }
1967
1968 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1969 {
1970         struct port_info *pi = netdev_priv(dev);
1971         struct adapter *adapter = pi->adapter;
1972         struct qset_params *qsp = &adapter->params.sge.qset[0];
1973         struct sge_qset *qs = &adapter->sge.qs[0];
1974
1975         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1976                 return -EINVAL;
1977
1978         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1979         t3_update_qset_coalesce(qs, qsp);
1980         return 0;
1981 }
1982
1983 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1984 {
1985         struct port_info *pi = netdev_priv(dev);
1986         struct adapter *adapter = pi->adapter;
1987         struct qset_params *q = adapter->params.sge.qset;
1988
1989         c->rx_coalesce_usecs = q->coalesce_usecs;
1990         return 0;
1991 }
1992
1993 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1994                       u8 * data)
1995 {
1996         struct port_info *pi = netdev_priv(dev);
1997         struct adapter *adapter = pi->adapter;
1998         int i, err = 0;
1999
2000         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2001         if (!buf)
2002                 return -ENOMEM;
2003
2004         e->magic = EEPROM_MAGIC;
2005         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2006                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2007
2008         if (!err)
2009                 memcpy(data, buf + e->offset, e->len);
2010         kfree(buf);
2011         return err;
2012 }
2013
2014 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2015                       u8 * data)
2016 {
2017         struct port_info *pi = netdev_priv(dev);
2018         struct adapter *adapter = pi->adapter;
2019         u32 aligned_offset, aligned_len;
2020         __le32 *p;
2021         u8 *buf;
2022         int err;
2023
2024         if (eeprom->magic != EEPROM_MAGIC)
2025                 return -EINVAL;
2026
2027         aligned_offset = eeprom->offset & ~3;
2028         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2029
2030         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2031                 buf = kmalloc(aligned_len, GFP_KERNEL);
2032                 if (!buf)
2033                         return -ENOMEM;
2034                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2035                 if (!err && aligned_len > 4)
2036                         err = t3_seeprom_read(adapter,
2037                                               aligned_offset + aligned_len - 4,
2038                                               (__le32 *) & buf[aligned_len - 4]);
2039                 if (err)
2040                         goto out;
2041                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2042         } else
2043                 buf = data;
2044
2045         err = t3_seeprom_wp(adapter, 0);
2046         if (err)
2047                 goto out;
2048
2049         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2050                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2051                 aligned_offset += 4;
2052         }
2053
2054         if (!err)
2055                 err = t3_seeprom_wp(adapter, 1);
2056 out:
2057         if (buf != data)
2058                 kfree(buf);
2059         return err;
2060 }
2061
2062 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2063 {
2064         wol->supported = 0;
2065         wol->wolopts = 0;
2066         memset(&wol->sopass, 0, sizeof(wol->sopass));
2067 }
2068
2069 static const struct ethtool_ops cxgb_ethtool_ops = {
2070         .get_settings = get_settings,
2071         .set_settings = set_settings,
2072         .get_drvinfo = get_drvinfo,
2073         .get_msglevel = get_msglevel,
2074         .set_msglevel = set_msglevel,
2075         .get_ringparam = get_sge_param,
2076         .set_ringparam = set_sge_param,
2077         .get_coalesce = get_coalesce,
2078         .set_coalesce = set_coalesce,
2079         .get_eeprom_len = get_eeprom_len,
2080         .get_eeprom = get_eeprom,
2081         .set_eeprom = set_eeprom,
2082         .get_pauseparam = get_pauseparam,
2083         .set_pauseparam = set_pauseparam,
2084         .get_rx_csum = get_rx_csum,
2085         .set_rx_csum = set_rx_csum,
2086         .set_tx_csum = ethtool_op_set_tx_csum,
2087         .set_sg = ethtool_op_set_sg,
2088         .get_link = ethtool_op_get_link,
2089         .get_strings = get_strings,
2090         .phys_id = cxgb3_phys_id,
2091         .nway_reset = restart_autoneg,
2092         .get_sset_count = get_sset_count,
2093         .get_ethtool_stats = get_stats,
2094         .get_regs_len = get_regs_len,
2095         .get_regs = get_regs,
2096         .get_wol = get_wol,
2097         .set_tso = ethtool_op_set_tso,
2098 };
2099
2100 static int in_range(int val, int lo, int hi)
2101 {
2102         return val < 0 || (val <= hi && val >= lo);
2103 }
2104
2105 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2106 {
2107         struct port_info *pi = netdev_priv(dev);
2108         struct adapter *adapter = pi->adapter;
2109         u32 cmd;
2110         int ret;
2111
2112         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2113                 return -EFAULT;
2114
2115         switch (cmd) {
2116         case CHELSIO_SET_QSET_PARAMS:{
2117                 int i;
2118                 struct qset_params *q;
2119                 struct ch_qset_params t;
2120                 int q1 = pi->first_qset;
2121                 int nqsets = pi->nqsets;
2122
2123                 if (!capable(CAP_NET_ADMIN))
2124                         return -EPERM;
2125                 if (copy_from_user(&t, useraddr, sizeof(t)))
2126                         return -EFAULT;
2127                 if (t.qset_idx >= SGE_QSETS)
2128                         return -EINVAL;
2129                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2130                     !in_range(t.cong_thres, 0, 255) ||
2131                     !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2132                               MAX_TXQ_ENTRIES) ||
2133                     !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2134                               MAX_TXQ_ENTRIES) ||
2135                     !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2136                               MAX_CTRL_TXQ_ENTRIES) ||
2137                     !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2138                               MAX_RX_BUFFERS) ||
2139                     !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2140                               MAX_RX_JUMBO_BUFFERS) ||
2141                     !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2142                               MAX_RSPQ_ENTRIES))
2143                         return -EINVAL;
2144
2145                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
2146                         for_each_port(adapter, i) {
2147                                 pi = adap2pinfo(adapter, i);
2148                                 if (t.qset_idx >= pi->first_qset &&
2149                                     t.qset_idx < pi->first_qset + pi->nqsets &&
2150                                     !(pi->rx_offload & T3_RX_CSUM))
2151                                         return -EINVAL;
2152                         }
2153
2154                 if ((adapter->flags & FULL_INIT_DONE) &&
2155                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2156                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2157                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2158                         t.polling >= 0 || t.cong_thres >= 0))
2159                         return -EBUSY;
2160
2161                 /* Allow setting of any available qset when offload enabled */
2162                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2163                         q1 = 0;
2164                         for_each_port(adapter, i) {
2165                                 pi = adap2pinfo(adapter, i);
2166                                 nqsets += pi->first_qset + pi->nqsets;
2167                         }
2168                 }
2169
2170                 if (t.qset_idx < q1)
2171                         return -EINVAL;
2172                 if (t.qset_idx > q1 + nqsets - 1)
2173                         return -EINVAL;
2174
2175                 q = &adapter->params.sge.qset[t.qset_idx];
2176
2177                 if (t.rspq_size >= 0)
2178                         q->rspq_size = t.rspq_size;
2179                 if (t.fl_size[0] >= 0)
2180                         q->fl_size = t.fl_size[0];
2181                 if (t.fl_size[1] >= 0)
2182                         q->jumbo_size = t.fl_size[1];
2183                 if (t.txq_size[0] >= 0)
2184                         q->txq_size[0] = t.txq_size[0];
2185                 if (t.txq_size[1] >= 0)
2186                         q->txq_size[1] = t.txq_size[1];
2187                 if (t.txq_size[2] >= 0)
2188                         q->txq_size[2] = t.txq_size[2];
2189                 if (t.cong_thres >= 0)
2190                         q->cong_thres = t.cong_thres;
2191                 if (t.intr_lat >= 0) {
2192                         struct sge_qset *qs =
2193                                 &adapter->sge.qs[t.qset_idx];
2194
2195                         q->coalesce_usecs = t.intr_lat;
2196                         t3_update_qset_coalesce(qs, q);
2197                 }
2198                 if (t.polling >= 0) {
2199                         if (adapter->flags & USING_MSIX)
2200                                 q->polling = t.polling;
2201                         else {
2202                                 /* No polling with INTx for T3A */
2203                                 if (adapter->params.rev == 0 &&
2204                                         !(adapter->flags & USING_MSI))
2205                                         t.polling = 0;
2206
2207                                 for (i = 0; i < SGE_QSETS; i++) {
2208                                         q = &adapter->params.sge.
2209                                                 qset[i];
2210                                         q->polling = t.polling;
2211                                 }
2212                         }
2213                 }
2214                 if (t.lro >= 0)
2215                         set_qset_lro(dev, t.qset_idx, t.lro);
2216
2217                 break;
2218         }
2219         case CHELSIO_GET_QSET_PARAMS:{
2220                 struct qset_params *q;
2221                 struct ch_qset_params t;
2222                 int q1 = pi->first_qset;
2223                 int nqsets = pi->nqsets;
2224                 int i;
2225
2226                 if (copy_from_user(&t, useraddr, sizeof(t)))
2227                         return -EFAULT;
2228
2229                 /* Display qsets for all ports when offload enabled */
2230                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2231                         q1 = 0;
2232                         for_each_port(adapter, i) {
2233                                 pi = adap2pinfo(adapter, i);
2234                                 nqsets = pi->first_qset + pi->nqsets;
2235                         }
2236                 }
2237
2238                 if (t.qset_idx >= nqsets)
2239                         return -EINVAL;
2240
2241                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2242                 t.rspq_size = q->rspq_size;
2243                 t.txq_size[0] = q->txq_size[0];
2244                 t.txq_size[1] = q->txq_size[1];
2245                 t.txq_size[2] = q->txq_size[2];
2246                 t.fl_size[0] = q->fl_size;
2247                 t.fl_size[1] = q->jumbo_size;
2248                 t.polling = q->polling;
2249                 t.lro = q->lro;
2250                 t.intr_lat = q->coalesce_usecs;
2251                 t.cong_thres = q->cong_thres;
2252                 t.qnum = q1;
2253
2254                 if (adapter->flags & USING_MSIX)
2255                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2256                 else
2257                         t.vector = adapter->pdev->irq;
2258
2259                 if (copy_to_user(useraddr, &t, sizeof(t)))
2260                         return -EFAULT;
2261                 break;
2262         }
2263         case CHELSIO_SET_QSET_NUM:{
2264                 struct ch_reg edata;
2265                 unsigned int i, first_qset = 0, other_qsets = 0;
2266
2267                 if (!capable(CAP_NET_ADMIN))
2268                         return -EPERM;
2269                 if (adapter->flags & FULL_INIT_DONE)
2270                         return -EBUSY;
2271                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2272                         return -EFAULT;
2273                 if (edata.val < 1 ||
2274                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2275                         return -EINVAL;
2276
2277                 for_each_port(adapter, i)
2278                         if (adapter->port[i] && adapter->port[i] != dev)
2279                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2280
2281                 if (edata.val + other_qsets > SGE_QSETS)
2282                         return -EINVAL;
2283
2284                 pi->nqsets = edata.val;
2285
2286                 for_each_port(adapter, i)
2287                         if (adapter->port[i]) {
2288                                 pi = adap2pinfo(adapter, i);
2289                                 pi->first_qset = first_qset;
2290                                 first_qset += pi->nqsets;
2291                         }
2292                 break;
2293         }
2294         case CHELSIO_GET_QSET_NUM:{
2295                 struct ch_reg edata;
2296
2297                 edata.cmd = CHELSIO_GET_QSET_NUM;
2298                 edata.val = pi->nqsets;
2299                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2300                         return -EFAULT;
2301                 break;
2302         }
2303         case CHELSIO_LOAD_FW:{
2304                 u8 *fw_data;
2305                 struct ch_mem_range t;
2306
2307                 if (!capable(CAP_SYS_RAWIO))
2308                         return -EPERM;
2309                 if (copy_from_user(&t, useraddr, sizeof(t)))
2310                         return -EFAULT;
2311                 /* Check t.len sanity ? */
2312                 fw_data = kmalloc(t.len, GFP_KERNEL);
2313                 if (!fw_data)
2314                         return -ENOMEM;
2315
2316                 if (copy_from_user
2317                         (fw_data, useraddr + sizeof(t), t.len)) {
2318                         kfree(fw_data);
2319                         return -EFAULT;
2320                 }
2321
2322                 ret = t3_load_fw(adapter, fw_data, t.len);
2323                 kfree(fw_data);
2324                 if (ret)
2325                         return ret;
2326                 break;
2327         }
2328         case CHELSIO_SETMTUTAB:{
2329                 struct ch_mtus m;
2330                 int i;
2331
2332                 if (!is_offload(adapter))
2333                         return -EOPNOTSUPP;
2334                 if (!capable(CAP_NET_ADMIN))
2335                         return -EPERM;
2336                 if (offload_running(adapter))
2337                         return -EBUSY;
2338                 if (copy_from_user(&m, useraddr, sizeof(m)))
2339                         return -EFAULT;
2340                 if (m.nmtus != NMTUS)
2341                         return -EINVAL;
2342                 if (m.mtus[0] < 81)     /* accommodate SACK */
2343                         return -EINVAL;
2344
2345                 /* MTUs must be in ascending order */
2346                 for (i = 1; i < NMTUS; ++i)
2347                         if (m.mtus[i] < m.mtus[i - 1])
2348                                 return -EINVAL;
2349
2350                 memcpy(adapter->params.mtus, m.mtus,
2351                         sizeof(adapter->params.mtus));
2352                 break;
2353         }
2354         case CHELSIO_GET_PM:{
2355                 struct tp_params *p = &adapter->params.tp;
2356                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2357
2358                 if (!is_offload(adapter))
2359                         return -EOPNOTSUPP;
2360                 m.tx_pg_sz = p->tx_pg_size;
2361                 m.tx_num_pg = p->tx_num_pgs;
2362                 m.rx_pg_sz = p->rx_pg_size;
2363                 m.rx_num_pg = p->rx_num_pgs;
2364                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2365                 if (copy_to_user(useraddr, &m, sizeof(m)))
2366                         return -EFAULT;
2367                 break;
2368         }
2369         case CHELSIO_SET_PM:{
2370                 struct ch_pm m;
2371                 struct tp_params *p = &adapter->params.tp;
2372
2373                 if (!is_offload(adapter))
2374                         return -EOPNOTSUPP;
2375                 if (!capable(CAP_NET_ADMIN))
2376                         return -EPERM;
2377                 if (adapter->flags & FULL_INIT_DONE)
2378                         return -EBUSY;
2379                 if (copy_from_user(&m, useraddr, sizeof(m)))
2380                         return -EFAULT;
2381                 if (!is_power_of_2(m.rx_pg_sz) ||
2382                         !is_power_of_2(m.tx_pg_sz))
2383                         return -EINVAL; /* not power of 2 */
2384                 if (!(m.rx_pg_sz & 0x14000))
2385                         return -EINVAL; /* not 16KB or 64KB */
2386                 if (!(m.tx_pg_sz & 0x1554000))
2387                         return -EINVAL;
2388                 if (m.tx_num_pg == -1)
2389                         m.tx_num_pg = p->tx_num_pgs;
2390                 if (m.rx_num_pg == -1)
2391                         m.rx_num_pg = p->rx_num_pgs;
2392                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2393                         return -EINVAL;
2394                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2395                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2396                         return -EINVAL;
2397                 p->rx_pg_size = m.rx_pg_sz;
2398                 p->tx_pg_size = m.tx_pg_sz;
2399                 p->rx_num_pgs = m.rx_num_pg;
2400                 p->tx_num_pgs = m.tx_num_pg;
2401                 break;
2402         }
2403         case CHELSIO_GET_MEM:{
2404                 struct ch_mem_range t;
2405                 struct mc7 *mem;
2406                 u64 buf[32];
2407
2408                 if (!is_offload(adapter))
2409                         return -EOPNOTSUPP;
2410                 if (!(adapter->flags & FULL_INIT_DONE))
2411                         return -EIO;    /* need the memory controllers */
2412                 if (copy_from_user(&t, useraddr, sizeof(t)))
2413                         return -EFAULT;
2414                 if ((t.addr & 7) || (t.len & 7))
2415                         return -EINVAL;
2416                 if (t.mem_id == MEM_CM)
2417                         mem = &adapter->cm;
2418                 else if (t.mem_id == MEM_PMRX)
2419                         mem = &adapter->pmrx;
2420                 else if (t.mem_id == MEM_PMTX)
2421                         mem = &adapter->pmtx;
2422                 else
2423                         return -EINVAL;
2424
2425                 /*
2426                  * Version scheme:
2427                  * bits 0..9: chip version
2428                  * bits 10..15: chip revision
2429                  */
2430                 t.version = 3 | (adapter->params.rev << 10);
2431                 if (copy_to_user(useraddr, &t, sizeof(t)))
2432                         return -EFAULT;
2433
2434                 /*
2435                  * Read 256 bytes at a time as len can be large and we don't
2436                  * want to use huge intermediate buffers.
2437                  */
2438                 useraddr += sizeof(t);  /* advance to start of buffer */
2439                 while (t.len) {
2440                         unsigned int chunk =
2441                                 min_t(unsigned int, t.len, sizeof(buf));
2442
2443                         ret =
2444                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2445                                                 buf);
2446                         if (ret)
2447                                 return ret;
2448                         if (copy_to_user(useraddr, buf, chunk))
2449                                 return -EFAULT;
2450                         useraddr += chunk;
2451                         t.addr += chunk;
2452                         t.len -= chunk;
2453                 }
2454                 break;
2455         }
2456         case CHELSIO_SET_TRACE_FILTER:{
2457                 struct ch_trace t;
2458                 const struct trace_params *tp;
2459
2460                 if (!capable(CAP_NET_ADMIN))
2461                         return -EPERM;
2462                 if (!offload_running(adapter))
2463                         return -EAGAIN;
2464                 if (copy_from_user(&t, useraddr, sizeof(t)))
2465                         return -EFAULT;
2466
2467                 tp = (const struct trace_params *)&t.sip;
2468                 if (t.config_tx)
2469                         t3_config_trace_filter(adapter, tp, 0,
2470                                                 t.invert_match,
2471                                                 t.trace_tx);
2472                 if (t.config_rx)
2473                         t3_config_trace_filter(adapter, tp, 1,
2474                                                 t.invert_match,
2475                                                 t.trace_rx);
2476                 break;
2477         }
2478         default:
2479                 return -EOPNOTSUPP;
2480         }
2481         return 0;
2482 }
2483
2484 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2485 {
2486         struct mii_ioctl_data *data = if_mii(req);
2487         struct port_info *pi = netdev_priv(dev);
2488         struct adapter *adapter = pi->adapter;
2489
2490         switch (cmd) {
2491         case SIOCGMIIREG:
2492         case SIOCSMIIREG:
2493                 /* Convert phy_id from older PRTAD/DEVAD format */
2494                 if (is_10G(adapter) &&
2495                     !mdio_phy_id_is_c45(data->phy_id) &&
2496                     (data->phy_id & 0x1f00) &&
2497                     !(data->phy_id & 0xe0e0))
2498                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2499                                                        data->phy_id & 0x1f);
2500                 /* FALLTHRU */
2501         case SIOCGMIIPHY:
2502                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2503         case SIOCCHIOCTL:
2504                 return cxgb_extension_ioctl(dev, req->ifr_data);
2505         default:
2506                 return -EOPNOTSUPP;
2507         }
2508 }
2509
2510 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2511 {
2512         struct port_info *pi = netdev_priv(dev);
2513         struct adapter *adapter = pi->adapter;
2514         int ret;
2515
2516         if (new_mtu < 81)       /* accommodate SACK */
2517                 return -EINVAL;
2518         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2519                 return ret;
2520         dev->mtu = new_mtu;
2521         init_port_mtus(adapter);
2522         if (adapter->params.rev == 0 && offload_running(adapter))
2523                 t3_load_mtus(adapter, adapter->params.mtus,
2524                              adapter->params.a_wnd, adapter->params.b_wnd,
2525                              adapter->port[0]->mtu);
2526         return 0;
2527 }
2528
2529 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2530 {
2531         struct port_info *pi = netdev_priv(dev);
2532         struct adapter *adapter = pi->adapter;
2533         struct sockaddr *addr = p;
2534
2535         if (!is_valid_ether_addr(addr->sa_data))
2536                 return -EINVAL;
2537
2538         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2539         t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2540         if (offload_running(adapter))
2541                 write_smt_entry(adapter, pi->port_id);
2542         return 0;
2543 }
2544
2545 /**
2546  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2547  * @adap: the adapter
2548  * @p: the port
2549  *
2550  * Ensures that current Rx processing on any of the queues associated with
2551  * the given port completes before returning.  We do this by acquiring and
2552  * releasing the locks of the response queues associated with the port.
2553  */
2554 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2555 {
2556         int i;
2557
2558         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2559                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2560
2561                 spin_lock_irq(&q->lock);
2562                 spin_unlock_irq(&q->lock);
2563         }
2564 }
2565
2566 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2567 {
2568         struct port_info *pi = netdev_priv(dev);
2569         struct adapter *adapter = pi->adapter;
2570
2571         pi->vlan_grp = grp;
2572         if (adapter->params.rev > 0)
2573                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2574         else {
2575                 /* single control for all ports */
2576                 unsigned int i, have_vlans = 0;
2577                 for_each_port(adapter, i)
2578                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2579
2580                 t3_set_vlan_accel(adapter, 1, have_vlans);
2581         }
2582         t3_synchronize_rx(adapter, pi);
2583 }
2584
2585 #ifdef CONFIG_NET_POLL_CONTROLLER
2586 static void cxgb_netpoll(struct net_device *dev)
2587 {
2588         struct port_info *pi = netdev_priv(dev);
2589         struct adapter *adapter = pi->adapter;
2590         int qidx;
2591
2592         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2593                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2594                 void *source;
2595
2596                 if (adapter->flags & USING_MSIX)
2597                         source = qs;
2598                 else
2599                         source = adapter;
2600
2601                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2602         }
2603 }
2604 #endif
2605
2606 /*
2607  * Periodic accumulation of MAC statistics.
2608  */
2609 static void mac_stats_update(struct adapter *adapter)
2610 {
2611         int i;
2612
2613         for_each_port(adapter, i) {
2614                 struct net_device *dev = adapter->port[i];
2615                 struct port_info *p = netdev_priv(dev);
2616
2617                 if (netif_running(dev)) {
2618                         spin_lock(&adapter->stats_lock);
2619                         t3_mac_update_stats(&p->mac);
2620                         spin_unlock(&adapter->stats_lock);
2621                 }
2622         }
2623 }
2624
2625 static void check_link_status(struct adapter *adapter)
2626 {
2627         int i;
2628
2629         for_each_port(adapter, i) {
2630                 struct net_device *dev = adapter->port[i];
2631                 struct port_info *p = netdev_priv(dev);
2632                 int link_fault;
2633
2634                 spin_lock_irq(&adapter->work_lock);
2635                 link_fault = p->link_fault;
2636                 spin_unlock_irq(&adapter->work_lock);
2637
2638                 if (link_fault) {
2639                         t3_link_fault(adapter, i);
2640                         continue;
2641                 }
2642
2643                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2644                         t3_xgm_intr_disable(adapter, i);
2645                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2646
2647                         t3_link_changed(adapter, i);
2648                         t3_xgm_intr_enable(adapter, i);
2649                 }
2650         }
2651 }
2652
2653 static void check_t3b2_mac(struct adapter *adapter)
2654 {
2655         int i;
2656
2657         if (!rtnl_trylock())    /* synchronize with ifdown */
2658                 return;
2659
2660         for_each_port(adapter, i) {
2661                 struct net_device *dev = adapter->port[i];
2662                 struct port_info *p = netdev_priv(dev);
2663                 int status;
2664
2665                 if (!netif_running(dev))
2666                         continue;
2667
2668                 status = 0;
2669                 if (netif_running(dev) && netif_carrier_ok(dev))
2670                         status = t3b2_mac_watchdog_task(&p->mac);
2671                 if (status == 1)
2672                         p->mac.stats.num_toggled++;
2673                 else if (status == 2) {
2674                         struct cmac *mac = &p->mac;
2675
2676                         t3_mac_set_mtu(mac, dev->mtu);
2677                         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2678                         cxgb_set_rxmode(dev);
2679                         t3_link_start(&p->phy, mac, &p->link_config);
2680                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2681                         t3_port_intr_enable(adapter, p->port_id);
2682                         p->mac.stats.num_resets++;
2683                 }
2684         }
2685         rtnl_unlock();
2686 }
2687
2688
2689 static void t3_adap_check_task(struct work_struct *work)
2690 {
2691         struct adapter *adapter = container_of(work, struct adapter,
2692                                                adap_check_task.work);
2693         const struct adapter_params *p = &adapter->params;
2694         int port;
2695         unsigned int v, status, reset;
2696
2697         adapter->check_task_cnt++;
2698
2699         check_link_status(adapter);
2700
2701         /* Accumulate MAC stats if needed */
2702         if (!p->linkpoll_period ||
2703             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2704             p->stats_update_period) {
2705                 mac_stats_update(adapter);
2706                 adapter->check_task_cnt = 0;
2707         }
2708
2709         if (p->rev == T3_REV_B2)
2710                 check_t3b2_mac(adapter);
2711
2712         /*
2713          * Scan the XGMAC's to check for various conditions which we want to
2714          * monitor in a periodic polling manner rather than via an interrupt
2715          * condition.  This is used for conditions which would otherwise flood
2716          * the system with interrupts and we only really need to know that the
2717          * conditions are "happening" ...  For each condition we count the
2718          * detection of the condition and reset it for the next polling loop.
2719          */
2720         for_each_port(adapter, port) {
2721                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2722                 u32 cause;
2723
2724                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2725                 reset = 0;
2726                 if (cause & F_RXFIFO_OVERFLOW) {
2727                         mac->stats.rx_fifo_ovfl++;
2728                         reset |= F_RXFIFO_OVERFLOW;
2729                 }
2730
2731                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2732         }
2733
2734         /*
2735          * We do the same as above for FL_EMPTY interrupts.
2736          */
2737         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2738         reset = 0;
2739
2740         if (status & F_FLEMPTY) {
2741                 struct sge_qset *qs = &adapter->sge.qs[0];
2742                 int i = 0;
2743
2744                 reset |= F_FLEMPTY;
2745
2746                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2747                     0xffff;
2748
2749                 while (v) {
2750                         qs->fl[i].empty += (v & 1);
2751                         if (i)
2752                                 qs++;
2753                         i ^= 1;
2754                         v >>= 1;
2755                 }
2756         }
2757
2758         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2759
2760         /* Schedule the next check update if any port is active. */
2761         spin_lock_irq(&adapter->work_lock);
2762         if (adapter->open_device_map & PORT_MASK)
2763                 schedule_chk_task(adapter);
2764         spin_unlock_irq(&adapter->work_lock);
2765 }
2766
2767 static void db_full_task(struct work_struct *work)
2768 {
2769         struct adapter *adapter = container_of(work, struct adapter,
2770                                                db_full_task);
2771
2772         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2773 }
2774
2775 static void db_empty_task(struct work_struct *work)
2776 {
2777         struct adapter *adapter = container_of(work, struct adapter,
2778                                                db_empty_task);
2779
2780         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2781 }
2782
2783 static void db_drop_task(struct work_struct *work)
2784 {
2785         struct adapter *adapter = container_of(work, struct adapter,
2786                                                db_drop_task);
2787         unsigned long delay = 1000;
2788         unsigned short r;
2789
2790         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2791
2792         /*
2793          * Sleep a while before ringing the driver qset dbs.
2794          * The delay is between 1000-2023 usecs.
2795          */
2796         get_random_bytes(&r, 2);
2797         delay += r & 1023;
2798         set_current_state(TASK_UNINTERRUPTIBLE);
2799         schedule_timeout(usecs_to_jiffies(delay));
2800         ring_dbs(adapter);
2801 }
2802
2803 /*
2804  * Processes external (PHY) interrupts in process context.
2805  */
2806 static void ext_intr_task(struct work_struct *work)
2807 {
2808         struct adapter *adapter = container_of(work, struct adapter,
2809                                                ext_intr_handler_task);
2810         int i;
2811
2812         /* Disable link fault interrupts */
2813         for_each_port(adapter, i) {
2814                 struct net_device *dev = adapter->port[i];
2815                 struct port_info *p = netdev_priv(dev);
2816
2817                 t3_xgm_intr_disable(adapter, i);
2818                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2819         }
2820
2821         /* Re-enable link fault interrupts */
2822         t3_phy_intr_handler(adapter);
2823
2824         for_each_port(adapter, i)
2825                 t3_xgm_intr_enable(adapter, i);
2826
2827         /* Now reenable external interrupts */
2828         spin_lock_irq(&adapter->work_lock);
2829         if (adapter->slow_intr_mask) {
2830                 adapter->slow_intr_mask |= F_T3DBG;
2831                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2832                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2833                              adapter->slow_intr_mask);
2834         }
2835         spin_unlock_irq(&adapter->work_lock);
2836 }
2837
2838 /*
2839  * Interrupt-context handler for external (PHY) interrupts.
2840  */
2841 void t3_os_ext_intr_handler(struct adapter *adapter)
2842 {
2843         /*
2844          * Schedule a task to handle external interrupts as they may be slow
2845          * and we use a mutex to protect MDIO registers.  We disable PHY
2846          * interrupts in the meantime and let the task reenable them when
2847          * it's done.
2848          */
2849         spin_lock(&adapter->work_lock);
2850         if (adapter->slow_intr_mask) {
2851                 adapter->slow_intr_mask &= ~F_T3DBG;
2852                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2853                              adapter->slow_intr_mask);
2854                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2855         }
2856         spin_unlock(&adapter->work_lock);
2857 }
2858
2859 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2860 {
2861         struct net_device *netdev = adapter->port[port_id];
2862         struct port_info *pi = netdev_priv(netdev);
2863
2864         spin_lock(&adapter->work_lock);
2865         pi->link_fault = 1;
2866         spin_unlock(&adapter->work_lock);
2867 }
2868
2869 static int t3_adapter_error(struct adapter *adapter, int reset)
2870 {
2871         int i, ret = 0;
2872
2873         if (is_offload(adapter) &&
2874             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2875                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2876                 offload_close(&adapter->tdev);
2877         }
2878
2879         /* Stop all ports */
2880         for_each_port(adapter, i) {
2881                 struct net_device *netdev = adapter->port[i];
2882
2883                 if (netif_running(netdev))
2884                         cxgb_close(netdev);
2885         }
2886
2887         /* Stop SGE timers */
2888         t3_stop_sge_timers(adapter);
2889
2890         adapter->flags &= ~FULL_INIT_DONE;
2891
2892         if (reset)
2893                 ret = t3_reset_adapter(adapter);
2894
2895         pci_disable_device(adapter->pdev);
2896
2897         return ret;
2898 }
2899
2900 static int t3_reenable_adapter(struct adapter *adapter)
2901 {
2902         if (pci_enable_device(adapter->pdev)) {
2903                 dev_err(&adapter->pdev->dev,
2904                         "Cannot re-enable PCI device after reset.\n");
2905                 goto err;
2906         }
2907         pci_set_master(adapter->pdev);
2908         pci_restore_state(adapter->pdev);
2909         pci_save_state(adapter->pdev);
2910
2911         /* Free sge resources */
2912         t3_free_sge_resources(adapter);
2913
2914         if (t3_replay_prep_adapter(adapter))
2915                 goto err;
2916
2917         return 0;
2918 err:
2919         return -1;
2920 }
2921
2922 static void t3_resume_ports(struct adapter *adapter)
2923 {
2924         int i;
2925
2926         /* Restart the ports */
2927         for_each_port(adapter, i) {
2928                 struct net_device *netdev = adapter->port[i];
2929
2930                 if (netif_running(netdev)) {
2931                         if (cxgb_open(netdev)) {
2932                                 dev_err(&adapter->pdev->dev,
2933                                         "can't bring device back up"
2934                                         " after reset\n");
2935                                 continue;
2936                         }
2937                 }
2938         }
2939
2940         if (is_offload(adapter) && !ofld_disable)
2941                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2942 }
2943
2944 /*
2945  * processes a fatal error.
2946  * Bring the ports down, reset the chip, bring the ports back up.
2947  */
2948 static void fatal_error_task(struct work_struct *work)
2949 {
2950         struct adapter *adapter = container_of(work, struct adapter,
2951                                                fatal_error_handler_task);
2952         int err = 0;
2953
2954         rtnl_lock();
2955         err = t3_adapter_error(adapter, 1);
2956         if (!err)
2957                 err = t3_reenable_adapter(adapter);
2958         if (!err)
2959                 t3_resume_ports(adapter);
2960
2961         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2962         rtnl_unlock();
2963 }
2964
2965 void t3_fatal_err(struct adapter *adapter)
2966 {
2967         unsigned int fw_status[4];
2968
2969         if (adapter->flags & FULL_INIT_DONE) {
2970                 t3_sge_stop(adapter);
2971                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2972                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2973                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2974                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2975
2976                 spin_lock(&adapter->work_lock);
2977                 t3_intr_disable(adapter);
2978                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2979                 spin_unlock(&adapter->work_lock);
2980         }
2981         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2982         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2983                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2984                          fw_status[0], fw_status[1],
2985                          fw_status[2], fw_status[3]);
2986 }
2987
2988 /**
2989  * t3_io_error_detected - called when PCI error is detected
2990  * @pdev: Pointer to PCI device
2991  * @state: The current pci connection state
2992  *
2993  * This function is called after a PCI bus error affecting
2994  * this device has been detected.
2995  */
2996 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2997                                              pci_channel_state_t state)
2998 {
2999         struct adapter *adapter = pci_get_drvdata(pdev);
3000         int ret;
3001
3002         if (state == pci_channel_io_perm_failure)
3003                 return PCI_ERS_RESULT_DISCONNECT;
3004
3005         ret = t3_adapter_error(adapter, 0);
3006
3007         /* Request a slot reset. */
3008         return PCI_ERS_RESULT_NEED_RESET;
3009 }
3010
3011 /**
3012  * t3_io_slot_reset - called after the pci bus has been reset.
3013  * @pdev: Pointer to PCI device
3014  *
3015  * Restart the card from scratch, as if from a cold-boot.
3016  */
3017 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3018 {
3019         struct adapter *adapter = pci_get_drvdata(pdev);
3020
3021         if (!t3_reenable_adapter(adapter))
3022                 return PCI_ERS_RESULT_RECOVERED;
3023
3024         return PCI_ERS_RESULT_DISCONNECT;
3025 }
3026
3027 /**
3028  * t3_io_resume - called when traffic can start flowing again.
3029  * @pdev: Pointer to PCI device
3030  *
3031  * This callback is called when the error recovery driver tells us that
3032  * its OK to resume normal operation.
3033  */
3034 static void t3_io_resume(struct pci_dev *pdev)
3035 {
3036         struct adapter *adapter = pci_get_drvdata(pdev);
3037
3038         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3039                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
3040
3041         t3_resume_ports(adapter);
3042 }
3043
3044 static struct pci_error_handlers t3_err_handler = {
3045         .error_detected = t3_io_error_detected,
3046         .slot_reset = t3_io_slot_reset,
3047         .resume = t3_io_resume,
3048 };
3049
3050 /*
3051  * Set the number of qsets based on the number of CPUs and the number of ports,
3052  * not to exceed the number of available qsets, assuming there are enough qsets
3053  * per port in HW.
3054  */
3055 static void set_nqsets(struct adapter *adap)
3056 {
3057         int i, j = 0;
3058         int num_cpus = num_online_cpus();
3059         int hwports = adap->params.nports;
3060         int nqsets = adap->msix_nvectors - 1;
3061
3062         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3063                 if (hwports == 2 &&
3064                     (hwports * nqsets > SGE_QSETS ||
3065                      num_cpus >= nqsets / hwports))
3066                         nqsets /= hwports;
3067                 if (nqsets > num_cpus)
3068                         nqsets = num_cpus;
3069                 if (nqsets < 1 || hwports == 4)
3070                         nqsets = 1;
3071         } else
3072                 nqsets = 1;
3073
3074         for_each_port(adap, i) {
3075                 struct port_info *pi = adap2pinfo(adap, i);
3076
3077                 pi->first_qset = j;
3078                 pi->nqsets = nqsets;
3079                 j = pi->first_qset + nqsets;
3080
3081                 dev_info(&adap->pdev->dev,
3082                          "Port %d using %d queue sets.\n", i, nqsets);
3083         }
3084 }
3085
3086 static int __devinit cxgb_enable_msix(struct adapter *adap)
3087 {
3088         struct msix_entry entries[SGE_QSETS + 1];
3089         int vectors;
3090         int i, err;
3091
3092         vectors = ARRAY_SIZE(entries);
3093         for (i = 0; i < vectors; ++i)
3094                 entries[i].entry = i;
3095
3096         while ((err = pci_enable_msix(adap->pdev, entries, vectors)) > 0)
3097                 vectors = err;
3098
3099         if (err < 0)
3100                 pci_disable_msix(adap->pdev);
3101
3102         if (!err && vectors < (adap->params.nports + 1)) {
3103                 pci_disable_msix(adap->pdev);
3104                 err = -1;
3105         }
3106
3107         if (!err) {
3108                 for (i = 0; i < vectors; ++i)
3109                         adap->msix_info[i].vec = entries[i].vector;
3110                 adap->msix_nvectors = vectors;
3111         }
3112
3113         return err;
3114 }
3115
3116 static void __devinit print_port_info(struct adapter *adap,
3117                                       const struct adapter_info *ai)
3118 {
3119         static const char *pci_variant[] = {
3120                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3121         };
3122
3123         int i;
3124         char buf[80];
3125
3126         if (is_pcie(adap))
3127                 snprintf(buf, sizeof(buf), "%s x%d",
3128                          pci_variant[adap->params.pci.variant],
3129                          adap->params.pci.width);
3130         else
3131                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3132                          pci_variant[adap->params.pci.variant],
3133                          adap->params.pci.speed, adap->params.pci.width);
3134
3135         for_each_port(adap, i) {
3136                 struct net_device *dev = adap->port[i];
3137                 const struct port_info *pi = netdev_priv(dev);
3138
3139                 if (!test_bit(i, &adap->registered_device_map))
3140                         continue;
3141                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
3142                        dev->name, ai->desc, pi->phy.desc,
3143                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
3144                        (adap->flags & USING_MSIX) ? " MSI-X" :
3145                        (adap->flags & USING_MSI) ? " MSI" : "");
3146                 if (adap->name == dev->name && adap->params.vpd.mclk)
3147                         printk(KERN_INFO
3148                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3149                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3150                                t3_mc7_size(&adap->pmtx) >> 20,
3151                                t3_mc7_size(&adap->pmrx) >> 20,
3152                                adap->params.vpd.sn);
3153         }
3154 }
3155
3156 static const struct net_device_ops cxgb_netdev_ops = {
3157         .ndo_open               = cxgb_open,
3158         .ndo_stop               = cxgb_close,
3159         .ndo_start_xmit         = t3_eth_xmit,
3160         .ndo_get_stats          = cxgb_get_stats,
3161         .ndo_validate_addr      = eth_validate_addr,
3162         .ndo_set_multicast_list = cxgb_set_rxmode,
3163         .ndo_do_ioctl           = cxgb_ioctl,
3164         .ndo_change_mtu         = cxgb_change_mtu,
3165         .ndo_set_mac_address    = cxgb_set_mac_addr,
3166         .ndo_vlan_rx_register   = vlan_rx_register,
3167 #ifdef CONFIG_NET_POLL_CONTROLLER
3168         .ndo_poll_controller    = cxgb_netpoll,
3169 #endif
3170 };
3171
3172 static void __devinit cxgb3_init_iscsi_mac(struct net_device *dev)
3173 {
3174         struct port_info *pi = netdev_priv(dev);
3175
3176         memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3177         pi->iscsic.mac_addr[3] |= 0x80;
3178 }
3179
3180 static int __devinit init_one(struct pci_dev *pdev,
3181                               const struct pci_device_id *ent)
3182 {
3183         static int version_printed;
3184
3185         int i, err, pci_using_dac = 0;
3186         resource_size_t mmio_start, mmio_len;
3187         const struct adapter_info *ai;
3188         struct adapter *adapter = NULL;
3189         struct port_info *pi;
3190
3191         if (!version_printed) {
3192                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3193                 ++version_printed;
3194         }
3195
3196         if (!cxgb3_wq) {
3197                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3198                 if (!cxgb3_wq) {
3199                         printk(KERN_ERR DRV_NAME
3200                                ": cannot initialize work queue\n");
3201                         return -ENOMEM;
3202                 }
3203         }
3204
3205         err = pci_request_regions(pdev, DRV_NAME);
3206         if (err) {
3207                 /* Just info, some other driver may have claimed the device. */
3208                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3209                 return err;
3210         }
3211
3212         err = pci_enable_device(pdev);
3213         if (err) {
3214                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3215                 goto out_release_regions;
3216         }
3217
3218         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3219                 pci_using_dac = 1;
3220                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3221                 if (err) {
3222                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3223                                "coherent allocations\n");
3224                         goto out_disable_device;
3225                 }
3226         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3227                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3228                 goto out_disable_device;
3229         }
3230
3231         pci_set_master(pdev);
3232         pci_save_state(pdev);
3233
3234         mmio_start = pci_resource_start(pdev, 0);
3235         mmio_len = pci_resource_len(pdev, 0);
3236         ai = t3_get_adapter_info(ent->driver_data);
3237
3238         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3239         if (!adapter) {
3240                 err = -ENOMEM;
3241                 goto out_disable_device;
3242         }
3243
3244         adapter->nofail_skb =
3245                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3246         if (!adapter->nofail_skb) {
3247                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3248                 err = -ENOMEM;
3249                 goto out_free_adapter;
3250         }
3251
3252         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3253         if (!adapter->regs) {
3254                 dev_err(&pdev->dev, "cannot map device registers\n");
3255                 err = -ENOMEM;
3256                 goto out_free_adapter;
3257         }
3258
3259         adapter->pdev = pdev;
3260         adapter->name = pci_name(pdev);
3261         adapter->msg_enable = dflt_msg_enable;
3262         adapter->mmio_len = mmio_len;
3263
3264         mutex_init(&adapter->mdio_lock);
3265         spin_lock_init(&adapter->work_lock);
3266         spin_lock_init(&adapter->stats_lock);
3267
3268         INIT_LIST_HEAD(&adapter->adapter_list);
3269         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3270         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3271
3272         INIT_WORK(&adapter->db_full_task, db_full_task);
3273         INIT_WORK(&adapter->db_empty_task, db_empty_task);
3274         INIT_WORK(&adapter->db_drop_task, db_drop_task);
3275
3276         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3277
3278         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3279                 struct net_device *netdev;
3280
3281                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3282                 if (!netdev) {
3283                         err = -ENOMEM;
3284                         goto out_free_dev;
3285                 }
3286
3287                 SET_NETDEV_DEV(netdev, &pdev->dev);
3288
3289                 adapter->port[i] = netdev;
3290                 pi = netdev_priv(netdev);
3291                 pi->adapter = adapter;
3292                 pi->rx_offload = T3_RX_CSUM | T3_LRO;
3293                 pi->port_id = i;
3294                 netif_carrier_off(netdev);
3295                 netif_tx_stop_all_queues(netdev);
3296                 netdev->irq = pdev->irq;
3297                 netdev->mem_start = mmio_start;
3298                 netdev->mem_end = mmio_start + mmio_len - 1;
3299                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
3300                 netdev->features |= NETIF_F_GRO;
3301                 if (pci_using_dac)
3302                         netdev->features |= NETIF_F_HIGHDMA;
3303
3304                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3305                 netdev->netdev_ops = &cxgb_netdev_ops;
3306                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3307         }
3308
3309         pci_set_drvdata(pdev, adapter);
3310         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3311                 err = -ENODEV;
3312                 goto out_free_dev;
3313         }
3314
3315         /*
3316          * The card is now ready to go.  If any errors occur during device
3317          * registration we do not fail the whole card but rather proceed only
3318          * with the ports we manage to register successfully.  However we must
3319          * register at least one net device.
3320          */
3321         for_each_port(adapter, i) {
3322                 err = register_netdev(adapter->port[i]);
3323                 if (err)
3324                         dev_warn(&pdev->dev,
3325                                  "cannot register net device %s, skipping\n",
3326                                  adapter->port[i]->name);
3327                 else {
3328                         /*
3329                          * Change the name we use for messages to the name of
3330                          * the first successfully registered interface.
3331                          */
3332                         if (!adapter->registered_device_map)
3333                                 adapter->name = adapter->port[i]->name;
3334
3335                         __set_bit(i, &adapter->registered_device_map);
3336                 }
3337         }
3338         if (!adapter->registered_device_map) {
3339                 dev_err(&pdev->dev, "could not register any net devices\n");
3340                 goto out_free_dev;
3341         }
3342
3343         for_each_port(adapter, i)
3344                 cxgb3_init_iscsi_mac(adapter->port[i]);
3345
3346         /* Driver's ready. Reflect it on LEDs */
3347         t3_led_ready(adapter);
3348
3349         if (is_offload(adapter)) {
3350                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3351                 cxgb3_adapter_ofld(adapter);
3352         }
3353
3354         /* See what interrupts we'll be using */
3355         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3356                 adapter->flags |= USING_MSIX;
3357         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3358                 adapter->flags |= USING_MSI;
3359
3360         set_nqsets(adapter);
3361
3362         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3363                                  &cxgb3_attr_group);
3364
3365         print_port_info(adapter, ai);
3366         return 0;
3367
3368 out_free_dev:
3369         iounmap(adapter->regs);
3370         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3371                 if (adapter->port[i])
3372                         free_netdev(adapter->port[i]);
3373
3374 out_free_adapter:
3375         kfree(adapter);
3376
3377 out_disable_device:
3378         pci_disable_device(pdev);
3379 out_release_regions:
3380         pci_release_regions(pdev);
3381         pci_set_drvdata(pdev, NULL);
3382         return err;
3383 }
3384
3385 static void __devexit remove_one(struct pci_dev *pdev)
3386 {
3387         struct adapter *adapter = pci_get_drvdata(pdev);
3388
3389         if (adapter) {
3390                 int i;
3391
3392                 t3_sge_stop(adapter);
3393                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3394                                    &cxgb3_attr_group);
3395
3396                 if (is_offload(adapter)) {
3397                         cxgb3_adapter_unofld(adapter);
3398                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3399                                      &adapter->open_device_map))
3400                                 offload_close(&adapter->tdev);
3401                 }
3402
3403                 for_each_port(adapter, i)
3404                     if (test_bit(i, &adapter->registered_device_map))
3405                         unregister_netdev(adapter->port[i]);
3406
3407                 t3_stop_sge_timers(adapter);
3408                 t3_free_sge_resources(adapter);
3409                 cxgb_disable_msi(adapter);
3410
3411                 for_each_port(adapter, i)
3412                         if (adapter->port[i])
3413                                 free_netdev(adapter->port[i]);
3414
3415                 iounmap(adapter->regs);
3416                 if (adapter->nofail_skb)
3417                         kfree_skb(adapter->nofail_skb);
3418                 kfree(adapter);
3419                 pci_release_regions(pdev);
3420                 pci_disable_device(pdev);
3421                 pci_set_drvdata(pdev, NULL);
3422         }
3423 }
3424
3425 static struct pci_driver driver = {
3426         .name = DRV_NAME,
3427         .id_table = cxgb3_pci_tbl,
3428         .probe = init_one,
3429         .remove = __devexit_p(remove_one),
3430         .err_handler = &t3_err_handler,
3431 };
3432
3433 static int __init cxgb3_init_module(void)
3434 {
3435         int ret;
3436
3437         cxgb3_offload_init();
3438
3439         ret = pci_register_driver(&driver);
3440         return ret;
3441 }
3442
3443 static void __exit cxgb3_cleanup_module(void)
3444 {
3445         pci_unregister_driver(&driver);
3446         if (cxgb3_wq)
3447                 destroy_workqueue(cxgb3_wq);
3448 }
3449
3450 module_init(cxgb3_init_module);
3451 module_exit(cxgb3_cleanup_module);