drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496         struct be_drv_stats *drvs = &adapter->drv_stats;
 497         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 498
 499         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 500         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 501         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 502         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 503         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 504         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 505         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 506         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 507         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 508         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 509         drvs->rx_dropped_tcp_length =
 510                                 pport_stats->rx_dropped_invalid_tcp_length;
 511         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 512         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 513         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 514         drvs->rx_dropped_header_too_small =
 515                                 pport_stats->rx_dropped_header_too_small;
 516         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 517         drvs->rx_address_filtered =
 518                                         pport_stats->rx_address_filtered +
 519                                         pport_stats->rx_vlan_filtered;
 520         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 521         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 522         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 523         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 524         drvs->jabber_events = pport_stats->rx_jabbers;
 525         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 526         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 527         drvs->rx_drops_too_many_frags =
 528                                 pport_stats->rx_drops_too_many_frags_lo;
 529 }
 530
 531 static void accumulate_16bit_val(u32 *acc, u16 val)
 532 {
 533 #define lo(x)                   (x & 0xFFFF)
 534 #define hi(x)                   (x & 0xFFFF0000)
 535         bool wrapped = val < lo(*acc);
 536         u32 newacc = hi(*acc) + val;
 537
 538         if (wrapped)
 539                 newacc += 65536;
 540         ACCESS_ONCE(*acc) = newacc;
 541 }
 542
 543 static void populate_erx_stats(struct be_adapter *adapter,
 544                                struct be_rx_obj *rxo, u32 erx_stat)
 545 {
 546         if (!BEx_chip(adapter))
 547                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 548         else
 549                 /* below erx HW counter can actually wrap around after
 550                  * 65535. Driver accumulates a 32-bit value
 551                  */
 552                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 553                                      (u16)erx_stat);
 554 }
 555
 556 void be_parse_stats(struct be_adapter *adapter)
 557 {
 558         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 559         struct be_rx_obj *rxo;
 560         int i;
 561         u32 erx_stat;
 562
 563         if (lancer_chip(adapter)) {
 564                 populate_lancer_stats(adapter);
 565         } else {
 566                 if (BE2_chip(adapter))
 567                         populate_be_v0_stats(adapter);
 568                 else if (BE3_chip(adapter))
 569                         /* for BE3 */
 570                         populate_be_v1_stats(adapter);
 571                 else
 572                         populate_be_v2_stats(adapter);
 573
 574                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 575                 for_all_rx_queues(adapter, rxo, i) {
 576                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 577                         populate_erx_stats(adapter, rxo, erx_stat);
 578                 }
 579         }
 580 }
 581
 582 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 583                                                 struct rtnl_link_stats64 *stats)
 584 {
 585         struct be_adapter *adapter = netdev_priv(netdev);
 586         struct be_drv_stats *drvs = &adapter->drv_stats;
 587         struct be_rx_obj *rxo;
 588         struct be_tx_obj *txo;
 589         u64 pkts, bytes;
 590         unsigned int start;
 591         int i;
 592
 593         for_all_rx_queues(adapter, rxo, i) {
 594                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 595
 596                 do {
 597                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 598                         pkts = rx_stats(rxo)->rx_pkts;
 599                         bytes = rx_stats(rxo)->rx_bytes;
 600                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 601                 stats->rx_packets += pkts;
 602                 stats->rx_bytes += bytes;
 603                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 604                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 605                                         rx_stats(rxo)->rx_drops_no_frags;
 606         }
 607
 608         for_all_tx_queues(adapter, txo, i) {
 609                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 610
 611                 do {
 612                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 613                         pkts = tx_stats(txo)->tx_pkts;
 614                         bytes = tx_stats(txo)->tx_bytes;
 615                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 616                 stats->tx_packets += pkts;
 617                 stats->tx_bytes += bytes;
 618         }
 619
 620         /* bad pkts received */
 621         stats->rx_errors = drvs->rx_crc_errors +
 622                 drvs->rx_alignment_symbol_errors +
 623                 drvs->rx_in_range_errors +
 624                 drvs->rx_out_range_errors +
 625                 drvs->rx_frame_too_long +
 626                 drvs->rx_dropped_too_small +
 627                 drvs->rx_dropped_too_short +
 628                 drvs->rx_dropped_header_too_small +
 629                 drvs->rx_dropped_tcp_length +
 630                 drvs->rx_dropped_runt;
 631
 632         /* detailed rx errors */
 633         stats->rx_length_errors = drvs->rx_in_range_errors +
 634                 drvs->rx_out_range_errors +
 635                 drvs->rx_frame_too_long;
 636
 637         stats->rx_crc_errors = drvs->rx_crc_errors;
 638
 639         /* frame alignment errors */
 640         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 641
 642         /* receiver fifo overrun */
 643         /* drops_no_pbuf is no per i/f, it's per BE card */
 644         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 645                                 drvs->rx_input_fifo_overflow_drop +
 646                                 drvs->rx_drops_no_pbuf;
 647         return stats;
 648 }
 649
 650 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 651 {
 652         struct net_device *netdev = adapter->netdev;
 653
 654         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 655                 netif_carrier_off(netdev);
 656                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 657         }
 658
 659         if (link_status)
 660                 netif_carrier_on(netdev);
 661         else
 662                 netif_carrier_off(netdev);
 663 }
 664
 665 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 666 {
 667         struct be_tx_stats *stats = tx_stats(txo);
 668
 669         u64_stats_update_begin(&stats->sync);
 670         stats->tx_reqs++;
 671         stats->tx_bytes += skb->len;
 672         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 673         u64_stats_update_end(&stats->sync);
 674 }
 675
 676 /* Returns number of WRBs needed for the skb */
 677 static u32 skb_wrb_cnt(struct sk_buff *skb)
 678 {
 679         /* +1 for the header wrb */
 680         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 681 }
 682
 683 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 684 {
 685         wrb->frag_pa_hi = upper_32_bits(addr);
 686         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 687         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 688         wrb->rsvd0 = 0;
 689 }
 690
 691 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 692                                      struct sk_buff *skb)
 693 {
 694         u8 vlan_prio;
 695         u16 vlan_tag;
 696
 697         vlan_tag = skb_vlan_tag_get(skb);
 698         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 699         /* If vlan priority provided by OS is NOT in available bmap */
 700         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 701                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 702                                 adapter->recommended_prio;
 703
 704         return vlan_tag;
 705 }
 706
 707 /* Used only for IP tunnel packets */
 708 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 709 {
 710         return (inner_ip_hdr(skb)->version == 4) ?
 711                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 712 }
 713
 714 static u16 skb_ip_proto(struct sk_buff *skb)
 715 {
 716         return (ip_hdr(skb)->version == 4) ?
 717                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 718 }
 719
 720 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 721                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 722                          bool skip_hw_vlan)
 723 {
 724         u16 vlan_tag, proto;
 725
 726         memset(hdr, 0, sizeof(*hdr));
 727
 728         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 729
 730         if (skb_is_gso(skb)) {
 731                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 732                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 733                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 734                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 735         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 736                 if (skb->encapsulation) {
 737                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 738                         proto = skb_inner_ip_proto(skb);
 739                 } else {
 740                         proto = skb_ip_proto(skb);
 741                 }
 742                 if (proto == IPPROTO_TCP)
 743                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 744                 else if (proto == IPPROTO_UDP)
 745                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 746         }
 747
 748         if (skb_vlan_tag_present(skb)) {
 749                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 750                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 751                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 752         }
 753
 754         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 755         SET_TX_WRB_HDR_BITS(len, hdr, len);
 756
 757         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0
 758          * When this hack is not needed, the evt bit is set while ringing DB
 759          */
 760         if (skip_hw_vlan)
 761                 SET_TX_WRB_HDR_BITS(event, hdr, 1);
 762 }
 763
 764 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 765                           bool unmap_single)
 766 {
 767         dma_addr_t dma;
 768
 769         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 770
 771         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 772         if (wrb->frag_len) {
 773                 if (unmap_single)
 774                         dma_unmap_single(dev, dma, wrb->frag_len,
 775                                          DMA_TO_DEVICE);
 776                 else
 777                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 778         }
 779 }
 780
 781 /* Returns the number of WRBs used up by the skb */
 782 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 783                            struct sk_buff *skb, bool skip_hw_vlan)
 784 {
 785         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 786         struct device *dev = &adapter->pdev->dev;
 787         struct be_queue_info *txq = &txo->q;
 788         struct be_eth_hdr_wrb *hdr;
 789         bool map_single = false;
 790         struct be_eth_wrb *wrb;
 791         dma_addr_t busaddr;
 792         u16 head = txq->head;
 793
 794         hdr = queue_head_node(txq);
 795         wrb_fill_hdr(adapter, hdr, skb, wrb_cnt, skb->len, skip_hw_vlan);
 796         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 797
 798         queue_head_inc(txq);
 799
 800         if (skb->len > skb->data_len) {
 801                 int len = skb_headlen(skb);
 802
 803                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 804                 if (dma_mapping_error(dev, busaddr))
 805                         goto dma_err;
 806                 map_single = true;
 807                 wrb = queue_head_node(txq);
 808                 wrb_fill(wrb, busaddr, len);
 809                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 810                 queue_head_inc(txq);
 811                 copied += len;
 812         }
 813
 814         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 815                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 816
 817                 busaddr = skb_frag_dma_map(dev, frag, 0,
 818                                            skb_frag_size(frag), DMA_TO_DEVICE);
 819                 if (dma_mapping_error(dev, busaddr))
 820                         goto dma_err;
 821                 wrb = queue_head_node(txq);
 822                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 823                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 824                 queue_head_inc(txq);
 825                 copied += skb_frag_size(frag);
 826         }
 827
 828         BUG_ON(txo->sent_skb_list[head]);
 829         txo->sent_skb_list[head] = skb;
 830         txo->last_req_hdr = head;
 831         atomic_add(wrb_cnt, &txq->used);
 832         txo->last_req_wrb_cnt = wrb_cnt;
 833         txo->pend_wrb_cnt += wrb_cnt;
 834
 835         be_tx_stats_update(txo, skb);
 836         return wrb_cnt;
 837
 838 dma_err:
 839         /* Bring the queue back to the state it was in before this
 840          * routine was invoked.
 841          */
 842         txq->head = head;
 843         /* skip the first wrb (hdr); it's not mapped */
 844         queue_head_inc(txq);
 845         while (copied) {
 846                 wrb = queue_head_node(txq);
 847                 unmap_tx_frag(dev, wrb, map_single);
 848                 map_single = false;
 849                 copied -= wrb->frag_len;
 850                 adapter->drv_stats.dma_map_errors++;
 851                 queue_head_inc(txq);
 852         }
 853         txq->head = head;
 854         return 0;
 855 }
 856
 857 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 858                                              struct sk_buff *skb,
 859                                              bool *skip_hw_vlan)
 860 {
 861         u16 vlan_tag = 0;
 862
 863         skb = skb_share_check(skb, GFP_ATOMIC);
 864         if (unlikely(!skb))
 865                 return skb;
 866
 867         if (skb_vlan_tag_present(skb))
 868                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 869
 870         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 871                 if (!vlan_tag)
 872                         vlan_tag = adapter->pvid;
 873                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 874                  * skip VLAN insertion
 875                  */
 876                 if (skip_hw_vlan)
 877                         *skip_hw_vlan = true;
 878         }
 879
 880         if (vlan_tag) {
 881                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 882                                                 vlan_tag);
 883                 if (unlikely(!skb))
 884                         return skb;
 885                 skb->vlan_tci = 0;
 886         }
 887
 888         /* Insert the outer VLAN, if any */
 889         if (adapter->qnq_vid) {
 890                 vlan_tag = adapter->qnq_vid;
 891                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 892                                                 vlan_tag);
 893                 if (unlikely(!skb))
 894                         return skb;
 895                 if (skip_hw_vlan)
 896                         *skip_hw_vlan = true;
 897         }
 898
 899         return skb;
 900 }
 901
 902 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 903 {
 904         struct ethhdr *eh = (struct ethhdr *)skb->data;
 905         u16 offset = ETH_HLEN;
 906
 907         if (eh->h_proto == htons(ETH_P_IPV6)) {
 908                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 909
 910                 offset += sizeof(struct ipv6hdr);
 911                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 912                     ip6h->nexthdr != NEXTHDR_UDP) {
 913                         struct ipv6_opt_hdr *ehdr =
 914                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 915
 916                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 917                         if (ehdr->hdrlen == 0xff)
 918                                 return true;
 919                 }
 920         }
 921         return false;
 922 }
 923
 924 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 925 {
 926         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 927 }
 928
 929 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 930 {
 931         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 932 }
 933
 934 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 935                                                   struct sk_buff *skb,
 936                                                   bool *skip_hw_vlan)
 937 {
 938         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 939         unsigned int eth_hdr_len;
 940         struct iphdr *ip;
 941
 942         /* For padded packets, BE HW modifies tot_len field in IP header
 943          * incorrecly when VLAN tag is inserted by HW.
 944          * For padded packets, Lancer computes incorrect checksum.
 945          */
 946         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 947                                                 VLAN_ETH_HLEN : ETH_HLEN;
 948         if (skb->len <= 60 &&
 949             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
 950             is_ipv4_pkt(skb)) {
 951                 ip = (struct iphdr *)ip_hdr(skb);
 952                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 953         }
 954
 955         /* If vlan tag is already inlined in the packet, skip HW VLAN
 956          * tagging in pvid-tagging mode
 957          */
 958         if (be_pvid_tagging_enabled(adapter) &&
 959             veh->h_vlan_proto == htons(ETH_P_8021Q))
 960                 *skip_hw_vlan = true;
 961
 962         /* HW has a bug wherein it will calculate CSUM for VLAN
 963          * pkts even though it is disabled.
 964          * Manually insert VLAN in pkt.
 965          */
 966         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 967             skb_vlan_tag_present(skb)) {
 968                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 969                 if (unlikely(!skb))
 970                         goto err;
 971         }
 972
 973         /* HW may lockup when VLAN HW tagging is requested on
 974          * certain ipv6 packets. Drop such pkts if the HW workaround to
 975          * skip HW tagging is not enabled by FW.
 976          */
 977         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 978                      (adapter->pvid || adapter->qnq_vid) &&
 979                      !qnq_async_evt_rcvd(adapter)))
 980                 goto tx_drop;
 981
 982         /* Manual VLAN tag insertion to prevent:
 983          * ASIC lockup when the ASIC inserts VLAN tag into
 984          * certain ipv6 packets. Insert VLAN tags in driver,
 985          * and set event, completion, vlan bits accordingly
 986          * in the Tx WRB.
 987          */
 988         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 989             be_vlan_tag_tx_chk(adapter, skb)) {
 990                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 991                 if (unlikely(!skb))
 992                         goto err;
 993         }
 994
 995         return skb;
 996 tx_drop:
 997         dev_kfree_skb_any(skb);
 998 err:
 999         return NULL;
1000 }
1001
1002 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1003                                            struct sk_buff *skb,
1004                                            bool *skip_hw_vlan)
1005 {
1006         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1007          * less may cause a transmit stall on that port. So the work-around is
1008          * to pad short packets (<= 32 bytes) to a 36-byte length.
1009          */
1010         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1011                 if (skb_put_padto(skb, 36))
1012                         return NULL;
1013         }
1014
1015         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1016                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1017                 if (!skb)
1018                         return NULL;
1019         }
1020
1021         return skb;
1022 }
1023
1024 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1025 {
1026         struct be_queue_info *txq = &txo->q;
1027         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1028
1029         /* Mark the last request eventable if it hasn't been marked already */
1030         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1031                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1032
1033         /* compose a dummy wrb if there are odd set of wrbs to notify */
1034         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1035                 wrb_fill(queue_head_node(txq), 0, 0);
1036                 queue_head_inc(txq);
1037                 atomic_inc(&txq->used);
1038                 txo->pend_wrb_cnt++;
1039                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1040                                            TX_HDR_WRB_NUM_SHIFT);
1041                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1042                                           TX_HDR_WRB_NUM_SHIFT);
1043         }
1044         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1045         txo->pend_wrb_cnt = 0;
1046 }
1047
1048 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1049 {
1050         bool skip_hw_vlan = false, flush = !skb->xmit_more;
1051         struct be_adapter *adapter = netdev_priv(netdev);
1052         u16 q_idx = skb_get_queue_mapping(skb);
1053         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1054         struct be_queue_info *txq = &txo->q;
1055         u16 wrb_cnt;
1056
1057         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1058         if (unlikely(!skb))
1059                 goto drop;
1060
1061         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, skip_hw_vlan);
1062         if (unlikely(!wrb_cnt)) {
1063                 dev_kfree_skb_any(skb);
1064                 goto drop;
1065         }
1066
1067         if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) {
1068                 netif_stop_subqueue(netdev, q_idx);
1069                 tx_stats(txo)->tx_stops++;
1070         }
1071
1072         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1073                 be_xmit_flush(adapter, txo);
1074
1075         return NETDEV_TX_OK;
1076 drop:
1077         tx_stats(txo)->tx_drv_drops++;
1078         /* Flush the already enqueued tx requests */
1079         if (flush && txo->pend_wrb_cnt)
1080                 be_xmit_flush(adapter, txo);
1081
1082         return NETDEV_TX_OK;
1083 }
1084
1085 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1086 {
1087         struct be_adapter *adapter = netdev_priv(netdev);
1088         struct device *dev = &adapter->pdev->dev;
1089
1090         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1091                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1092                          BE_MIN_MTU, BE_MAX_MTU);
1093                 return -EINVAL;
1094         }
1095
1096         dev_info(dev, "MTU changed from %d to %d bytes\n",
1097                  netdev->mtu, new_mtu);
1098         netdev->mtu = new_mtu;
1099         return 0;
1100 }
1101
1102 /*
1103  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1104  * If the user configures more, place BE in vlan promiscuous mode.
1105  */
1106 static int be_vid_config(struct be_adapter *adapter)
1107 {
1108         struct device *dev = &adapter->pdev->dev;
1109         u16 vids[BE_NUM_VLANS_SUPPORTED];
1110         u16 num = 0, i = 0;
1111         int status = 0;
1112
1113         /* No need to further configure vids if in promiscuous mode */
1114         if (adapter->promiscuous)
1115                 return 0;
1116
1117         if (adapter->vlans_added > be_max_vlans(adapter))
1118                 goto set_vlan_promisc;
1119
1120         /* Construct VLAN Table to give to HW */
1121         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1122                 vids[num++] = cpu_to_le16(i);
1123
1124         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1125         if (status) {
1126                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1127                 if (addl_status(status) ==
1128                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1129                         goto set_vlan_promisc;
1130                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1131         } else {
1132                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1133                         /* hw VLAN filtering re-enabled. */
1134                         status = be_cmd_rx_filter(adapter,
1135                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1136                         if (!status) {
1137                                 dev_info(dev,
1138                                          "Disabling VLAN Promiscuous mode\n");
1139                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1140                         }
1141                 }
1142         }
1143
1144         return status;
1145
1146 set_vlan_promisc:
1147         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1148                 return 0;
1149
1150         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1151         if (!status) {
1152                 dev_info(dev, "Enable VLAN Promiscuous mode\n");
1153                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1154         } else
1155                 dev_err(dev, "Failed to enable VLAN Promiscuous mode\n");
1156         return status;
1157 }
1158
1159 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1160 {
1161         struct be_adapter *adapter = netdev_priv(netdev);
1162         int status = 0;
1163
1164         /* Packets with VID 0 are always received by Lancer by default */
1165         if (lancer_chip(adapter) && vid == 0)
1166                 return status;
1167
1168         if (test_bit(vid, adapter->vids))
1169                 return status;
1170
1171         set_bit(vid, adapter->vids);
1172         adapter->vlans_added++;
1173
1174         status = be_vid_config(adapter);
1175         if (status) {
1176                 adapter->vlans_added--;
1177                 clear_bit(vid, adapter->vids);
1178         }
1179
1180         return status;
1181 }
1182
1183 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1184 {
1185         struct be_adapter *adapter = netdev_priv(netdev);
1186
1187         /* Packets with VID 0 are always received by Lancer by default */
1188         if (lancer_chip(adapter) && vid == 0)
1189                 return 0;
1190
1191         clear_bit(vid, adapter->vids);
1192         adapter->vlans_added--;
1193
1194         return be_vid_config(adapter);
1195 }
1196
1197 static void be_clear_promisc(struct be_adapter *adapter)
1198 {
1199         adapter->promiscuous = false;
1200         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1201
1202         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1203 }
1204
1205 static void be_set_rx_mode(struct net_device *netdev)
1206 {
1207         struct be_adapter *adapter = netdev_priv(netdev);
1208         int status;
1209
1210         if (netdev->flags & IFF_PROMISC) {
1211                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1212                 adapter->promiscuous = true;
1213                 goto done;
1214         }
1215
1216         /* BE was previously in promiscuous mode; disable it */
1217         if (adapter->promiscuous) {
1218                 be_clear_promisc(adapter);
1219                 if (adapter->vlans_added)
1220                         be_vid_config(adapter);
1221         }
1222
1223         /* Enable multicast promisc if num configured exceeds what we support */
1224         if (netdev->flags & IFF_ALLMULTI ||
1225             netdev_mc_count(netdev) > be_max_mc(adapter))
1226                 goto set_mcast_promisc;
1227
1228         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1229                 struct netdev_hw_addr *ha;
1230                 int i = 1; /* First slot is claimed by the Primary MAC */
1231
1232                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1233                         be_cmd_pmac_del(adapter, adapter->if_handle,
1234                                         adapter->pmac_id[i], 0);
1235                 }
1236
1237                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1238                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1239                         adapter->promiscuous = true;
1240                         goto done;
1241                 }
1242
1243                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1244                         adapter->uc_macs++; /* First slot is for Primary MAC */
1245                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1246                                         adapter->if_handle,
1247                                         &adapter->pmac_id[adapter->uc_macs], 0);
1248                 }
1249         }
1250
1251         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1252         if (!status) {
1253                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1254                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1255                 goto done;
1256         }
1257
1258 set_mcast_promisc:
1259         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1260                 return;
1261
1262         /* Set to MCAST promisc mode if setting MULTICAST address fails
1263          * or if num configured exceeds what we support
1264          */
1265         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1266         if (!status)
1267                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1268 done:
1269         return;
1270 }
1271
1272 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1273 {
1274         struct be_adapter *adapter = netdev_priv(netdev);
1275         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1276         int status;
1277
1278         if (!sriov_enabled(adapter))
1279                 return -EPERM;
1280
1281         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1282                 return -EINVAL;
1283
1284         /* Proceed further only if user provided MAC is different
1285          * from active MAC
1286          */
1287         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1288                 return 0;
1289
1290         if (BEx_chip(adapter)) {
1291                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1292                                 vf + 1);
1293
1294                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1295                                          &vf_cfg->pmac_id, vf + 1);
1296         } else {
1297                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1298                                         vf + 1);
1299         }
1300
1301         if (status) {
1302                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1303                         mac, vf, status);
1304                 return be_cmd_status(status);
1305         }
1306
1307         ether_addr_copy(vf_cfg->mac_addr, mac);
1308
1309         return 0;
1310 }
1311
1312 static int be_get_vf_config(struct net_device *netdev, int vf,
1313                             struct ifla_vf_info *vi)
1314 {
1315         struct be_adapter *adapter = netdev_priv(netdev);
1316         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1317
1318         if (!sriov_enabled(adapter))
1319                 return -EPERM;
1320
1321         if (vf >= adapter->num_vfs)
1322                 return -EINVAL;
1323
1324         vi->vf = vf;
1325         vi->max_tx_rate = vf_cfg->tx_rate;
1326         vi->min_tx_rate = 0;
1327         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1328         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1329         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1330         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1331
1332         return 0;
1333 }
1334
1335 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1336 {
1337         struct be_adapter *adapter = netdev_priv(netdev);
1338         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1339         int status = 0;
1340
1341         if (!sriov_enabled(adapter))
1342                 return -EPERM;
1343
1344         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1345                 return -EINVAL;
1346
1347         if (vlan || qos) {
1348                 vlan |= qos << VLAN_PRIO_SHIFT;
1349                 if (vf_cfg->vlan_tag != vlan)
1350                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1351                                                        vf_cfg->if_handle, 0);
1352         } else {
1353                 /* Reset Transparent Vlan Tagging. */
1354                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1355                                                vf + 1, vf_cfg->if_handle, 0);
1356         }
1357
1358         if (status) {
1359                 dev_err(&adapter->pdev->dev,
1360                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1361                         vf, status);
1362                 return be_cmd_status(status);
1363         }
1364
1365         vf_cfg->vlan_tag = vlan;
1366
1367         return 0;
1368 }
1369
1370 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1371                              int min_tx_rate, int max_tx_rate)
1372 {
1373         struct be_adapter *adapter = netdev_priv(netdev);
1374         struct device *dev = &adapter->pdev->dev;
1375         int percent_rate, status = 0;
1376         u16 link_speed = 0;
1377         u8 link_status;
1378
1379         if (!sriov_enabled(adapter))
1380                 return -EPERM;
1381
1382         if (vf >= adapter->num_vfs)
1383                 return -EINVAL;
1384
1385         if (min_tx_rate)
1386                 return -EINVAL;
1387
1388         if (!max_tx_rate)
1389                 goto config_qos;
1390
1391         status = be_cmd_link_status_query(adapter, &link_speed,
1392                                           &link_status, 0);
1393         if (status)
1394                 goto err;
1395
1396         if (!link_status) {
1397                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1398                 status = -ENETDOWN;
1399                 goto err;
1400         }
1401
1402         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1403                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1404                         link_speed);
1405                 status = -EINVAL;
1406                 goto err;
1407         }
1408
1409         /* On Skyhawk the QOS setting must be done only as a % value */
1410         percent_rate = link_speed / 100;
1411         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1412                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1413                         percent_rate);
1414                 status = -EINVAL;
1415                 goto err;
1416         }
1417
1418 config_qos:
1419         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1420         if (status)
1421                 goto err;
1422
1423         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1424         return 0;
1425
1426 err:
1427         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1428                 max_tx_rate, vf);
1429         return be_cmd_status(status);
1430 }
1431
1432 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1433                                 int link_state)
1434 {
1435         struct be_adapter *adapter = netdev_priv(netdev);
1436         int status;
1437
1438         if (!sriov_enabled(adapter))
1439                 return -EPERM;
1440
1441         if (vf >= adapter->num_vfs)
1442                 return -EINVAL;
1443
1444         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1445         if (status) {
1446                 dev_err(&adapter->pdev->dev,
1447                         "Link state change on VF %d failed: %#x\n", vf, status);
1448                 return be_cmd_status(status);
1449         }
1450
1451         adapter->vf_cfg[vf].plink_tracking = link_state;
1452
1453         return 0;
1454 }
1455
1456 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1457                           ulong now)
1458 {
1459         aic->rx_pkts_prev = rx_pkts;
1460         aic->tx_reqs_prev = tx_pkts;
1461         aic->jiffies = now;
1462 }
1463
1464 static void be_eqd_update(struct be_adapter *adapter)
1465 {
1466         struct be_set_eqd set_eqd[MAX_EVT_QS];
1467         int eqd, i, num = 0, start;
1468         struct be_aic_obj *aic;
1469         struct be_eq_obj *eqo;
1470         struct be_rx_obj *rxo;
1471         struct be_tx_obj *txo;
1472         u64 rx_pkts, tx_pkts;
1473         ulong now;
1474         u32 pps, delta;
1475
1476         for_all_evt_queues(adapter, eqo, i) {
1477                 aic = &adapter->aic_obj[eqo->idx];
1478                 if (!aic->enable) {
1479                         if (aic->jiffies)
1480                                 aic->jiffies = 0;
1481                         eqd = aic->et_eqd;
1482                         goto modify_eqd;
1483                 }
1484
1485                 rxo = &adapter->rx_obj[eqo->idx];
1486                 do {
1487                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1488                         rx_pkts = rxo->stats.rx_pkts;
1489                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1490
1491                 txo = &adapter->tx_obj[eqo->idx];
1492                 do {
1493                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1494                         tx_pkts = txo->stats.tx_reqs;
1495                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1496
1497                 /* Skip, if wrapped around or first calculation */
1498                 now = jiffies;
1499                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1500                     rx_pkts < aic->rx_pkts_prev ||
1501                     tx_pkts < aic->tx_reqs_prev) {
1502                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1503                         continue;
1504                 }
1505
1506                 delta = jiffies_to_msecs(now - aic->jiffies);
1507                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1508                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1509                 eqd = (pps / 15000) << 2;
1510
1511                 if (eqd < 8)
1512                         eqd = 0;
1513                 eqd = min_t(u32, eqd, aic->max_eqd);
1514                 eqd = max_t(u32, eqd, aic->min_eqd);
1515
1516                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1517 modify_eqd:
1518                 if (eqd != aic->prev_eqd) {
1519                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1520                         set_eqd[num].eq_id = eqo->q.id;
1521                         aic->prev_eqd = eqd;
1522                         num++;
1523                 }
1524         }
1525
1526         if (num)
1527                 be_cmd_modify_eqd(adapter, set_eqd, num);
1528 }
1529
1530 static void be_rx_stats_update(struct be_rx_obj *rxo,
1531                                struct be_rx_compl_info *rxcp)
1532 {
1533         struct be_rx_stats *stats = rx_stats(rxo);
1534
1535         u64_stats_update_begin(&stats->sync);
1536         stats->rx_compl++;
1537         stats->rx_bytes += rxcp->pkt_size;
1538         stats->rx_pkts++;
1539         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1540                 stats->rx_mcast_pkts++;
1541         if (rxcp->err)
1542                 stats->rx_compl_err++;
1543         u64_stats_update_end(&stats->sync);
1544 }
1545
1546 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1547 {
1548         /* L4 checksum is not reliable for non TCP/UDP packets.
1549          * Also ignore ipcksm for ipv6 pkts
1550          */
1551         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1552                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1553 }
1554
1555 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1556 {
1557         struct be_adapter *adapter = rxo->adapter;
1558         struct be_rx_page_info *rx_page_info;
1559         struct be_queue_info *rxq = &rxo->q;
1560         u16 frag_idx = rxq->tail;
1561
1562         rx_page_info = &rxo->page_info_tbl[frag_idx];
1563         BUG_ON(!rx_page_info->page);
1564
1565         if (rx_page_info->last_frag) {
1566                 dma_unmap_page(&adapter->pdev->dev,
1567                                dma_unmap_addr(rx_page_info, bus),
1568                                adapter->big_page_size, DMA_FROM_DEVICE);
1569                 rx_page_info->last_frag = false;
1570         } else {
1571                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1572                                         dma_unmap_addr(rx_page_info, bus),
1573                                         rx_frag_size, DMA_FROM_DEVICE);
1574         }
1575
1576         queue_tail_inc(rxq);
1577         atomic_dec(&rxq->used);
1578         return rx_page_info;
1579 }
1580
1581 /* Throwaway the data in the Rx completion */
1582 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1583                                 struct be_rx_compl_info *rxcp)
1584 {
1585         struct be_rx_page_info *page_info;
1586         u16 i, num_rcvd = rxcp->num_rcvd;
1587
1588         for (i = 0; i < num_rcvd; i++) {
1589                 page_info = get_rx_page_info(rxo);
1590                 put_page(page_info->page);
1591                 memset(page_info, 0, sizeof(*page_info));
1592         }
1593 }
1594
1595 /*
1596  * skb_fill_rx_data forms a complete skb for an ether frame
1597  * indicated by rxcp.
1598  */
1599 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1600                              struct be_rx_compl_info *rxcp)
1601 {
1602         struct be_rx_page_info *page_info;
1603         u16 i, j;
1604         u16 hdr_len, curr_frag_len, remaining;
1605         u8 *start;
1606
1607         page_info = get_rx_page_info(rxo);
1608         start = page_address(page_info->page) + page_info->page_offset;
1609         prefetch(start);
1610
1611         /* Copy data in the first descriptor of this completion */
1612         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1613
1614         skb->len = curr_frag_len;
1615         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1616                 memcpy(skb->data, start, curr_frag_len);
1617                 /* Complete packet has now been moved to data */
1618                 put_page(page_info->page);
1619                 skb->data_len = 0;
1620                 skb->tail += curr_frag_len;
1621         } else {
1622                 hdr_len = ETH_HLEN;
1623                 memcpy(skb->data, start, hdr_len);
1624                 skb_shinfo(skb)->nr_frags = 1;
1625                 skb_frag_set_page(skb, 0, page_info->page);
1626                 skb_shinfo(skb)->frags[0].page_offset =
1627                                         page_info->page_offset + hdr_len;
1628                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1629                                   curr_frag_len - hdr_len);
1630                 skb->data_len = curr_frag_len - hdr_len;
1631                 skb->truesize += rx_frag_size;
1632                 skb->tail += hdr_len;
1633         }
1634         page_info->page = NULL;
1635
1636         if (rxcp->pkt_size <= rx_frag_size) {
1637                 BUG_ON(rxcp->num_rcvd != 1);
1638                 return;
1639         }
1640
1641         /* More frags present for this completion */
1642         remaining = rxcp->pkt_size - curr_frag_len;
1643         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1644                 page_info = get_rx_page_info(rxo);
1645                 curr_frag_len = min(remaining, rx_frag_size);
1646
1647                 /* Coalesce all frags from the same physical page in one slot */
1648                 if (page_info->page_offset == 0) {
1649                         /* Fresh page */
1650                         j++;
1651                         skb_frag_set_page(skb, j, page_info->page);
1652                         skb_shinfo(skb)->frags[j].page_offset =
1653                                                         page_info->page_offset;
1654                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1655                         skb_shinfo(skb)->nr_frags++;
1656                 } else {
1657                         put_page(page_info->page);
1658                 }
1659
1660                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1661                 skb->len += curr_frag_len;
1662                 skb->data_len += curr_frag_len;
1663                 skb->truesize += rx_frag_size;
1664                 remaining -= curr_frag_len;
1665                 page_info->page = NULL;
1666         }
1667         BUG_ON(j > MAX_SKB_FRAGS);
1668 }
1669
1670 /* Process the RX completion indicated by rxcp when GRO is disabled */
1671 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1672                                 struct be_rx_compl_info *rxcp)
1673 {
1674         struct be_adapter *adapter = rxo->adapter;
1675         struct net_device *netdev = adapter->netdev;
1676         struct sk_buff *skb;
1677
1678         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1679         if (unlikely(!skb)) {
1680                 rx_stats(rxo)->rx_drops_no_skbs++;
1681                 be_rx_compl_discard(rxo, rxcp);
1682                 return;
1683         }
1684
1685         skb_fill_rx_data(rxo, skb, rxcp);
1686
1687         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1688                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1689         else
1690                 skb_checksum_none_assert(skb);
1691
1692         skb->protocol = eth_type_trans(skb, netdev);
1693         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1694         if (netdev->features & NETIF_F_RXHASH)
1695                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1696
1697         skb->csum_level = rxcp->tunneled;
1698         skb_mark_napi_id(skb, napi);
1699
1700         if (rxcp->vlanf)
1701                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1702
1703         netif_receive_skb(skb);
1704 }
1705
1706 /* Process the RX completion indicated by rxcp when GRO is enabled */
1707 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1708                                     struct napi_struct *napi,
1709                                     struct be_rx_compl_info *rxcp)
1710 {
1711         struct be_adapter *adapter = rxo->adapter;
1712         struct be_rx_page_info *page_info;
1713         struct sk_buff *skb = NULL;
1714         u16 remaining, curr_frag_len;
1715         u16 i, j;
1716
1717         skb = napi_get_frags(napi);
1718         if (!skb) {
1719                 be_rx_compl_discard(rxo, rxcp);
1720                 return;
1721         }
1722
1723         remaining = rxcp->pkt_size;
1724         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1725                 page_info = get_rx_page_info(rxo);
1726
1727                 curr_frag_len = min(remaining, rx_frag_size);
1728
1729                 /* Coalesce all frags from the same physical page in one slot */
1730                 if (i == 0 || page_info->page_offset == 0) {
1731                         /* First frag or Fresh page */
1732                         j++;
1733                         skb_frag_set_page(skb, j, page_info->page);
1734                         skb_shinfo(skb)->frags[j].page_offset =
1735                                                         page_info->page_offset;
1736                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1737                 } else {
1738                         put_page(page_info->page);
1739                 }
1740                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1741                 skb->truesize += rx_frag_size;
1742                 remaining -= curr_frag_len;
1743                 memset(page_info, 0, sizeof(*page_info));
1744         }
1745         BUG_ON(j > MAX_SKB_FRAGS);
1746
1747         skb_shinfo(skb)->nr_frags = j + 1;
1748         skb->len = rxcp->pkt_size;
1749         skb->data_len = rxcp->pkt_size;
1750         skb->ip_summed = CHECKSUM_UNNECESSARY;
1751         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1752         if (adapter->netdev->features & NETIF_F_RXHASH)
1753                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1754
1755         skb->csum_level = rxcp->tunneled;
1756         skb_mark_napi_id(skb, napi);
1757
1758         if (rxcp->vlanf)
1759                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1760
1761         napi_gro_frags(napi);
1762 }
1763
1764 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1765                                  struct be_rx_compl_info *rxcp)
1766 {
1767         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1768         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1769         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1770         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1771         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1772         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1773         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1774         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1775         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1776         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1777         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1778         if (rxcp->vlanf) {
1779                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1780                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1781         }
1782         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1783         rxcp->tunneled =
1784                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1785 }
1786
1787 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1788                                  struct be_rx_compl_info *rxcp)
1789 {
1790         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1791         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1792         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1793         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1794         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1795         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1796         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1797         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1798         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1799         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1800         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1801         if (rxcp->vlanf) {
1802                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1803                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1804         }
1805         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1806         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1807 }
1808
1809 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1810 {
1811         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1812         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1813         struct be_adapter *adapter = rxo->adapter;
1814
1815         /* For checking the valid bit it is Ok to use either definition as the
1816          * valid bit is at the same position in both v0 and v1 Rx compl */
1817         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1818                 return NULL;
1819
1820         rmb();
1821         be_dws_le_to_cpu(compl, sizeof(*compl));
1822
1823         if (adapter->be3_native)
1824                 be_parse_rx_compl_v1(compl, rxcp);
1825         else
1826                 be_parse_rx_compl_v0(compl, rxcp);
1827
1828         if (rxcp->ip_frag)
1829                 rxcp->l4_csum = 0;
1830
1831         if (rxcp->vlanf) {
1832                 /* In QNQ modes, if qnq bit is not set, then the packet was
1833                  * tagged only with the transparent outer vlan-tag and must
1834                  * not be treated as a vlan packet by host
1835                  */
1836                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1837                         rxcp->vlanf = 0;
1838
1839                 if (!lancer_chip(adapter))
1840                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1841
1842                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1843                     !test_bit(rxcp->vlan_tag, adapter->vids))
1844                         rxcp->vlanf = 0;
1845         }
1846
1847         /* As the compl has been parsed, reset it; we wont touch it again */
1848         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1849
1850         queue_tail_inc(&rxo->cq);
1851         return rxcp;
1852 }
1853
1854 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1855 {
1856         u32 order = get_order(size);
1857
1858         if (order > 0)
1859                 gfp |= __GFP_COMP;
1860         return  alloc_pages(gfp, order);
1861 }
1862
1863 /*
1864  * Allocate a page, split it to fragments of size rx_frag_size and post as
1865  * receive buffers to BE
1866  */
1867 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1868 {
1869         struct be_adapter *adapter = rxo->adapter;
1870         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1871         struct be_queue_info *rxq = &rxo->q;
1872         struct page *pagep = NULL;
1873         struct device *dev = &adapter->pdev->dev;
1874         struct be_eth_rx_d *rxd;
1875         u64 page_dmaaddr = 0, frag_dmaaddr;
1876         u32 posted, page_offset = 0, notify = 0;
1877
1878         page_info = &rxo->page_info_tbl[rxq->head];
1879         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1880                 if (!pagep) {
1881                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1882                         if (unlikely(!pagep)) {
1883                                 rx_stats(rxo)->rx_post_fail++;
1884                                 break;
1885                         }
1886                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1887                                                     adapter->big_page_size,
1888                                                     DMA_FROM_DEVICE);
1889                         if (dma_mapping_error(dev, page_dmaaddr)) {
1890                                 put_page(pagep);
1891                                 pagep = NULL;
1892                                 adapter->drv_stats.dma_map_errors++;
1893                                 break;
1894                         }
1895                         page_offset = 0;
1896                 } else {
1897                         get_page(pagep);
1898                         page_offset += rx_frag_size;
1899                 }
1900                 page_info->page_offset = page_offset;
1901                 page_info->page = pagep;
1902
1903                 rxd = queue_head_node(rxq);
1904                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1905                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1906                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1907
1908                 /* Any space left in the current big page for another frag? */
1909                 if ((page_offset + rx_frag_size + rx_frag_size) >
1910                                         adapter->big_page_size) {
1911                         pagep = NULL;
1912                         page_info->last_frag = true;
1913                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1914                 } else {
1915                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1916                 }
1917
1918                 prev_page_info = page_info;
1919                 queue_head_inc(rxq);
1920                 page_info = &rxo->page_info_tbl[rxq->head];
1921         }
1922
1923         /* Mark the last frag of a page when we break out of the above loop
1924          * with no more slots available in the RXQ
1925          */
1926         if (pagep) {
1927                 prev_page_info->last_frag = true;
1928                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1929         }
1930
1931         if (posted) {
1932                 atomic_add(posted, &rxq->used);
1933                 if (rxo->rx_post_starved)
1934                         rxo->rx_post_starved = false;
1935                 do {
1936                         notify = min(256u, posted);
1937                         be_rxq_notify(adapter, rxq->id, notify);
1938                         posted -= notify;
1939                 } while (posted);
1940         } else if (atomic_read(&rxq->used) == 0) {
1941                 /* Let be_worker replenish when memory is available */
1942                 rxo->rx_post_starved = true;
1943         }
1944 }
1945
1946 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1947 {
1948         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1949
1950         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1951                 return NULL;
1952
1953         rmb();
1954         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1955
1956         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1957
1958         queue_tail_inc(tx_cq);
1959         return txcp;
1960 }
1961
1962 static u16 be_tx_compl_process(struct be_adapter *adapter,
1963                                struct be_tx_obj *txo, u16 last_index)
1964 {
1965         struct sk_buff **sent_skbs = txo->sent_skb_list;
1966         struct be_queue_info *txq = &txo->q;
1967         u16 frag_index, num_wrbs = 0;
1968         struct sk_buff *skb = NULL;
1969         bool unmap_skb_hdr = false;
1970         struct be_eth_wrb *wrb;
1971
1972         do {
1973                 if (sent_skbs[txq->tail]) {
1974                         /* Free skb from prev req */
1975                         if (skb)
1976                                 dev_consume_skb_any(skb);
1977                         skb = sent_skbs[txq->tail];
1978                         sent_skbs[txq->tail] = NULL;
1979                         queue_tail_inc(txq);  /* skip hdr wrb */
1980                         num_wrbs++;
1981                         unmap_skb_hdr = true;
1982                 }
1983                 wrb = queue_tail_node(txq);
1984                 frag_index = txq->tail;
1985                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1986                               (unmap_skb_hdr && skb_headlen(skb)));
1987                 unmap_skb_hdr = false;
1988                 queue_tail_inc(txq);
1989                 num_wrbs++;
1990         } while (frag_index != last_index);
1991         dev_consume_skb_any(skb);
1992
1993         return num_wrbs;
1994 }
1995
1996 /* Return the number of events in the event queue */
1997 static inline int events_get(struct be_eq_obj *eqo)
1998 {
1999         struct be_eq_entry *eqe;
2000         int num = 0;
2001
2002         do {
2003                 eqe = queue_tail_node(&eqo->q);
2004                 if (eqe->evt == 0)
2005                         break;
2006
2007                 rmb();
2008                 eqe->evt = 0;
2009                 num++;
2010                 queue_tail_inc(&eqo->q);
2011         } while (true);
2012
2013         return num;
2014 }
2015
2016 /* Leaves the EQ is disarmed state */
2017 static void be_eq_clean(struct be_eq_obj *eqo)
2018 {
2019         int num = events_get(eqo);
2020
2021         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2022 }
2023
2024 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2025 {
2026         struct be_rx_page_info *page_info;
2027         struct be_queue_info *rxq = &rxo->q;
2028         struct be_queue_info *rx_cq = &rxo->cq;
2029         struct be_rx_compl_info *rxcp;
2030         struct be_adapter *adapter = rxo->adapter;
2031         int flush_wait = 0;
2032
2033         /* Consume pending rx completions.
2034          * Wait for the flush completion (identified by zero num_rcvd)
2035          * to arrive. Notify CQ even when there are no more CQ entries
2036          * for HW to flush partially coalesced CQ entries.
2037          * In Lancer, there is no need to wait for flush compl.
2038          */
2039         for (;;) {
2040                 rxcp = be_rx_compl_get(rxo);
2041                 if (!rxcp) {
2042                         if (lancer_chip(adapter))
2043                                 break;
2044
2045                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2046                                 dev_warn(&adapter->pdev->dev,
2047                                          "did not receive flush compl\n");
2048                                 break;
2049                         }
2050                         be_cq_notify(adapter, rx_cq->id, true, 0);
2051                         mdelay(1);
2052                 } else {
2053                         be_rx_compl_discard(rxo, rxcp);
2054                         be_cq_notify(adapter, rx_cq->id, false, 1);
2055                         if (rxcp->num_rcvd == 0)
2056                                 break;
2057                 }
2058         }
2059
2060         /* After cleanup, leave the CQ in unarmed state */
2061         be_cq_notify(adapter, rx_cq->id, false, 0);
2062
2063         /* Then free posted rx buffers that were not used */
2064         while (atomic_read(&rxq->used) > 0) {
2065                 page_info = get_rx_page_info(rxo);
2066                 put_page(page_info->page);
2067                 memset(page_info, 0, sizeof(*page_info));
2068         }
2069         BUG_ON(atomic_read(&rxq->used));
2070         rxq->tail = 0;
2071         rxq->head = 0;
2072 }
2073
2074 static void be_tx_compl_clean(struct be_adapter *adapter)
2075 {
2076         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2077         struct device *dev = &adapter->pdev->dev;
2078         struct be_tx_obj *txo;
2079         struct be_queue_info *txq;
2080         struct be_eth_tx_compl *txcp;
2081         int i, pending_txqs;
2082
2083         /* Stop polling for compls when HW has been silent for 10ms */
2084         do {
2085                 pending_txqs = adapter->num_tx_qs;
2086
2087                 for_all_tx_queues(adapter, txo, i) {
2088                         cmpl = 0;
2089                         num_wrbs = 0;
2090                         txq = &txo->q;
2091                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2092                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2093                                 num_wrbs += be_tx_compl_process(adapter, txo,
2094                                                                 end_idx);
2095                                 cmpl++;
2096                         }
2097                         if (cmpl) {
2098                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2099                                 atomic_sub(num_wrbs, &txq->used);
2100                                 timeo = 0;
2101                         }
2102                         if (atomic_read(&txq->used) == txo->pend_wrb_cnt)
2103                                 pending_txqs--;
2104                 }
2105
2106                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2107                         break;
2108
2109                 mdelay(1);
2110         } while (true);
2111
2112         /* Free enqueued TX that was never notified to HW */
2113         for_all_tx_queues(adapter, txo, i) {
2114                 txq = &txo->q;
2115
2116                 if (atomic_read(&txq->used)) {
2117                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2118                                  i, atomic_read(&txq->used));
2119                         notified_idx = txq->tail;
2120                         end_idx = txq->tail;
2121                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2122                                   txq->len);
2123                         /* Use the tx-compl process logic to handle requests
2124                          * that were not sent to the HW.
2125                          */
2126                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2127                         atomic_sub(num_wrbs, &txq->used);
2128                         BUG_ON(atomic_read(&txq->used));
2129                         txo->pend_wrb_cnt = 0;
2130                         /* Since hw was never notified of these requests,
2131                          * reset TXQ indices
2132                          */
2133                         txq->head = notified_idx;
2134                         txq->tail = notified_idx;
2135                 }
2136         }
2137 }
2138
2139 static void be_evt_queues_destroy(struct be_adapter *adapter)
2140 {
2141         struct be_eq_obj *eqo;
2142         int i;
2143
2144         for_all_evt_queues(adapter, eqo, i) {
2145                 if (eqo->q.created) {
2146                         be_eq_clean(eqo);
2147                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2148                         napi_hash_del(&eqo->napi);
2149                         netif_napi_del(&eqo->napi);
2150                 }
2151                 be_queue_free(adapter, &eqo->q);
2152         }
2153 }
2154
2155 static int be_evt_queues_create(struct be_adapter *adapter)
2156 {
2157         struct be_queue_info *eq;
2158         struct be_eq_obj *eqo;
2159         struct be_aic_obj *aic;
2160         int i, rc;
2161
2162         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2163                                     adapter->cfg_num_qs);
2164
2165         for_all_evt_queues(adapter, eqo, i) {
2166                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2167                                BE_NAPI_WEIGHT);
2168                 napi_hash_add(&eqo->napi);
2169                 aic = &adapter->aic_obj[i];
2170                 eqo->adapter = adapter;
2171                 eqo->idx = i;
2172                 aic->max_eqd = BE_MAX_EQD;
2173                 aic->enable = true;
2174
2175                 eq = &eqo->q;
2176                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2177                                     sizeof(struct be_eq_entry));
2178                 if (rc)
2179                         return rc;
2180
2181                 rc = be_cmd_eq_create(adapter, eqo);
2182                 if (rc)
2183                         return rc;
2184         }
2185         return 0;
2186 }
2187
2188 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2189 {
2190         struct be_queue_info *q;
2191
2192         q = &adapter->mcc_obj.q;
2193         if (q->created)
2194                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2195         be_queue_free(adapter, q);
2196
2197         q = &adapter->mcc_obj.cq;
2198         if (q->created)
2199                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2200         be_queue_free(adapter, q);
2201 }
2202
2203 /* Must be called only after TX qs are created as MCC shares TX EQ */
2204 static int be_mcc_queues_create(struct be_adapter *adapter)
2205 {
2206         struct be_queue_info *q, *cq;
2207
2208         cq = &adapter->mcc_obj.cq;
2209         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2210                            sizeof(struct be_mcc_compl)))
2211                 goto err;
2212
2213         /* Use the default EQ for MCC completions */
2214         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2215                 goto mcc_cq_free;
2216
2217         q = &adapter->mcc_obj.q;
2218         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2219                 goto mcc_cq_destroy;
2220
2221         if (be_cmd_mccq_create(adapter, q, cq))
2222                 goto mcc_q_free;
2223
2224         return 0;
2225
2226 mcc_q_free:
2227         be_queue_free(adapter, q);
2228 mcc_cq_destroy:
2229         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2230 mcc_cq_free:
2231         be_queue_free(adapter, cq);
2232 err:
2233         return -1;
2234 }
2235
2236 static void be_tx_queues_destroy(struct be_adapter *adapter)
2237 {
2238         struct be_queue_info *q;
2239         struct be_tx_obj *txo;
2240         u8 i;
2241
2242         for_all_tx_queues(adapter, txo, i) {
2243                 q = &txo->q;
2244                 if (q->created)
2245                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2246                 be_queue_free(adapter, q);
2247
2248                 q = &txo->cq;
2249                 if (q->created)
2250                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2251                 be_queue_free(adapter, q);
2252         }
2253 }
2254
2255 static int be_tx_qs_create(struct be_adapter *adapter)
2256 {
2257         struct be_queue_info *cq, *eq;
2258         struct be_tx_obj *txo;
2259         int status, i;
2260
2261         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2262
2263         for_all_tx_queues(adapter, txo, i) {
2264                 cq = &txo->cq;
2265                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2266                                         sizeof(struct be_eth_tx_compl));
2267                 if (status)
2268                         return status;
2269
2270                 u64_stats_init(&txo->stats.sync);
2271                 u64_stats_init(&txo->stats.sync_compl);
2272
2273                 /* If num_evt_qs is less than num_tx_qs, then more than
2274                  * one txq share an eq
2275                  */
2276                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2277                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2278                 if (status)
2279                         return status;
2280
2281                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2282                                         sizeof(struct be_eth_wrb));
2283                 if (status)
2284                         return status;
2285
2286                 status = be_cmd_txq_create(adapter, txo);
2287                 if (status)
2288                         return status;
2289         }
2290
2291         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2292                  adapter->num_tx_qs);
2293         return 0;
2294 }
2295
2296 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2297 {
2298         struct be_queue_info *q;
2299         struct be_rx_obj *rxo;
2300         int i;
2301
2302         for_all_rx_queues(adapter, rxo, i) {
2303                 q = &rxo->cq;
2304                 if (q->created)
2305                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2306                 be_queue_free(adapter, q);
2307         }
2308 }
2309
2310 static int be_rx_cqs_create(struct be_adapter *adapter)
2311 {
2312         struct be_queue_info *eq, *cq;
2313         struct be_rx_obj *rxo;
2314         int rc, i;
2315
2316         /* We can create as many RSS rings as there are EQs. */
2317         adapter->num_rx_qs = adapter->num_evt_qs;
2318
2319         /* We'll use RSS only if atleast 2 RSS rings are supported.
2320          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2321          */
2322         if (adapter->num_rx_qs > 1)
2323                 adapter->num_rx_qs++;
2324
2325         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2326         for_all_rx_queues(adapter, rxo, i) {
2327                 rxo->adapter = adapter;
2328                 cq = &rxo->cq;
2329                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2330                                     sizeof(struct be_eth_rx_compl));
2331                 if (rc)
2332                         return rc;
2333
2334                 u64_stats_init(&rxo->stats.sync);
2335                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2336                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2337                 if (rc)
2338                         return rc;
2339         }
2340
2341         dev_info(&adapter->pdev->dev,
2342                  "created %d RSS queue(s) and 1 default RX queue\n",
2343                  adapter->num_rx_qs - 1);
2344         return 0;
2345 }
2346
2347 static irqreturn_t be_intx(int irq, void *dev)
2348 {
2349         struct be_eq_obj *eqo = dev;
2350         struct be_adapter *adapter = eqo->adapter;
2351         int num_evts = 0;
2352
2353         /* IRQ is not expected when NAPI is scheduled as the EQ
2354          * will not be armed.
2355          * But, this can happen on Lancer INTx where it takes
2356          * a while to de-assert INTx or in BE2 where occasionaly
2357          * an interrupt may be raised even when EQ is unarmed.
2358          * If NAPI is already scheduled, then counting & notifying
2359          * events will orphan them.
2360          */
2361         if (napi_schedule_prep(&eqo->napi)) {
2362                 num_evts = events_get(eqo);
2363                 __napi_schedule(&eqo->napi);
2364                 if (num_evts)
2365                         eqo->spurious_intr = 0;
2366         }
2367         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2368
2369         /* Return IRQ_HANDLED only for the the first spurious intr
2370          * after a valid intr to stop the kernel from branding
2371          * this irq as a bad one!
2372          */
2373         if (num_evts || eqo->spurious_intr++ == 0)
2374                 return IRQ_HANDLED;
2375         else
2376                 return IRQ_NONE;
2377 }
2378
2379 static irqreturn_t be_msix(int irq, void *dev)
2380 {
2381         struct be_eq_obj *eqo = dev;
2382
2383         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2384         napi_schedule(&eqo->napi);
2385         return IRQ_HANDLED;
2386 }
2387
2388 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2389 {
2390         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2391 }
2392
2393 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2394                          int budget, int polling)
2395 {
2396         struct be_adapter *adapter = rxo->adapter;
2397         struct be_queue_info *rx_cq = &rxo->cq;
2398         struct be_rx_compl_info *rxcp;
2399         u32 work_done;
2400         u32 frags_consumed = 0;
2401
2402         for (work_done = 0; work_done < budget; work_done++) {
2403                 rxcp = be_rx_compl_get(rxo);
2404                 if (!rxcp)
2405                         break;
2406
2407                 /* Is it a flush compl that has no data */
2408                 if (unlikely(rxcp->num_rcvd == 0))
2409                         goto loop_continue;
2410
2411                 /* Discard compl with partial DMA Lancer B0 */
2412                 if (unlikely(!rxcp->pkt_size)) {
2413                         be_rx_compl_discard(rxo, rxcp);
2414                         goto loop_continue;
2415                 }
2416
2417                 /* On BE drop pkts that arrive due to imperfect filtering in
2418                  * promiscuous mode on some skews
2419                  */
2420                 if (unlikely(rxcp->port != adapter->port_num &&
2421                              !lancer_chip(adapter))) {
2422                         be_rx_compl_discard(rxo, rxcp);
2423                         goto loop_continue;
2424                 }
2425
2426                 /* Don't do gro when we're busy_polling */
2427                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2428                         be_rx_compl_process_gro(rxo, napi, rxcp);
2429                 else
2430                         be_rx_compl_process(rxo, napi, rxcp);
2431
2432 loop_continue:
2433                 frags_consumed += rxcp->num_rcvd;
2434                 be_rx_stats_update(rxo, rxcp);
2435         }
2436
2437         if (work_done) {
2438                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2439
2440                 /* When an rx-obj gets into post_starved state, just
2441                  * let be_worker do the posting.
2442                  */
2443                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2444                     !rxo->rx_post_starved)
2445                         be_post_rx_frags(rxo, GFP_ATOMIC,
2446                                          max_t(u32, MAX_RX_POST,
2447                                                frags_consumed));
2448         }
2449
2450         return work_done;
2451 }
2452
2453 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2454 {
2455         switch (status) {
2456         case BE_TX_COMP_HDR_PARSE_ERR:
2457                 tx_stats(txo)->tx_hdr_parse_err++;
2458                 break;
2459         case BE_TX_COMP_NDMA_ERR:
2460                 tx_stats(txo)->tx_dma_err++;
2461                 break;
2462         case BE_TX_COMP_ACL_ERR:
2463                 tx_stats(txo)->tx_spoof_check_err++;
2464                 break;
2465         }
2466 }
2467
2468 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2469 {
2470         switch (status) {
2471         case LANCER_TX_COMP_LSO_ERR:
2472                 tx_stats(txo)->tx_tso_err++;
2473                 break;
2474         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2475         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2476                 tx_stats(txo)->tx_spoof_check_err++;
2477                 break;
2478         case LANCER_TX_COMP_QINQ_ERR:
2479                 tx_stats(txo)->tx_qinq_err++;
2480                 break;
2481         case LANCER_TX_COMP_PARITY_ERR:
2482                 tx_stats(txo)->tx_internal_parity_err++;
2483                 break;
2484         case LANCER_TX_COMP_DMA_ERR:
2485                 tx_stats(txo)->tx_dma_err++;
2486                 break;
2487         }
2488 }
2489
2490 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2491                           int idx)
2492 {
2493         struct be_eth_tx_compl *txcp;
2494         int num_wrbs = 0, work_done = 0;
2495         u32 compl_status;
2496         u16 last_idx;
2497
2498         while ((txcp = be_tx_compl_get(&txo->cq))) {
2499                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2500                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2501                 work_done++;
2502
2503                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2504                 if (compl_status) {
2505                         if (lancer_chip(adapter))
2506                                 lancer_update_tx_err(txo, compl_status);
2507                         else
2508                                 be_update_tx_err(txo, compl_status);
2509                 }
2510         }
2511
2512         if (work_done) {
2513                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2514                 atomic_sub(num_wrbs, &txo->q.used);
2515
2516                 /* As Tx wrbs have been freed up, wake up netdev queue
2517                  * if it was stopped due to lack of tx wrbs.  */
2518                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2519                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2520                         netif_wake_subqueue(adapter->netdev, idx);
2521                 }
2522
2523                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2524                 tx_stats(txo)->tx_compl += work_done;
2525                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2526         }
2527 }
2528
2529 int be_poll(struct napi_struct *napi, int budget)
2530 {
2531         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2532         struct be_adapter *adapter = eqo->adapter;
2533         int max_work = 0, work, i, num_evts;
2534         struct be_rx_obj *rxo;
2535         struct be_tx_obj *txo;
2536
2537         num_evts = events_get(eqo);
2538
2539         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2540                 be_process_tx(adapter, txo, i);
2541
2542         if (be_lock_napi(eqo)) {
2543                 /* This loop will iterate twice for EQ0 in which
2544                  * completions of the last RXQ (default one) are also processed
2545                  * For other EQs the loop iterates only once
2546                  */
2547                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2548                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2549                         max_work = max(work, max_work);
2550                 }
2551                 be_unlock_napi(eqo);
2552         } else {
2553                 max_work = budget;
2554         }
2555
2556         if (is_mcc_eqo(eqo))
2557                 be_process_mcc(adapter);
2558
2559         if (max_work < budget) {
2560                 napi_complete(napi);
2561                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2562         } else {
2563                 /* As we'll continue in polling mode, count and clear events */
2564                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2565         }
2566         return max_work;
2567 }
2568
2569 #ifdef CONFIG_NET_RX_BUSY_POLL
2570 static int be_busy_poll(struct napi_struct *napi)
2571 {
2572         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2573         struct be_adapter *adapter = eqo->adapter;
2574         struct be_rx_obj *rxo;
2575         int i, work = 0;
2576
2577         if (!be_lock_busy_poll(eqo))
2578                 return LL_FLUSH_BUSY;
2579
2580         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2581                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2582                 if (work)
2583                         break;
2584         }
2585
2586         be_unlock_busy_poll(eqo);
2587         return work;
2588 }
2589 #endif
2590
2591 void be_detect_error(struct be_adapter *adapter)
2592 {
2593         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2594         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2595         u32 i;
2596         bool error_detected = false;
2597         struct device *dev = &adapter->pdev->dev;
2598         struct net_device *netdev = adapter->netdev;
2599
2600         if (be_hw_error(adapter))
2601                 return;
2602
2603         if (lancer_chip(adapter)) {
2604                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2605                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2606                         sliport_err1 = ioread32(adapter->db +
2607                                                 SLIPORT_ERROR1_OFFSET);
2608                         sliport_err2 = ioread32(adapter->db +
2609                                                 SLIPORT_ERROR2_OFFSET);
2610                         adapter->hw_error = true;
2611                         /* Do not log error messages if its a FW reset */
2612                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2613                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2614                                 dev_info(dev, "Firmware update in progress\n");
2615                         } else {
2616                                 error_detected = true;
2617                                 dev_err(dev, "Error detected in the card\n");
2618                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2619                                         sliport_status);
2620                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2621                                         sliport_err1);
2622                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2623                                         sliport_err2);
2624                         }
2625                 }
2626         } else {
2627                 pci_read_config_dword(adapter->pdev,
2628                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2629                 pci_read_config_dword(adapter->pdev,
2630                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2631                 pci_read_config_dword(adapter->pdev,
2632                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2633                 pci_read_config_dword(adapter->pdev,
2634                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2635
2636                 ue_lo = (ue_lo & ~ue_lo_mask);
2637                 ue_hi = (ue_hi & ~ue_hi_mask);
2638
2639                 /* On certain platforms BE hardware can indicate spurious UEs.
2640                  * Allow HW to stop working completely in case of a real UE.
2641                  * Hence not setting the hw_error for UE detection.
2642                  */
2643
2644                 if (ue_lo || ue_hi) {
2645                         error_detected = true;
2646                         dev_err(dev,
2647                                 "Unrecoverable Error detected in the adapter");
2648                         dev_err(dev, "Please reboot server to recover");
2649                         if (skyhawk_chip(adapter))
2650                                 adapter->hw_error = true;
2651                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2652                                 if (ue_lo & 1)
2653                                         dev_err(dev, "UE: %s bit set\n",
2654                                                 ue_status_low_desc[i]);
2655                         }
2656                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2657                                 if (ue_hi & 1)
2658                                         dev_err(dev, "UE: %s bit set\n",
2659                                                 ue_status_hi_desc[i]);
2660                         }
2661                 }
2662         }
2663         if (error_detected)
2664                 netif_carrier_off(netdev);
2665 }
2666
2667 static void be_msix_disable(struct be_adapter *adapter)
2668 {
2669         if (msix_enabled(adapter)) {
2670                 pci_disable_msix(adapter->pdev);
2671                 adapter->num_msix_vec = 0;
2672                 adapter->num_msix_roce_vec = 0;
2673         }
2674 }
2675
2676 static int be_msix_enable(struct be_adapter *adapter)
2677 {
2678         int i, num_vec;
2679         struct device *dev = &adapter->pdev->dev;
2680
2681         /* If RoCE is supported, program the max number of NIC vectors that
2682          * may be configured via set-channels, along with vectors needed for
2683          * RoCe. Else, just program the number we'll use initially.
2684          */
2685         if (be_roce_supported(adapter))
2686                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2687                                 2 * num_online_cpus());
2688         else
2689                 num_vec = adapter->cfg_num_qs;
2690
2691         for (i = 0; i < num_vec; i++)
2692                 adapter->msix_entries[i].entry = i;
2693
2694         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2695                                         MIN_MSIX_VECTORS, num_vec);
2696         if (num_vec < 0)
2697                 goto fail;
2698
2699         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2700                 adapter->num_msix_roce_vec = num_vec / 2;
2701                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2702                          adapter->num_msix_roce_vec);
2703         }
2704
2705         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2706
2707         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2708                  adapter->num_msix_vec);
2709         return 0;
2710
2711 fail:
2712         dev_warn(dev, "MSIx enable failed\n");
2713
2714         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2715         if (!be_physfn(adapter))
2716                 return num_vec;
2717         return 0;
2718 }
2719
2720 static inline int be_msix_vec_get(struct be_adapter *adapter,
2721                                   struct be_eq_obj *eqo)
2722 {
2723         return adapter->msix_entries[eqo->msix_idx].vector;
2724 }
2725
2726 static int be_msix_register(struct be_adapter *adapter)
2727 {
2728         struct net_device *netdev = adapter->netdev;
2729         struct be_eq_obj *eqo;
2730         int status, i, vec;
2731
2732         for_all_evt_queues(adapter, eqo, i) {
2733                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2734                 vec = be_msix_vec_get(adapter, eqo);
2735                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2736                 if (status)
2737                         goto err_msix;
2738         }
2739
2740         return 0;
2741 err_msix:
2742         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2743                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2744         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2745                  status);
2746         be_msix_disable(adapter);
2747         return status;
2748 }
2749
2750 static int be_irq_register(struct be_adapter *adapter)
2751 {
2752         struct net_device *netdev = adapter->netdev;
2753         int status;
2754
2755         if (msix_enabled(adapter)) {
2756                 status = be_msix_register(adapter);
2757                 if (status == 0)
2758                         goto done;
2759                 /* INTx is not supported for VF */
2760                 if (!be_physfn(adapter))
2761                         return status;
2762         }
2763
2764         /* INTx: only the first EQ is used */
2765         netdev->irq = adapter->pdev->irq;
2766         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2767                              &adapter->eq_obj[0]);
2768         if (status) {
2769                 dev_err(&adapter->pdev->dev,
2770                         "INTx request IRQ failed - err %d\n", status);
2771                 return status;
2772         }
2773 done:
2774         adapter->isr_registered = true;
2775         return 0;
2776 }
2777
2778 static void be_irq_unregister(struct be_adapter *adapter)
2779 {
2780         struct net_device *netdev = adapter->netdev;
2781         struct be_eq_obj *eqo;
2782         int i;
2783
2784         if (!adapter->isr_registered)
2785                 return;
2786
2787         /* INTx */
2788         if (!msix_enabled(adapter)) {
2789                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2790                 goto done;
2791         }
2792
2793         /* MSIx */
2794         for_all_evt_queues(adapter, eqo, i)
2795                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2796
2797 done:
2798         adapter->isr_registered = false;
2799 }
2800
2801 static void be_rx_qs_destroy(struct be_adapter *adapter)
2802 {
2803         struct be_queue_info *q;
2804         struct be_rx_obj *rxo;
2805         int i;
2806
2807         for_all_rx_queues(adapter, rxo, i) {
2808                 q = &rxo->q;
2809                 if (q->created) {
2810                         be_cmd_rxq_destroy(adapter, q);
2811                         be_rx_cq_clean(rxo);
2812                 }
2813                 be_queue_free(adapter, q);
2814         }
2815 }
2816
2817 static int be_close(struct net_device *netdev)
2818 {
2819         struct be_adapter *adapter = netdev_priv(netdev);
2820         struct be_eq_obj *eqo;
2821         int i;
2822
2823         /* This protection is needed as be_close() may be called even when the
2824          * adapter is in cleared state (after eeh perm failure)
2825          */
2826         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2827                 return 0;
2828
2829         be_roce_dev_close(adapter);
2830
2831         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2832                 for_all_evt_queues(adapter, eqo, i) {
2833                         napi_disable(&eqo->napi);
2834                         be_disable_busy_poll(eqo);
2835                 }
2836                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2837         }
2838
2839         be_async_mcc_disable(adapter);
2840
2841         /* Wait for all pending tx completions to arrive so that
2842          * all tx skbs are freed.
2843          */
2844         netif_tx_disable(netdev);
2845         be_tx_compl_clean(adapter);
2846
2847         be_rx_qs_destroy(adapter);
2848
2849         for (i = 1; i < (adapter->uc_macs + 1); i++)
2850                 be_cmd_pmac_del(adapter, adapter->if_handle,
2851                                 adapter->pmac_id[i], 0);
2852         adapter->uc_macs = 0;
2853
2854         for_all_evt_queues(adapter, eqo, i) {
2855                 if (msix_enabled(adapter))
2856                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2857                 else
2858                         synchronize_irq(netdev->irq);
2859                 be_eq_clean(eqo);
2860         }
2861
2862         be_irq_unregister(adapter);
2863
2864         return 0;
2865 }
2866
2867 static int be_rx_qs_create(struct be_adapter *adapter)
2868 {
2869         struct rss_info *rss = &adapter->rss_info;
2870         u8 rss_key[RSS_HASH_KEY_LEN];
2871         struct be_rx_obj *rxo;
2872         int rc, i, j;
2873
2874         for_all_rx_queues(adapter, rxo, i) {
2875                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2876                                     sizeof(struct be_eth_rx_d));
2877                 if (rc)
2878                         return rc;
2879         }
2880
2881         /* The FW would like the default RXQ to be created first */
2882         rxo = default_rxo(adapter);
2883         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2884                                adapter->if_handle, false, &rxo->rss_id);
2885         if (rc)
2886                 return rc;
2887
2888         for_all_rss_queues(adapter, rxo, i) {
2889                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2890                                        rx_frag_size, adapter->if_handle,
2891                                        true, &rxo->rss_id);
2892                 if (rc)
2893                         return rc;
2894         }
2895
2896         if (be_multi_rxq(adapter)) {
2897                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2898                         j += adapter->num_rx_qs - 1) {
2899                         for_all_rss_queues(adapter, rxo, i) {
2900                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2901                                         break;
2902                                 rss->rsstable[j + i] = rxo->rss_id;
2903                                 rss->rss_queue[j + i] = i;
2904                         }
2905                 }
2906                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2907                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2908
2909                 if (!BEx_chip(adapter))
2910                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2911                                 RSS_ENABLE_UDP_IPV6;
2912         } else {
2913                 /* Disable RSS, if only default RX Q is created */
2914                 rss->rss_flags = RSS_ENABLE_NONE;
2915         }
2916
2917         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
2918         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2919                                128, rss_key);
2920         if (rc) {
2921                 rss->rss_flags = RSS_ENABLE_NONE;
2922                 return rc;
2923         }
2924
2925         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
2926
2927         /* First time posting */
2928         for_all_rx_queues(adapter, rxo, i)
2929                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
2930         return 0;
2931 }
2932
2933 static int be_open(struct net_device *netdev)
2934 {
2935         struct be_adapter *adapter = netdev_priv(netdev);
2936         struct be_eq_obj *eqo;
2937         struct be_rx_obj *rxo;
2938         struct be_tx_obj *txo;
2939         u8 link_status;
2940         int status, i;
2941
2942         status = be_rx_qs_create(adapter);
2943         if (status)
2944                 goto err;
2945
2946         status = be_irq_register(adapter);
2947         if (status)
2948                 goto err;
2949
2950         for_all_rx_queues(adapter, rxo, i)
2951                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2952
2953         for_all_tx_queues(adapter, txo, i)
2954                 be_cq_notify(adapter, txo->cq.id, true, 0);
2955
2956         be_async_mcc_enable(adapter);
2957
2958         for_all_evt_queues(adapter, eqo, i) {
2959                 napi_enable(&eqo->napi);
2960                 be_enable_busy_poll(eqo);
2961                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2962         }
2963         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2964
2965         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2966         if (!status)
2967                 be_link_status_update(adapter, link_status);
2968
2969         netif_tx_start_all_queues(netdev);
2970         be_roce_dev_open(adapter);
2971
2972 #ifdef CONFIG_BE2NET_VXLAN
2973         if (skyhawk_chip(adapter))
2974                 vxlan_get_rx_port(netdev);
2975 #endif
2976
2977         return 0;
2978 err:
2979         be_close(adapter->netdev);
2980         return -EIO;
2981 }
2982
2983 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2984 {
2985         struct be_dma_mem cmd;
2986         int status = 0;
2987         u8 mac[ETH_ALEN];
2988
2989         memset(mac, 0, ETH_ALEN);
2990
2991         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2992         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2993                                      GFP_KERNEL);
2994         if (!cmd.va)
2995                 return -ENOMEM;
2996
2997         if (enable) {
2998                 status = pci_write_config_dword(adapter->pdev,
2999                                                 PCICFG_PM_CONTROL_OFFSET,
3000                                                 PCICFG_PM_CONTROL_MASK);
3001                 if (status) {
3002                         dev_err(&adapter->pdev->dev,
3003                                 "Could not enable Wake-on-lan\n");
3004                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3005                                           cmd.dma);
3006                         return status;
3007                 }
3008                 status = be_cmd_enable_magic_wol(adapter,
3009                                                  adapter->netdev->dev_addr,
3010                                                  &cmd);
3011                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3012                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3013         } else {
3014                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3015                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3016                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3017         }
3018
3019         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3020         return status;
3021 }
3022
3023 /*
3024  * Generate a seed MAC address from the PF MAC Address using jhash.
3025  * MAC Address for VFs are assigned incrementally starting from the seed.
3026  * These addresses are programmed in the ASIC by the PF and the VF driver
3027  * queries for the MAC address during its probe.
3028  */
3029 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3030 {
3031         u32 vf;
3032         int status = 0;
3033         u8 mac[ETH_ALEN];
3034         struct be_vf_cfg *vf_cfg;
3035
3036         be_vf_eth_addr_generate(adapter, mac);
3037
3038         for_all_vfs(adapter, vf_cfg, vf) {
3039                 if (BEx_chip(adapter))
3040                         status = be_cmd_pmac_add(adapter, mac,
3041                                                  vf_cfg->if_handle,
3042                                                  &vf_cfg->pmac_id, vf + 1);
3043                 else
3044                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3045                                                 vf + 1);
3046
3047                 if (status)
3048                         dev_err(&adapter->pdev->dev,
3049                                 "Mac address assignment failed for VF %d\n",
3050                                 vf);
3051                 else
3052                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3053
3054                 mac[5] += 1;
3055         }
3056         return status;
3057 }
3058
3059 static int be_vfs_mac_query(struct be_adapter *adapter)
3060 {
3061         int status, vf;
3062         u8 mac[ETH_ALEN];
3063         struct be_vf_cfg *vf_cfg;
3064
3065         for_all_vfs(adapter, vf_cfg, vf) {
3066                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3067                                                mac, vf_cfg->if_handle,
3068                                                false, vf+1);
3069                 if (status)
3070                         return status;
3071                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3072         }
3073         return 0;
3074 }
3075
3076 static void be_vf_clear(struct be_adapter *adapter)
3077 {
3078         struct be_vf_cfg *vf_cfg;
3079         u32 vf;
3080
3081         if (pci_vfs_assigned(adapter->pdev)) {
3082                 dev_warn(&adapter->pdev->dev,
3083                          "VFs are assigned to VMs: not disabling VFs\n");
3084                 goto done;
3085         }
3086
3087         pci_disable_sriov(adapter->pdev);
3088
3089         for_all_vfs(adapter, vf_cfg, vf) {
3090                 if (BEx_chip(adapter))
3091                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3092                                         vf_cfg->pmac_id, vf + 1);
3093                 else
3094                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3095                                        vf + 1);
3096
3097                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3098         }
3099 done:
3100         kfree(adapter->vf_cfg);
3101         adapter->num_vfs = 0;
3102         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3103 }
3104
3105 static void be_clear_queues(struct be_adapter *adapter)
3106 {
3107         be_mcc_queues_destroy(adapter);
3108         be_rx_cqs_destroy(adapter);
3109         be_tx_queues_destroy(adapter);
3110         be_evt_queues_destroy(adapter);
3111 }
3112
3113 static void be_cancel_worker(struct be_adapter *adapter)
3114 {
3115         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3116                 cancel_delayed_work_sync(&adapter->work);
3117                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3118         }
3119 }
3120
3121 static void be_mac_clear(struct be_adapter *adapter)
3122 {
3123         int i;
3124
3125         if (adapter->pmac_id) {
3126                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3127                         be_cmd_pmac_del(adapter, adapter->if_handle,
3128                                         adapter->pmac_id[i], 0);
3129                 adapter->uc_macs = 0;
3130
3131                 kfree(adapter->pmac_id);
3132                 adapter->pmac_id = NULL;
3133         }
3134 }
3135
3136 #ifdef CONFIG_BE2NET_VXLAN
3137 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3138 {
3139         struct net_device *netdev = adapter->netdev;
3140
3141         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3142                 be_cmd_manage_iface(adapter, adapter->if_handle,
3143                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3144
3145         if (adapter->vxlan_port)
3146                 be_cmd_set_vxlan_port(adapter, 0);
3147
3148         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3149         adapter->vxlan_port = 0;
3150
3151         netdev->hw_enc_features = 0;
3152         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3153         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3154 }
3155 #endif
3156
3157 static int be_clear(struct be_adapter *adapter)
3158 {
3159         be_cancel_worker(adapter);
3160
3161         if (sriov_enabled(adapter))
3162                 be_vf_clear(adapter);
3163
3164         /* Re-configure FW to distribute resources evenly across max-supported
3165          * number of VFs, only when VFs are not already enabled.
3166          */
3167         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3168                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3169                                         pci_sriov_get_totalvfs(adapter->pdev));
3170
3171 #ifdef CONFIG_BE2NET_VXLAN
3172         be_disable_vxlan_offloads(adapter);
3173 #endif
3174         /* delete the primary mac along with the uc-mac list */
3175         be_mac_clear(adapter);
3176
3177         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3178
3179         be_clear_queues(adapter);
3180
3181         be_msix_disable(adapter);
3182         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3183         return 0;
3184 }
3185
3186 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3187                         u32 cap_flags, u32 vf)
3188 {
3189         u32 en_flags;
3190         int status;
3191
3192         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3193                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3194                    BE_IF_FLAGS_RSS;
3195
3196         en_flags &= cap_flags;
3197
3198         status = be_cmd_if_create(adapter, cap_flags, en_flags,
3199                                   if_handle, vf);
3200
3201         return status;
3202 }
3203
3204 static int be_vfs_if_create(struct be_adapter *adapter)
3205 {
3206         struct be_resources res = {0};
3207         struct be_vf_cfg *vf_cfg;
3208         u32 cap_flags, vf;
3209         int status;
3210
3211         /* If a FW profile exists, then cap_flags are updated */
3212         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3213                     BE_IF_FLAGS_MULTICAST;
3214
3215         for_all_vfs(adapter, vf_cfg, vf) {
3216                 if (!BE3_chip(adapter)) {
3217                         status = be_cmd_get_profile_config(adapter, &res,
3218                                                            vf + 1);
3219                         if (!status)
3220                                 cap_flags = res.if_cap_flags;
3221                 }
3222
3223                 status = be_if_create(adapter, &vf_cfg->if_handle,
3224                                       cap_flags, vf + 1);
3225                 if (status)
3226                         return status;
3227         }
3228
3229         return 0;
3230 }
3231
3232 static int be_vf_setup_init(struct be_adapter *adapter)
3233 {
3234         struct be_vf_cfg *vf_cfg;
3235         int vf;
3236
3237         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3238                                   GFP_KERNEL);
3239         if (!adapter->vf_cfg)
3240                 return -ENOMEM;
3241
3242         for_all_vfs(adapter, vf_cfg, vf) {
3243                 vf_cfg->if_handle = -1;
3244                 vf_cfg->pmac_id = -1;
3245         }
3246         return 0;
3247 }
3248
3249 static int be_vf_setup(struct be_adapter *adapter)
3250 {
3251         struct device *dev = &adapter->pdev->dev;
3252         struct be_vf_cfg *vf_cfg;
3253         int status, old_vfs, vf;
3254         u32 privileges;
3255
3256         old_vfs = pci_num_vf(adapter->pdev);
3257
3258         status = be_vf_setup_init(adapter);
3259         if (status)
3260                 goto err;
3261
3262         if (old_vfs) {
3263                 for_all_vfs(adapter, vf_cfg, vf) {
3264                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3265                         if (status)
3266                                 goto err;
3267                 }
3268
3269                 status = be_vfs_mac_query(adapter);
3270                 if (status)
3271                         goto err;
3272         } else {
3273                 status = be_vfs_if_create(adapter);
3274                 if (status)
3275                         goto err;
3276
3277                 status = be_vf_eth_addr_config(adapter);
3278                 if (status)
3279                         goto err;
3280         }
3281
3282         for_all_vfs(adapter, vf_cfg, vf) {
3283                 /* Allow VFs to programs MAC/VLAN filters */
3284                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3285                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3286                         status = be_cmd_set_fn_privileges(adapter,
3287                                                           privileges |
3288                                                           BE_PRIV_FILTMGMT,
3289                                                           vf + 1);
3290                         if (!status)
3291                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3292                                          vf);
3293                 }
3294
3295                 /* Allow full available bandwidth */
3296                 if (!old_vfs)
3297                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3298
3299                 if (!old_vfs) {
3300                         be_cmd_enable_vf(adapter, vf + 1);
3301                         be_cmd_set_logical_link_config(adapter,
3302                                                        IFLA_VF_LINK_STATE_AUTO,
3303                                                        vf+1);
3304                 }
3305         }
3306
3307         if (!old_vfs) {
3308                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3309                 if (status) {
3310                         dev_err(dev, "SRIOV enable failed\n");
3311                         adapter->num_vfs = 0;
3312                         goto err;
3313                 }
3314         }
3315
3316         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3317         return 0;
3318 err:
3319         dev_err(dev, "VF setup failed\n");
3320         be_vf_clear(adapter);
3321         return status;
3322 }
3323
3324 /* Converting function_mode bits on BE3 to SH mc_type enums */
3325
3326 static u8 be_convert_mc_type(u32 function_mode)
3327 {
3328         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3329                 return vNIC1;
3330         else if (function_mode & QNQ_MODE)
3331                 return FLEX10;
3332         else if (function_mode & VNIC_MODE)
3333                 return vNIC2;
3334         else if (function_mode & UMC_ENABLED)
3335                 return UMC;
3336         else
3337                 return MC_NONE;
3338 }
3339
3340 /* On BE2/BE3 FW does not suggest the supported limits */
3341 static void BEx_get_resources(struct be_adapter *adapter,
3342                               struct be_resources *res)
3343 {
3344         bool use_sriov = adapter->num_vfs ? 1 : 0;
3345
3346         if (be_physfn(adapter))
3347                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3348         else
3349                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3350
3351         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3352
3353         if (be_is_mc(adapter)) {
3354                 /* Assuming that there are 4 channels per port,
3355                  * when multi-channel is enabled
3356                  */
3357                 if (be_is_qnq_mode(adapter))
3358                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3359                 else
3360                         /* In a non-qnq multichannel mode, the pvid
3361                          * takes up one vlan entry
3362                          */
3363                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3364         } else {
3365                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3366         }
3367
3368         res->max_mcast_mac = BE_MAX_MC;
3369
3370         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3371          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3372          *    *only* if it is RSS-capable.
3373          */
3374         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3375             !be_physfn(adapter) || (be_is_mc(adapter) &&
3376             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3377                 res->max_tx_qs = 1;
3378         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3379                 struct be_resources super_nic_res = {0};
3380
3381                 /* On a SuperNIC profile, the driver needs to use the
3382                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3383                  */
3384                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3385                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3386                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3387         } else {
3388                 res->max_tx_qs = BE3_MAX_TX_QS;
3389         }
3390
3391         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3392             !use_sriov && be_physfn(adapter))
3393                 res->max_rss_qs = (adapter->be3_native) ?
3394                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3395         res->max_rx_qs = res->max_rss_qs + 1;
3396
3397         if (be_physfn(adapter))
3398                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3399                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3400         else
3401                 res->max_evt_qs = 1;
3402
3403         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3404         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3405                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3406 }
3407
3408 static void be_setup_init(struct be_adapter *adapter)
3409 {
3410         adapter->vlan_prio_bmap = 0xff;
3411         adapter->phy.link_speed = -1;
3412         adapter->if_handle = -1;
3413         adapter->be3_native = false;
3414         adapter->promiscuous = false;
3415         if (be_physfn(adapter))
3416                 adapter->cmd_privileges = MAX_PRIVILEGES;
3417         else
3418                 adapter->cmd_privileges = MIN_PRIVILEGES;
3419 }
3420
3421 static int be_get_sriov_config(struct be_adapter *adapter)
3422 {
3423         struct device *dev = &adapter->pdev->dev;
3424         struct be_resources res = {0};
3425         int max_vfs, old_vfs;
3426
3427         /* Some old versions of BE3 FW don't report max_vfs value */
3428         be_cmd_get_profile_config(adapter, &res, 0);
3429
3430         if (BE3_chip(adapter) && !res.max_vfs) {
3431                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3432                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3433         }
3434
3435         adapter->pool_res = res;
3436
3437         if (!be_max_vfs(adapter)) {
3438                 if (num_vfs)
3439                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3440                 adapter->num_vfs = 0;
3441                 return 0;
3442         }
3443
3444         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3445
3446         /* validate num_vfs module param */
3447         old_vfs = pci_num_vf(adapter->pdev);
3448         if (old_vfs) {
3449                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3450                 if (old_vfs != num_vfs)
3451                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3452                 adapter->num_vfs = old_vfs;
3453         } else {
3454                 if (num_vfs > be_max_vfs(adapter)) {
3455                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3456                                  num_vfs);
3457                         dev_info(dev, "Limiting to %d VFs\n",
3458                                  be_max_vfs(adapter));
3459                 }
3460                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3461         }
3462
3463         return 0;
3464 }
3465
3466 static int be_get_resources(struct be_adapter *adapter)
3467 {
3468         struct device *dev = &adapter->pdev->dev;
3469         struct be_resources res = {0};
3470         int status;
3471
3472         if (BEx_chip(adapter)) {
3473                 BEx_get_resources(adapter, &res);
3474                 adapter->res = res;
3475         }
3476
3477         /* For Lancer, SH etc read per-function resource limits from FW.
3478          * GET_FUNC_CONFIG returns per function guaranteed limits.
3479          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3480          */
3481         if (!BEx_chip(adapter)) {
3482                 status = be_cmd_get_func_config(adapter, &res);
3483                 if (status)
3484                         return status;
3485
3486                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3487                 if (be_roce_supported(adapter))
3488                         res.max_evt_qs /= 2;
3489                 adapter->res = res;
3490         }
3491
3492         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3493                  be_max_txqs(adapter), be_max_rxqs(adapter),
3494                  be_max_rss(adapter), be_max_eqs(adapter),
3495                  be_max_vfs(adapter));
3496         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3497                  be_max_uc(adapter), be_max_mc(adapter),
3498                  be_max_vlans(adapter));
3499
3500         return 0;
3501 }
3502
3503 static void be_sriov_config(struct be_adapter *adapter)
3504 {
3505         struct device *dev = &adapter->pdev->dev;
3506         int status;
3507
3508         status = be_get_sriov_config(adapter);
3509         if (status) {
3510                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3511                 dev_err(dev, "SR-IOV cannot be enabled\n");
3512                 return;
3513         }
3514
3515         /* When the HW is in SRIOV capable configuration, the PF-pool
3516          * resources are equally distributed across the max-number of
3517          * VFs. The user may request only a subset of the max-vfs to be
3518          * enabled. Based on num_vfs, redistribute the resources across
3519          * num_vfs so that each VF will have access to more number of
3520          * resources. This facility is not available in BE3 FW.
3521          * Also, this is done by FW in Lancer chip.
3522          */
3523         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3524                 status = be_cmd_set_sriov_config(adapter,
3525                                                  adapter->pool_res,
3526                                                  adapter->num_vfs);
3527                 if (status)
3528                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3529         }
3530 }
3531
3532 static int be_get_config(struct be_adapter *adapter)
3533 {
3534         u16 profile_id;
3535         int status;
3536
3537         status = be_cmd_query_fw_cfg(adapter);
3538         if (status)
3539                 return status;
3540
3541          if (be_physfn(adapter)) {
3542                 status = be_cmd_get_active_profile(adapter, &profile_id);
3543                 if (!status)
3544                         dev_info(&adapter->pdev->dev,
3545                                  "Using profile 0x%x\n", profile_id);
3546         }
3547
3548         if (!BE2_chip(adapter) && be_physfn(adapter))
3549                 be_sriov_config(adapter);
3550
3551         status = be_get_resources(adapter);
3552         if (status)
3553                 return status;
3554
3555         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3556                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3557         if (!adapter->pmac_id)
3558                 return -ENOMEM;
3559
3560         /* Sanitize cfg_num_qs based on HW and platform limits */
3561         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3562
3563         return 0;
3564 }
3565
3566 static int be_mac_setup(struct be_adapter *adapter)
3567 {
3568         u8 mac[ETH_ALEN];
3569         int status;
3570
3571         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3572                 status = be_cmd_get_perm_mac(adapter, mac);
3573                 if (status)
3574                         return status;
3575
3576                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3577                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3578         } else {
3579                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3580                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3581         }
3582
3583         /* For BE3-R VFs, the PF programs the initial MAC address */
3584         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3585                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3586                                 &adapter->pmac_id[0], 0);
3587         return 0;
3588 }
3589
3590 static void be_schedule_worker(struct be_adapter *adapter)
3591 {
3592         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3593         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3594 }
3595
3596 static int be_setup_queues(struct be_adapter *adapter)
3597 {
3598         struct net_device *netdev = adapter->netdev;
3599         int status;
3600
3601         status = be_evt_queues_create(adapter);
3602         if (status)
3603                 goto err;
3604
3605         status = be_tx_qs_create(adapter);
3606         if (status)
3607                 goto err;
3608
3609         status = be_rx_cqs_create(adapter);
3610         if (status)
3611                 goto err;
3612
3613         status = be_mcc_queues_create(adapter);
3614         if (status)
3615                 goto err;
3616
3617         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3618         if (status)
3619                 goto err;
3620
3621         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3622         if (status)
3623                 goto err;
3624
3625         return 0;
3626 err:
3627         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3628         return status;
3629 }
3630
3631 int be_update_queues(struct be_adapter *adapter)
3632 {
3633         struct net_device *netdev = adapter->netdev;
3634         int status;
3635
3636         if (netif_running(netdev))
3637                 be_close(netdev);
3638
3639         be_cancel_worker(adapter);
3640
3641         /* If any vectors have been shared with RoCE we cannot re-program
3642          * the MSIx table.
3643          */
3644         if (!adapter->num_msix_roce_vec)
3645                 be_msix_disable(adapter);
3646
3647         be_clear_queues(adapter);
3648
3649         if (!msix_enabled(adapter)) {
3650                 status = be_msix_enable(adapter);
3651                 if (status)
3652                         return status;
3653         }
3654
3655         status = be_setup_queues(adapter);
3656         if (status)
3657                 return status;
3658
3659         be_schedule_worker(adapter);
3660
3661         if (netif_running(netdev))
3662                 status = be_open(netdev);
3663
3664         return status;
3665 }
3666
3667 static int be_setup(struct be_adapter *adapter)
3668 {
3669         struct device *dev = &adapter->pdev->dev;
3670         int status;
3671
3672         be_setup_init(adapter);
3673
3674         if (!lancer_chip(adapter))
3675                 be_cmd_req_native_mode(adapter);
3676
3677         status = be_get_config(adapter);
3678         if (status)
3679                 goto err;
3680
3681         status = be_msix_enable(adapter);
3682         if (status)
3683                 goto err;
3684
3685         status = be_if_create(adapter, &adapter->if_handle,
3686                               be_if_cap_flags(adapter), 0);
3687         if (status)
3688                 goto err;
3689
3690         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3691         rtnl_lock();
3692         status = be_setup_queues(adapter);
3693         rtnl_unlock();
3694         if (status)
3695                 goto err;
3696
3697         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3698
3699         status = be_mac_setup(adapter);
3700         if (status)
3701                 goto err;
3702
3703         be_cmd_get_fw_ver(adapter);
3704         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3705
3706         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3707                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3708                         adapter->fw_ver);
3709                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3710         }
3711
3712         if (adapter->vlans_added)
3713                 be_vid_config(adapter);
3714
3715         be_set_rx_mode(adapter->netdev);
3716
3717         be_cmd_get_acpi_wol_cap(adapter);
3718
3719         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
3720                                          adapter->rx_fc);
3721         if (status)
3722                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
3723                                         &adapter->rx_fc);
3724
3725         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
3726                  adapter->tx_fc, adapter->rx_fc);
3727
3728         if (be_physfn(adapter))
3729                 be_cmd_set_logical_link_config(adapter,
3730                                                IFLA_VF_LINK_STATE_AUTO, 0);
3731
3732         if (adapter->num_vfs)
3733                 be_vf_setup(adapter);
3734
3735         status = be_cmd_get_phy_info(adapter);
3736         if (!status && be_pause_supported(adapter))
3737                 adapter->phy.fc_autoneg = 1;
3738
3739         be_schedule_worker(adapter);
3740         adapter->flags |= BE_FLAGS_SETUP_DONE;
3741         return 0;
3742 err:
3743         be_clear(adapter);
3744         return status;
3745 }
3746
3747 #ifdef CONFIG_NET_POLL_CONTROLLER
3748 static void be_netpoll(struct net_device *netdev)
3749 {
3750         struct be_adapter *adapter = netdev_priv(netdev);
3751         struct be_eq_obj *eqo;
3752         int i;
3753
3754         for_all_evt_queues(adapter, eqo, i) {
3755                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3756                 napi_schedule(&eqo->napi);
3757         }
3758 }
3759 #endif
3760
3761 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3762
3763 static bool phy_flashing_required(struct be_adapter *adapter)
3764 {
3765         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
3766                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3767 }
3768
3769 static bool is_comp_in_ufi(struct be_adapter *adapter,
3770                            struct flash_section_info *fsec, int type)
3771 {
3772         int i = 0, img_type = 0;
3773         struct flash_section_info_g2 *fsec_g2 = NULL;
3774
3775         if (BE2_chip(adapter))
3776                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3777
3778         for (i = 0; i < MAX_FLASH_COMP; i++) {
3779                 if (fsec_g2)
3780                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3781                 else
3782                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3783
3784                 if (img_type == type)
3785                         return true;
3786         }
3787         return false;
3788
3789 }
3790
3791 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3792                                                 int header_size,
3793                                                 const struct firmware *fw)
3794 {
3795         struct flash_section_info *fsec = NULL;
3796         const u8 *p = fw->data;
3797
3798         p += header_size;
3799         while (p < (fw->data + fw->size)) {
3800                 fsec = (struct flash_section_info *)p;
3801                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3802                         return fsec;
3803                 p += 32;
3804         }
3805         return NULL;
3806 }
3807
3808 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3809                               u32 img_offset, u32 img_size, int hdr_size,
3810                               u16 img_optype, bool *crc_match)
3811 {
3812         u32 crc_offset;
3813         int status;
3814         u8 crc[4];
3815
3816         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3817         if (status)
3818                 return status;
3819
3820         crc_offset = hdr_size + img_offset + img_size - 4;
3821
3822         /* Skip flashing, if crc of flashed region matches */
3823         if (!memcmp(crc, p + crc_offset, 4))
3824                 *crc_match = true;
3825         else
3826                 *crc_match = false;
3827
3828         return status;
3829 }
3830
3831 static int be_flash(struct be_adapter *adapter, const u8 *img,
3832                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3833 {
3834         struct be_cmd_write_flashrom *req = flash_cmd->va;
3835         u32 total_bytes, flash_op, num_bytes;
3836         int status;
3837
3838         total_bytes = img_size;
3839         while (total_bytes) {
3840                 num_bytes = min_t(u32, 32*1024, total_bytes);
3841
3842                 total_bytes -= num_bytes;
3843
3844                 if (!total_bytes) {
3845                         if (optype == OPTYPE_PHY_FW)
3846                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3847                         else
3848                                 flash_op = FLASHROM_OPER_FLASH;
3849                 } else {
3850                         if (optype == OPTYPE_PHY_FW)
3851                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3852                         else
3853                                 flash_op = FLASHROM_OPER_SAVE;
3854                 }
3855
3856                 memcpy(req->data_buf, img, num_bytes);
3857                 img += num_bytes;
3858                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3859                                                flash_op, num_bytes);
3860                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3861                     optype == OPTYPE_PHY_FW)
3862                         break;
3863                 else if (status)
3864                         return status;
3865         }
3866         return 0;
3867 }
3868
3869 /* For BE2, BE3 and BE3-R */
3870 static int be_flash_BEx(struct be_adapter *adapter,
3871                         const struct firmware *fw,
3872                         struct be_dma_mem *flash_cmd, int num_of_images)
3873 {
3874         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3875         struct device *dev = &adapter->pdev->dev;
3876         struct flash_section_info *fsec = NULL;
3877         int status, i, filehdr_size, num_comp;
3878         const struct flash_comp *pflashcomp;
3879         bool crc_match;
3880         const u8 *p;
3881
3882         struct flash_comp gen3_flash_types[] = {
3883                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3884                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3885                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3886                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3887                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3888                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3889                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3890                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3891                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3892                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3893                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3894                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3895                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3896                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3897                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3898                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3899                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3900                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3901                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3902                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3903         };
3904
3905         struct flash_comp gen2_flash_types[] = {
3906                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3907                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3908                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3909                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3910                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3911                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3912                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3913                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3914                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3915                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3916                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3917                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3918                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3919                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3920                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3921                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3922         };
3923
3924         if (BE3_chip(adapter)) {
3925                 pflashcomp = gen3_flash_types;
3926                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3927                 num_comp = ARRAY_SIZE(gen3_flash_types);
3928         } else {
3929                 pflashcomp = gen2_flash_types;
3930                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3931                 num_comp = ARRAY_SIZE(gen2_flash_types);
3932         }
3933
3934         /* Get flash section info*/
3935         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3936         if (!fsec) {
3937                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3938                 return -1;
3939         }
3940         for (i = 0; i < num_comp; i++) {
3941                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3942                         continue;
3943
3944                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3945                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3946                         continue;
3947
3948                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3949                     !phy_flashing_required(adapter))
3950                                 continue;
3951
3952                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3953                         status = be_check_flash_crc(adapter, fw->data,
3954                                                     pflashcomp[i].offset,
3955                                                     pflashcomp[i].size,
3956                                                     filehdr_size +
3957                                                     img_hdrs_size,
3958                                                     OPTYPE_REDBOOT, &crc_match);
3959                         if (status) {
3960                                 dev_err(dev,
3961                                         "Could not get CRC for 0x%x region\n",
3962                                         pflashcomp[i].optype);
3963                                 continue;
3964                         }
3965
3966                         if (crc_match)
3967                                 continue;
3968                 }
3969
3970                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3971                         img_hdrs_size;
3972                 if (p + pflashcomp[i].size > fw->data + fw->size)
3973                         return -1;
3974
3975                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3976                                   pflashcomp[i].size);
3977                 if (status) {
3978                         dev_err(dev, "Flashing section type 0x%x failed\n",
3979                                 pflashcomp[i].img_type);
3980                         return status;
3981                 }
3982         }
3983         return 0;
3984 }
3985
3986 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3987 {
3988         u32 img_type = le32_to_cpu(fsec_entry.type);
3989         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3990
3991         if (img_optype != 0xFFFF)
3992                 return img_optype;
3993
3994         switch (img_type) {
3995         case IMAGE_FIRMWARE_iSCSI:
3996                 img_optype = OPTYPE_ISCSI_ACTIVE;
3997                 break;
3998         case IMAGE_BOOT_CODE:
3999                 img_optype = OPTYPE_REDBOOT;
4000                 break;
4001         case IMAGE_OPTION_ROM_ISCSI:
4002                 img_optype = OPTYPE_BIOS;
4003                 break;
4004         case IMAGE_OPTION_ROM_PXE:
4005                 img_optype = OPTYPE_PXE_BIOS;
4006                 break;
4007         case IMAGE_OPTION_ROM_FCoE:
4008                 img_optype = OPTYPE_FCOE_BIOS;
4009                 break;
4010         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4011                 img_optype = OPTYPE_ISCSI_BACKUP;
4012                 break;
4013         case IMAGE_NCSI:
4014                 img_optype = OPTYPE_NCSI_FW;
4015                 break;
4016         case IMAGE_FLASHISM_JUMPVECTOR:
4017                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4018                 break;
4019         case IMAGE_FIRMWARE_PHY:
4020                 img_optype = OPTYPE_SH_PHY_FW;
4021                 break;
4022         case IMAGE_REDBOOT_DIR:
4023                 img_optype = OPTYPE_REDBOOT_DIR;
4024                 break;
4025         case IMAGE_REDBOOT_CONFIG:
4026                 img_optype = OPTYPE_REDBOOT_CONFIG;
4027                 break;
4028         case IMAGE_UFI_DIR:
4029                 img_optype = OPTYPE_UFI_DIR;
4030                 break;
4031         default:
4032                 break;
4033         }
4034
4035         return img_optype;
4036 }
4037
4038 static int be_flash_skyhawk(struct be_adapter *adapter,
4039                             const struct firmware *fw,
4040                             struct be_dma_mem *flash_cmd, int num_of_images)
4041 {
4042         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4043         struct device *dev = &adapter->pdev->dev;
4044         struct flash_section_info *fsec = NULL;
4045         u32 img_offset, img_size, img_type;
4046         int status, i, filehdr_size;
4047         bool crc_match, old_fw_img;
4048         u16 img_optype;
4049         const u8 *p;
4050
4051         filehdr_size = sizeof(struct flash_file_hdr_g3);
4052         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4053         if (!fsec) {
4054                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4055                 return -EINVAL;
4056         }
4057
4058         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4059                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4060                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4061                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4062                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4063                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4064
4065                 if (img_optype == 0xFFFF)
4066                         continue;
4067                 /* Don't bother verifying CRC if an old FW image is being
4068                  * flashed
4069                  */
4070                 if (old_fw_img)
4071                         goto flash;
4072
4073                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4074                                             img_size, filehdr_size +
4075                                             img_hdrs_size, img_optype,
4076                                             &crc_match);
4077                 /* The current FW image on the card does not recognize the new
4078                  * FLASH op_type. The FW download is partially complete.
4079                  * Reboot the server now to enable FW image to recognize the
4080                  * new FLASH op_type. To complete the remaining process,
4081                  * download the same FW again after the reboot.
4082                  */
4083                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4084                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4085                         dev_err(dev, "Flash incomplete. Reset the server\n");
4086                         dev_err(dev, "Download FW image again after reset\n");
4087                         return -EAGAIN;
4088                 } else if (status) {
4089                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4090                                 img_optype);
4091                         return -EFAULT;
4092                 }
4093
4094                 if (crc_match)
4095                         continue;
4096
4097 flash:
4098                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4099                 if (p + img_size > fw->data + fw->size)
4100                         return -1;
4101
4102                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4103                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4104                  * UFI_DIR region
4105                  */
4106                 if (old_fw_img &&
4107                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4108                      (img_optype == OPTYPE_UFI_DIR &&
4109                       base_status(status) == MCC_STATUS_FAILED))) {
4110                         continue;
4111                 } else if (status) {
4112                         dev_err(dev, "Flashing section type 0x%x failed\n",
4113                                 img_type);
4114                         return -EFAULT;
4115                 }
4116         }
4117         return 0;
4118 }
4119
4120 static int lancer_fw_download(struct be_adapter *adapter,
4121                               const struct firmware *fw)
4122 {
4123 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4124 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4125         struct device *dev = &adapter->pdev->dev;
4126         struct be_dma_mem flash_cmd;
4127         const u8 *data_ptr = NULL;
4128         u8 *dest_image_ptr = NULL;
4129         size_t image_size = 0;
4130         u32 chunk_size = 0;
4131         u32 data_written = 0;
4132         u32 offset = 0;
4133         int status = 0;
4134         u8 add_status = 0;
4135         u8 change_status;
4136
4137         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4138                 dev_err(dev, "FW image size should be multiple of 4\n");
4139                 return -EINVAL;
4140         }
4141
4142         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4143                                 + LANCER_FW_DOWNLOAD_CHUNK;
4144         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4145                                           &flash_cmd.dma, GFP_KERNEL);
4146         if (!flash_cmd.va)
4147                 return -ENOMEM;
4148
4149         dest_image_ptr = flash_cmd.va +
4150                                 sizeof(struct lancer_cmd_req_write_object);
4151         image_size = fw->size;
4152         data_ptr = fw->data;
4153
4154         while (image_size) {
4155                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4156
4157                 /* Copy the image chunk content. */
4158                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4159
4160                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4161                                                  chunk_size, offset,
4162                                                  LANCER_FW_DOWNLOAD_LOCATION,
4163                                                  &data_written, &change_status,
4164                                                  &add_status);
4165                 if (status)
4166                         break;
4167
4168                 offset += data_written;
4169                 data_ptr += data_written;
4170                 image_size -= data_written;
4171         }
4172
4173         if (!status) {
4174                 /* Commit the FW written */
4175                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4176                                                  0, offset,
4177                                                  LANCER_FW_DOWNLOAD_LOCATION,
4178                                                  &data_written, &change_status,
4179                                                  &add_status);
4180         }
4181
4182         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4183         if (status) {
4184                 dev_err(dev, "Firmware load error\n");
4185                 return be_cmd_status(status);
4186         }
4187
4188         dev_info(dev, "Firmware flashed successfully\n");
4189
4190         if (change_status == LANCER_FW_RESET_NEEDED) {
4191                 dev_info(dev, "Resetting adapter to activate new FW\n");
4192                 status = lancer_physdev_ctrl(adapter,
4193                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4194                 if (status) {
4195                         dev_err(dev, "Adapter busy, could not reset FW\n");
4196                         dev_err(dev, "Reboot server to activate new FW\n");
4197                 }
4198         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4199                 dev_info(dev, "Reboot server to activate new FW\n");
4200         }
4201
4202         return 0;
4203 }
4204
4205 #define UFI_TYPE2               2
4206 #define UFI_TYPE3               3
4207 #define UFI_TYPE3R              10
4208 #define UFI_TYPE4               4
4209 static int be_get_ufi_type(struct be_adapter *adapter,
4210                            struct flash_file_hdr_g3 *fhdr)
4211 {
4212         if (!fhdr)
4213                 goto be_get_ufi_exit;
4214
4215         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4216                 return UFI_TYPE4;
4217         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4218                 if (fhdr->asic_type_rev == 0x10)
4219                         return UFI_TYPE3R;
4220                 else
4221                         return UFI_TYPE3;
4222         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4223                 return UFI_TYPE2;
4224
4225 be_get_ufi_exit:
4226         dev_err(&adapter->pdev->dev,
4227                 "UFI and Interface are not compatible for flashing\n");
4228         return -1;
4229 }
4230
4231 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4232 {
4233         struct flash_file_hdr_g3 *fhdr3;
4234         struct image_hdr *img_hdr_ptr = NULL;
4235         struct be_dma_mem flash_cmd;
4236         const u8 *p;
4237         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4238
4239         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4240         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4241                                           &flash_cmd.dma, GFP_KERNEL);
4242         if (!flash_cmd.va) {
4243                 status = -ENOMEM;
4244                 goto be_fw_exit;
4245         }
4246
4247         p = fw->data;
4248         fhdr3 = (struct flash_file_hdr_g3 *)p;
4249
4250         ufi_type = be_get_ufi_type(adapter, fhdr3);
4251
4252         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4253         for (i = 0; i < num_imgs; i++) {
4254                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4255                                 (sizeof(struct flash_file_hdr_g3) +
4256                                  i * sizeof(struct image_hdr)));
4257                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4258                         switch (ufi_type) {
4259                         case UFI_TYPE4:
4260                                 status = be_flash_skyhawk(adapter, fw,
4261                                                           &flash_cmd, num_imgs);
4262                                 break;
4263                         case UFI_TYPE3R:
4264                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4265                                                       num_imgs);
4266                                 break;
4267                         case UFI_TYPE3:
4268                                 /* Do not flash this ufi on BE3-R cards */
4269                                 if (adapter->asic_rev < 0x10)
4270                                         status = be_flash_BEx(adapter, fw,
4271                                                               &flash_cmd,
4272                                                               num_imgs);
4273                                 else {
4274                                         status = -EINVAL;
4275                                         dev_err(&adapter->pdev->dev,
4276                                                 "Can't load BE3 UFI on BE3R\n");
4277                                 }
4278                         }
4279                 }
4280         }
4281
4282         if (ufi_type == UFI_TYPE2)
4283                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4284         else if (ufi_type == -1)
4285                 status = -EINVAL;
4286
4287         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4288                           flash_cmd.dma);
4289         if (status) {
4290                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4291                 goto be_fw_exit;
4292         }
4293
4294         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4295
4296 be_fw_exit:
4297         return status;
4298 }
4299
4300 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4301 {
4302         const struct firmware *fw;
4303         int status;
4304
4305         if (!netif_running(adapter->netdev)) {
4306                 dev_err(&adapter->pdev->dev,
4307                         "Firmware load not allowed (interface is down)\n");
4308                 return -ENETDOWN;
4309         }
4310
4311         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4312         if (status)
4313                 goto fw_exit;
4314
4315         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4316
4317         if (lancer_chip(adapter))
4318                 status = lancer_fw_download(adapter, fw);
4319         else
4320                 status = be_fw_download(adapter, fw);
4321
4322         if (!status)
4323                 be_cmd_get_fw_ver(adapter);
4324
4325 fw_exit:
4326         release_firmware(fw);
4327         return status;
4328 }
4329
4330 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4331 {
4332         struct be_adapter *adapter = netdev_priv(dev);
4333         struct nlattr *attr, *br_spec;
4334         int rem;
4335         int status = 0;
4336         u16 mode = 0;
4337
4338         if (!sriov_enabled(adapter))
4339                 return -EOPNOTSUPP;
4340
4341         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4342         if (!br_spec)
4343                 return -EINVAL;
4344
4345         nla_for_each_nested(attr, br_spec, rem) {
4346                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4347                         continue;
4348
4349                 if (nla_len(attr) < sizeof(mode))
4350                         return -EINVAL;
4351
4352                 mode = nla_get_u16(attr);
4353                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4354                         return -EINVAL;
4355
4356                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4357                                                adapter->if_handle,
4358                                                mode == BRIDGE_MODE_VEPA ?
4359                                                PORT_FWD_TYPE_VEPA :
4360                                                PORT_FWD_TYPE_VEB);
4361                 if (status)
4362                         goto err;
4363
4364                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4365                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4366
4367                 return status;
4368         }
4369 err:
4370         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4371                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4372
4373         return status;
4374 }
4375
4376 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4377                                  struct net_device *dev, u32 filter_mask)
4378 {
4379         struct be_adapter *adapter = netdev_priv(dev);
4380         int status = 0;
4381         u8 hsw_mode;
4382
4383         if (!sriov_enabled(adapter))
4384                 return 0;
4385
4386         /* BE and Lancer chips support VEB mode only */
4387         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4388                 hsw_mode = PORT_FWD_TYPE_VEB;
4389         } else {
4390                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4391                                                adapter->if_handle, &hsw_mode);
4392                 if (status)
4393                         return 0;
4394         }
4395
4396         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4397                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4398                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4399                                        0, 0);
4400 }
4401
4402 #ifdef CONFIG_BE2NET_VXLAN
4403 /* VxLAN offload Notes:
4404  *
4405  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4406  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4407  * is expected to work across all types of IP tunnels once exported. Skyhawk
4408  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4409  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4410  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4411  * those other tunnels are unexported on the fly through ndo_features_check().
4412  *
4413  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4414  * adds more than one port, disable offloads and don't re-enable them again
4415  * until after all the tunnels are removed.
4416  */
4417 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4418                               __be16 port)
4419 {
4420         struct be_adapter *adapter = netdev_priv(netdev);
4421         struct device *dev = &adapter->pdev->dev;
4422         int status;
4423
4424         if (lancer_chip(adapter) || BEx_chip(adapter))
4425                 return;
4426
4427         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4428                 dev_info(dev,
4429                          "Only one UDP port supported for VxLAN offloads\n");
4430                 dev_info(dev, "Disabling VxLAN offloads\n");
4431                 adapter->vxlan_port_count++;
4432                 goto err;
4433         }
4434
4435         if (adapter->vxlan_port_count++ >= 1)
4436                 return;
4437
4438         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4439                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4440         if (status) {
4441                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4442                 goto err;
4443         }
4444
4445         status = be_cmd_set_vxlan_port(adapter, port);
4446         if (status) {
4447                 dev_warn(dev, "Failed to add VxLAN port\n");
4448                 goto err;
4449         }
4450         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4451         adapter->vxlan_port = port;
4452
4453         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4454                                    NETIF_F_TSO | NETIF_F_TSO6 |
4455                                    NETIF_F_GSO_UDP_TUNNEL;
4456         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4457         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4458
4459         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4460                  be16_to_cpu(port));
4461         return;
4462 err:
4463         be_disable_vxlan_offloads(adapter);
4464 }
4465
4466 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4467                               __be16 port)
4468 {
4469         struct be_adapter *adapter = netdev_priv(netdev);
4470
4471         if (lancer_chip(adapter) || BEx_chip(adapter))
4472                 return;
4473
4474         if (adapter->vxlan_port != port)
4475                 goto done;
4476
4477         be_disable_vxlan_offloads(adapter);
4478
4479         dev_info(&adapter->pdev->dev,
4480                  "Disabled VxLAN offloads for UDP port %d\n",
4481                  be16_to_cpu(port));
4482 done:
4483         adapter->vxlan_port_count--;
4484 }
4485
4486 static netdev_features_t be_features_check(struct sk_buff *skb,
4487                                            struct net_device *dev,
4488                                            netdev_features_t features)
4489 {
4490         struct be_adapter *adapter = netdev_priv(dev);
4491         u8 l4_hdr = 0;
4492
4493         /* The code below restricts offload features for some tunneled packets.
4494          * Offload features for normal (non tunnel) packets are unchanged.
4495          */
4496         if (!skb->encapsulation ||
4497             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4498                 return features;
4499
4500         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4501          * should disable tunnel offload features if it's not a VxLAN packet,
4502          * as tunnel offloads have been enabled only for VxLAN. This is done to
4503          * allow other tunneled traffic like GRE work fine while VxLAN
4504          * offloads are configured in Skyhawk-R.
4505          */
4506         switch (vlan_get_protocol(skb)) {
4507         case htons(ETH_P_IP):
4508                 l4_hdr = ip_hdr(skb)->protocol;
4509                 break;
4510         case htons(ETH_P_IPV6):
4511                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4512                 break;
4513         default:
4514                 return features;
4515         }
4516
4517         if (l4_hdr != IPPROTO_UDP ||
4518             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4519             skb->inner_protocol != htons(ETH_P_TEB) ||
4520             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4521             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4522                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4523
4524         return features;
4525 }
4526 #endif
4527
4528 static const struct net_device_ops be_netdev_ops = {
4529         .ndo_open               = be_open,
4530         .ndo_stop               = be_close,
4531         .ndo_start_xmit         = be_xmit,
4532         .ndo_set_rx_mode        = be_set_rx_mode,
4533         .ndo_set_mac_address    = be_mac_addr_set,
4534         .ndo_change_mtu         = be_change_mtu,
4535         .ndo_get_stats64        = be_get_stats64,
4536         .ndo_validate_addr      = eth_validate_addr,
4537         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4538         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4539         .ndo_set_vf_mac         = be_set_vf_mac,
4540         .ndo_set_vf_vlan        = be_set_vf_vlan,
4541         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4542         .ndo_get_vf_config      = be_get_vf_config,
4543         .ndo_set_vf_link_state  = be_set_vf_link_state,
4544 #ifdef CONFIG_NET_POLL_CONTROLLER
4545         .ndo_poll_controller    = be_netpoll,
4546 #endif
4547         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4548         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4549 #ifdef CONFIG_NET_RX_BUSY_POLL
4550         .ndo_busy_poll          = be_busy_poll,
4551 #endif
4552 #ifdef CONFIG_BE2NET_VXLAN
4553         .ndo_add_vxlan_port     = be_add_vxlan_port,
4554         .ndo_del_vxlan_port     = be_del_vxlan_port,
4555         .ndo_features_check     = be_features_check,
4556 #endif
4557 };
4558
4559 static void be_netdev_init(struct net_device *netdev)
4560 {
4561         struct be_adapter *adapter = netdev_priv(netdev);
4562
4563         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4564                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4565                 NETIF_F_HW_VLAN_CTAG_TX;
4566         if (be_multi_rxq(adapter))
4567                 netdev->hw_features |= NETIF_F_RXHASH;
4568
4569         netdev->features |= netdev->hw_features |
4570                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4571
4572         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4573                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4574
4575         netdev->priv_flags |= IFF_UNICAST_FLT;
4576
4577         netdev->flags |= IFF_MULTICAST;
4578
4579         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4580
4581         netdev->netdev_ops = &be_netdev_ops;
4582
4583         netdev->ethtool_ops = &be_ethtool_ops;
4584 }
4585
4586 static void be_unmap_pci_bars(struct be_adapter *adapter)
4587 {
4588         if (adapter->csr)
4589                 pci_iounmap(adapter->pdev, adapter->csr);
4590         if (adapter->db)
4591                 pci_iounmap(adapter->pdev, adapter->db);
4592 }
4593
4594 static int db_bar(struct be_adapter *adapter)
4595 {
4596         if (lancer_chip(adapter) || !be_physfn(adapter))
4597                 return 0;
4598         else
4599                 return 4;
4600 }
4601
4602 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4603 {
4604         if (skyhawk_chip(adapter)) {
4605                 adapter->roce_db.size = 4096;
4606                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4607                                                               db_bar(adapter));
4608                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4609                                                                db_bar(adapter));
4610         }
4611         return 0;
4612 }
4613
4614 static int be_map_pci_bars(struct be_adapter *adapter)
4615 {
4616         u8 __iomem *addr;
4617
4618         if (BEx_chip(adapter) && be_physfn(adapter)) {
4619                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4620                 if (!adapter->csr)
4621                         return -ENOMEM;
4622         }
4623
4624         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4625         if (!addr)
4626                 goto pci_map_err;
4627         adapter->db = addr;
4628
4629         be_roce_map_pci_bars(adapter);
4630         return 0;
4631
4632 pci_map_err:
4633         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4634         be_unmap_pci_bars(adapter);
4635         return -ENOMEM;
4636 }
4637
4638 static void be_ctrl_cleanup(struct be_adapter *adapter)
4639 {
4640         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4641
4642         be_unmap_pci_bars(adapter);
4643
4644         if (mem->va)
4645                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4646                                   mem->dma);
4647
4648         mem = &adapter->rx_filter;
4649         if (mem->va)
4650                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4651                                   mem->dma);
4652 }
4653
4654 static int be_ctrl_init(struct be_adapter *adapter)
4655 {
4656         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4657         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4658         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4659         u32 sli_intf;
4660         int status;
4661
4662         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4663         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4664                                  SLI_INTF_FAMILY_SHIFT;
4665         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4666
4667         status = be_map_pci_bars(adapter);
4668         if (status)
4669                 goto done;
4670
4671         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4672         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4673                                                 mbox_mem_alloc->size,
4674                                                 &mbox_mem_alloc->dma,
4675                                                 GFP_KERNEL);
4676         if (!mbox_mem_alloc->va) {
4677                 status = -ENOMEM;
4678                 goto unmap_pci_bars;
4679         }
4680         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4681         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4682         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4683         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4684
4685         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4686         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4687                                             rx_filter->size, &rx_filter->dma,
4688                                             GFP_KERNEL);
4689         if (!rx_filter->va) {
4690                 status = -ENOMEM;
4691                 goto free_mbox;
4692         }
4693
4694         mutex_init(&adapter->mbox_lock);
4695         spin_lock_init(&adapter->mcc_lock);
4696         spin_lock_init(&adapter->mcc_cq_lock);
4697
4698         init_completion(&adapter->et_cmd_compl);
4699         pci_save_state(adapter->pdev);
4700         return 0;
4701
4702 free_mbox:
4703         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4704                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4705
4706 unmap_pci_bars:
4707         be_unmap_pci_bars(adapter);
4708
4709 done:
4710         return status;
4711 }
4712
4713 static void be_stats_cleanup(struct be_adapter *adapter)
4714 {
4715         struct be_dma_mem *cmd = &adapter->stats_cmd;
4716
4717         if (cmd->va)
4718                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4719                                   cmd->va, cmd->dma);
4720 }
4721
4722 static int be_stats_init(struct be_adapter *adapter)
4723 {
4724         struct be_dma_mem *cmd = &adapter->stats_cmd;
4725
4726         if (lancer_chip(adapter))
4727                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4728         else if (BE2_chip(adapter))
4729                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4730         else if (BE3_chip(adapter))
4731                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4732         else
4733                 /* ALL non-BE ASICs */
4734                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4735
4736         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4737                                       GFP_KERNEL);
4738         if (!cmd->va)
4739                 return -ENOMEM;
4740         return 0;
4741 }
4742
4743 static void be_remove(struct pci_dev *pdev)
4744 {
4745         struct be_adapter *adapter = pci_get_drvdata(pdev);
4746
4747         if (!adapter)
4748                 return;
4749
4750         be_roce_dev_remove(adapter);
4751         be_intr_set(adapter, false);
4752
4753         cancel_delayed_work_sync(&adapter->func_recovery_work);
4754
4755         unregister_netdev(adapter->netdev);
4756
4757         be_clear(adapter);
4758
4759         /* tell fw we're done with firing cmds */
4760         be_cmd_fw_clean(adapter);
4761
4762         be_stats_cleanup(adapter);
4763
4764         be_ctrl_cleanup(adapter);
4765
4766         pci_disable_pcie_error_reporting(pdev);
4767
4768         pci_release_regions(pdev);
4769         pci_disable_device(pdev);
4770
4771         free_netdev(adapter->netdev);
4772 }
4773
4774 static int be_get_initial_config(struct be_adapter *adapter)
4775 {
4776         int status, level;
4777
4778         status = be_cmd_get_cntl_attributes(adapter);
4779         if (status)
4780                 return status;
4781
4782         /* Must be a power of 2 or else MODULO will BUG_ON */
4783         adapter->be_get_temp_freq = 64;
4784
4785         if (BEx_chip(adapter)) {
4786                 level = be_cmd_get_fw_log_level(adapter);
4787                 adapter->msg_enable =
4788                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4789         }
4790
4791         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4792         return 0;
4793 }
4794
4795 static int lancer_recover_func(struct be_adapter *adapter)
4796 {
4797         struct device *dev = &adapter->pdev->dev;
4798         int status;
4799
4800         status = lancer_test_and_set_rdy_state(adapter);
4801         if (status)
4802                 goto err;
4803
4804         if (netif_running(adapter->netdev))
4805                 be_close(adapter->netdev);
4806
4807         be_clear(adapter);
4808
4809         be_clear_all_error(adapter);
4810
4811         status = be_setup(adapter);
4812         if (status)
4813                 goto err;
4814
4815         if (netif_running(adapter->netdev)) {
4816                 status = be_open(adapter->netdev);
4817                 if (status)
4818                         goto err;
4819         }
4820
4821         dev_err(dev, "Adapter recovery successful\n");
4822         return 0;
4823 err:
4824         if (status == -EAGAIN)
4825                 dev_err(dev, "Waiting for resource provisioning\n");
4826         else
4827                 dev_err(dev, "Adapter recovery failed\n");
4828
4829         return status;
4830 }
4831
4832 static void be_func_recovery_task(struct work_struct *work)
4833 {
4834         struct be_adapter *adapter =
4835                 container_of(work, struct be_adapter,  func_recovery_work.work);
4836         int status = 0;
4837
4838         be_detect_error(adapter);
4839
4840         if (adapter->hw_error && lancer_chip(adapter)) {
4841                 rtnl_lock();
4842                 netif_device_detach(adapter->netdev);
4843                 rtnl_unlock();
4844
4845                 status = lancer_recover_func(adapter);
4846                 if (!status)
4847                         netif_device_attach(adapter->netdev);
4848         }
4849
4850         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4851          * no need to attempt further recovery.
4852          */
4853         if (!status || status == -EAGAIN)
4854                 schedule_delayed_work(&adapter->func_recovery_work,
4855                                       msecs_to_jiffies(1000));
4856 }
4857
4858 static void be_worker(struct work_struct *work)
4859 {
4860         struct be_adapter *adapter =
4861                 container_of(work, struct be_adapter, work.work);
4862         struct be_rx_obj *rxo;
4863         int i;
4864
4865         /* when interrupts are not yet enabled, just reap any pending
4866         * mcc completions */
4867         if (!netif_running(adapter->netdev)) {
4868                 local_bh_disable();
4869                 be_process_mcc(adapter);
4870                 local_bh_enable();
4871                 goto reschedule;
4872         }
4873
4874         if (!adapter->stats_cmd_sent) {
4875                 if (lancer_chip(adapter))
4876                         lancer_cmd_get_pport_stats(adapter,
4877                                                    &adapter->stats_cmd);
4878                 else
4879                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4880         }
4881
4882         if (be_physfn(adapter) &&
4883             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4884                 be_cmd_get_die_temperature(adapter);
4885
4886         for_all_rx_queues(adapter, rxo, i) {
4887                 /* Replenish RX-queues starved due to memory
4888                  * allocation failures.
4889                  */
4890                 if (rxo->rx_post_starved)
4891                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
4892         }
4893
4894         be_eqd_update(adapter);
4895
4896 reschedule:
4897         adapter->work_counter++;
4898         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4899 }
4900
4901 /* If any VFs are already enabled don't FLR the PF */
4902 static bool be_reset_required(struct be_adapter *adapter)
4903 {
4904         return pci_num_vf(adapter->pdev) ? false : true;
4905 }
4906
4907 static char *mc_name(struct be_adapter *adapter)
4908 {
4909         char *str = ""; /* default */
4910
4911         switch (adapter->mc_type) {
4912         case UMC:
4913                 str = "UMC";
4914                 break;
4915         case FLEX10:
4916                 str = "FLEX10";
4917                 break;
4918         case vNIC1:
4919                 str = "vNIC-1";
4920                 break;
4921         case nPAR:
4922                 str = "nPAR";
4923                 break;
4924         case UFP:
4925                 str = "UFP";
4926                 break;
4927         case vNIC2:
4928                 str = "vNIC-2";
4929                 break;
4930         default:
4931                 str = "";
4932         }
4933
4934         return str;
4935 }
4936
4937 static inline char *func_name(struct be_adapter *adapter)
4938 {
4939         return be_physfn(adapter) ? "PF" : "VF";
4940 }
4941
4942 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4943 {
4944         int status = 0;
4945         struct be_adapter *adapter;
4946         struct net_device *netdev;
4947         char port_name;
4948
4949         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4950
4951         status = pci_enable_device(pdev);
4952         if (status)
4953                 goto do_none;
4954
4955         status = pci_request_regions(pdev, DRV_NAME);
4956         if (status)
4957                 goto disable_dev;
4958         pci_set_master(pdev);
4959
4960         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4961         if (!netdev) {
4962                 status = -ENOMEM;
4963                 goto rel_reg;
4964         }
4965         adapter = netdev_priv(netdev);
4966         adapter->pdev = pdev;
4967         pci_set_drvdata(pdev, adapter);
4968         adapter->netdev = netdev;
4969         SET_NETDEV_DEV(netdev, &pdev->dev);
4970
4971         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4972         if (!status) {
4973                 netdev->features |= NETIF_F_HIGHDMA;
4974         } else {
4975                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4976                 if (status) {
4977                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4978                         goto free_netdev;
4979                 }
4980         }
4981
4982         status = pci_enable_pcie_error_reporting(pdev);
4983         if (!status)
4984                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4985
4986         status = be_ctrl_init(adapter);
4987         if (status)
4988                 goto free_netdev;
4989
4990         /* sync up with fw's ready state */
4991         if (be_physfn(adapter)) {
4992                 status = be_fw_wait_ready(adapter);
4993                 if (status)
4994                         goto ctrl_clean;
4995         }
4996
4997         if (be_reset_required(adapter)) {
4998                 status = be_cmd_reset_function(adapter);
4999                 if (status)
5000                         goto ctrl_clean;
5001
5002                 /* Wait for interrupts to quiesce after an FLR */
5003                 msleep(100);
5004         }
5005
5006         /* Allow interrupts for other ULPs running on NIC function */
5007         be_intr_set(adapter, true);
5008
5009         /* tell fw we're ready to fire cmds */
5010         status = be_cmd_fw_init(adapter);
5011         if (status)
5012                 goto ctrl_clean;
5013
5014         status = be_stats_init(adapter);
5015         if (status)
5016                 goto ctrl_clean;
5017
5018         status = be_get_initial_config(adapter);
5019         if (status)
5020                 goto stats_clean;
5021
5022         INIT_DELAYED_WORK(&adapter->work, be_worker);
5023         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5024         adapter->rx_fc = true;
5025         adapter->tx_fc = true;
5026
5027         status = be_setup(adapter);
5028         if (status)
5029                 goto stats_clean;
5030
5031         be_netdev_init(netdev);
5032         status = register_netdev(netdev);
5033         if (status != 0)
5034                 goto unsetup;
5035
5036         be_roce_dev_add(adapter);
5037
5038         schedule_delayed_work(&adapter->func_recovery_work,
5039                               msecs_to_jiffies(1000));
5040
5041         be_cmd_query_port_name(adapter, &port_name);
5042
5043         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5044                  func_name(adapter), mc_name(adapter), port_name);
5045
5046         return 0;
5047
5048 unsetup:
5049         be_clear(adapter);
5050 stats_clean:
5051         be_stats_cleanup(adapter);
5052 ctrl_clean:
5053         be_ctrl_cleanup(adapter);
5054 free_netdev:
5055         free_netdev(netdev);
5056 rel_reg:
5057         pci_release_regions(pdev);
5058 disable_dev:
5059         pci_disable_device(pdev);
5060 do_none:
5061         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5062         return status;
5063 }
5064
5065 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5066 {
5067         struct be_adapter *adapter = pci_get_drvdata(pdev);
5068         struct net_device *netdev =  adapter->netdev;
5069
5070         if (adapter->wol_en)
5071                 be_setup_wol(adapter, true);
5072
5073         be_intr_set(adapter, false);
5074         cancel_delayed_work_sync(&adapter->func_recovery_work);
5075
5076         netif_device_detach(netdev);
5077         if (netif_running(netdev)) {
5078                 rtnl_lock();
5079                 be_close(netdev);
5080                 rtnl_unlock();
5081         }
5082         be_clear(adapter);
5083
5084         pci_save_state(pdev);
5085         pci_disable_device(pdev);
5086         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5087         return 0;
5088 }
5089
5090 static int be_resume(struct pci_dev *pdev)
5091 {
5092         int status = 0;
5093         struct be_adapter *adapter = pci_get_drvdata(pdev);
5094         struct net_device *netdev =  adapter->netdev;
5095
5096         netif_device_detach(netdev);
5097
5098         status = pci_enable_device(pdev);
5099         if (status)
5100                 return status;
5101
5102         pci_set_power_state(pdev, PCI_D0);
5103         pci_restore_state(pdev);
5104
5105         status = be_fw_wait_ready(adapter);
5106         if (status)
5107                 return status;
5108
5109         status = be_cmd_reset_function(adapter);
5110         if (status)
5111                 return status;
5112
5113         be_intr_set(adapter, true);
5114         /* tell fw we're ready to fire cmds */
5115         status = be_cmd_fw_init(adapter);
5116         if (status)
5117                 return status;
5118
5119         be_setup(adapter);
5120         if (netif_running(netdev)) {
5121                 rtnl_lock();
5122                 be_open(netdev);
5123                 rtnl_unlock();
5124         }
5125
5126         schedule_delayed_work(&adapter->func_recovery_work,
5127                               msecs_to_jiffies(1000));
5128         netif_device_attach(netdev);
5129
5130         if (adapter->wol_en)
5131                 be_setup_wol(adapter, false);
5132
5133         return 0;
5134 }
5135
5136 /*
5137  * An FLR will stop BE from DMAing any data.
5138  */
5139 static void be_shutdown(struct pci_dev *pdev)
5140 {
5141         struct be_adapter *adapter = pci_get_drvdata(pdev);
5142
5143         if (!adapter)
5144                 return;
5145
5146         be_roce_dev_shutdown(adapter);
5147         cancel_delayed_work_sync(&adapter->work);
5148         cancel_delayed_work_sync(&adapter->func_recovery_work);
5149
5150         netif_device_detach(adapter->netdev);
5151
5152         be_cmd_reset_function(adapter);
5153
5154         pci_disable_device(pdev);
5155 }
5156
5157 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5158                                             pci_channel_state_t state)
5159 {
5160         struct be_adapter *adapter = pci_get_drvdata(pdev);
5161         struct net_device *netdev =  adapter->netdev;
5162
5163         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5164
5165         if (!adapter->eeh_error) {
5166                 adapter->eeh_error = true;
5167
5168                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5169
5170                 rtnl_lock();
5171                 netif_device_detach(netdev);
5172                 if (netif_running(netdev))
5173                         be_close(netdev);
5174                 rtnl_unlock();
5175
5176                 be_clear(adapter);
5177         }
5178
5179         if (state == pci_channel_io_perm_failure)
5180                 return PCI_ERS_RESULT_DISCONNECT;
5181
5182         pci_disable_device(pdev);
5183
5184         /* The error could cause the FW to trigger a flash debug dump.
5185          * Resetting the card while flash dump is in progress
5186          * can cause it not to recover; wait for it to finish.
5187          * Wait only for first function as it is needed only once per
5188          * adapter.
5189          */
5190         if (pdev->devfn == 0)
5191                 ssleep(30);
5192
5193         return PCI_ERS_RESULT_NEED_RESET;
5194 }
5195
5196 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5197 {
5198         struct be_adapter *adapter = pci_get_drvdata(pdev);
5199         int status;
5200
5201         dev_info(&adapter->pdev->dev, "EEH reset\n");
5202
5203         status = pci_enable_device(pdev);
5204         if (status)
5205                 return PCI_ERS_RESULT_DISCONNECT;
5206
5207         pci_set_master(pdev);
5208         pci_set_power_state(pdev, PCI_D0);
5209         pci_restore_state(pdev);
5210
5211         /* Check if card is ok and fw is ready */
5212         dev_info(&adapter->pdev->dev,
5213                  "Waiting for FW to be ready after EEH reset\n");
5214         status = be_fw_wait_ready(adapter);
5215         if (status)
5216                 return PCI_ERS_RESULT_DISCONNECT;
5217
5218         pci_cleanup_aer_uncorrect_error_status(pdev);
5219         be_clear_all_error(adapter);
5220         return PCI_ERS_RESULT_RECOVERED;
5221 }
5222
5223 static void be_eeh_resume(struct pci_dev *pdev)
5224 {
5225         int status = 0;
5226         struct be_adapter *adapter = pci_get_drvdata(pdev);
5227         struct net_device *netdev =  adapter->netdev;
5228
5229         dev_info(&adapter->pdev->dev, "EEH resume\n");
5230
5231         pci_save_state(pdev);
5232
5233         status = be_cmd_reset_function(adapter);
5234         if (status)
5235                 goto err;
5236
5237         /* On some BE3 FW versions, after a HW reset,
5238          * interrupts will remain disabled for each function.
5239          * So, explicitly enable interrupts
5240          */
5241         be_intr_set(adapter, true);
5242
5243         /* tell fw we're ready to fire cmds */
5244         status = be_cmd_fw_init(adapter);
5245         if (status)
5246                 goto err;
5247
5248         status = be_setup(adapter);
5249         if (status)
5250                 goto err;
5251
5252         if (netif_running(netdev)) {
5253                 status = be_open(netdev);
5254                 if (status)
5255                         goto err;
5256         }
5257
5258         schedule_delayed_work(&adapter->func_recovery_work,
5259                               msecs_to_jiffies(1000));
5260         netif_device_attach(netdev);
5261         return;
5262 err:
5263         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5264 }
5265
5266 static const struct pci_error_handlers be_eeh_handlers = {
5267         .error_detected = be_eeh_err_detected,
5268         .slot_reset = be_eeh_reset,
5269         .resume = be_eeh_resume,
5270 };
5271
5272 static struct pci_driver be_driver = {
5273         .name = DRV_NAME,
5274         .id_table = be_dev_ids,
5275         .probe = be_probe,
5276         .remove = be_remove,
5277         .suspend = be_suspend,
5278         .resume = be_resume,
5279         .shutdown = be_shutdown,
5280         .err_handler = &be_eeh_handlers
5281 };
5282
5283 static int __init be_init_module(void)
5284 {
5285         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5286             rx_frag_size != 2048) {
5287                 printk(KERN_WARNING DRV_NAME
5288                         " : Module param rx_frag_size must be 2048/4096/8192."
5289                         " Using 2048\n");
5290                 rx_frag_size = 2048;
5291         }
5292
5293         return pci_register_driver(&be_driver);
5294 }
5295 module_init(be_init_module);
5296
5297 static void __exit be_exit_module(void)
5298 {
5299         pci_unregister_driver(&be_driver);
5300 }
5301 module_exit(be_exit_module);