drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW)
  73
  74 MODULE_LICENSE("GPL");
  75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  76
  77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  79 #define BNXT_RX_COPY_THRESH 256
  80
  81 #define BNXT_TX_PUSH_THRESH 164
  82
  83 enum board_idx {
  84         BCM57301,
  85         BCM57302,
  86         BCM57304,
  87         BCM57417_NPAR,
  88         BCM58700,
  89         BCM57311,
  90         BCM57312,
  91         BCM57402,
  92         BCM57404,
  93         BCM57406,
  94         BCM57402_NPAR,
  95         BCM57407,
  96         BCM57412,
  97         BCM57414,
  98         BCM57416,
  99         BCM57417,
 100         BCM57412_NPAR,
 101         BCM57314,
 102         BCM57417_SFP,
 103         BCM57416_SFP,
 104         BCM57404_NPAR,
 105         BCM57406_NPAR,
 106         BCM57407_SFP,
 107         BCM57407_NPAR,
 108         BCM57414_NPAR,
 109         BCM57416_NPAR,
 110         BCM57452,
 111         BCM57454,
 112         BCM5745x_NPAR,
 113         BCM57508,
 114         BCM57504,
 115         BCM57502,
 116         BCM57508_NPAR,
 117         BCM57504_NPAR,
 118         BCM57502_NPAR,
 119         BCM58802,
 120         BCM58804,
 121         BCM58808,
 122         NETXTREME_E_VF,
 123         NETXTREME_C_VF,
 124         NETXTREME_S_VF,
 125         NETXTREME_E_P5_VF,
 126 };
 127
 128 /* indexed by enum above */
 129 static const struct {
 130         char *name;
 131 } board_info[] = {
 132         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 133         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 134         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 135         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 136         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 137         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 138         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 139         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 140         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 141         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 142         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 143         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 144         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 145         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 146         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 148         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 149         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 150         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 151         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 152         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 153         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 154         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 155         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 156         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 157         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 158         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 159         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 160         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 161         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 162         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 163         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 164         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 165         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 166         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 167         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 168         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 169         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 170         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 171         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 172         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 173         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 174 };
 175
 176 static const struct pci_device_id bnxt_pci_tbl[] = {
 177         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 180         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 182         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 183         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 184         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 185         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 186         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 187         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 188         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 189         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 190         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 191         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 192         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 195         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 196         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 197         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 198         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 199         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 200         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 201         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 202         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 203         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 204         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 207         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 210         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 211         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 212         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 213         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 214         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 215         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 216         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 217         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 218         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 221         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 222         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 223 #ifdef CONFIG_BNXT_SRIOV
 224         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 225         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 226         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 227         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 233         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 234         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 235 #endif
 236         { 0 }
 237 };
 238
 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 240
 241 static const u16 bnxt_vf_req_snif[] = {
 242         HWRM_FUNC_CFG,
 243         HWRM_FUNC_VF_CFG,
 244         HWRM_PORT_PHY_QCFG,
 245         HWRM_CFA_L2_FILTER_ALLOC,
 246 };
 247
 248 static const u16 bnxt_async_events_arr[] = {
 249         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 250         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 251         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 252         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 253         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 254         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 255         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 256         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 257         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 258         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 259 };
 260
 261 static struct workqueue_struct *bnxt_pf_wq;
 262
 263 static bool bnxt_vf_pciid(enum board_idx idx)
 264 {
 265         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 266                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 267 }
 268
 269 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 270 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 271 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 272
 273 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 274                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 275
 276 #define BNXT_DB_CQ(db, idx)                                             \
 277         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 278
 279 #define BNXT_DB_NQ_P5(db, idx)                                          \
 280         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 281
 282 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 283         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 284
 285 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 286         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 287
 288 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 289 {
 290         if (bp->flags & BNXT_FLAG_CHIP_P5)
 291                 BNXT_DB_NQ_P5(db, idx);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 297 {
 298         if (bp->flags & BNXT_FLAG_CHIP_P5)
 299                 BNXT_DB_NQ_ARM_P5(db, idx);
 300         else
 301                 BNXT_DB_CQ_ARM(db, idx);
 302 }
 303
 304 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 305 {
 306         if (bp->flags & BNXT_FLAG_CHIP_P5)
 307                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 308                        db->doorbell);
 309         else
 310                 BNXT_DB_CQ(db, idx);
 311 }
 312
 313 const u16 bnxt_lhint_arr[] = {
 314         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 315         TX_BD_FLAGS_LHINT_512_TO_1023,
 316         TX_BD_FLAGS_LHINT_1024_TO_2047,
 317         TX_BD_FLAGS_LHINT_1024_TO_2047,
 318         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 319         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 332         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 333 };
 334
 335 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 336 {
 337         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 338
 339         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 340                 return 0;
 341
 342         return md_dst->u.port_info.port_id;
 343 }
 344
 345 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 346 {
 347         struct bnxt *bp = netdev_priv(dev);
 348         struct tx_bd *txbd;
 349         struct tx_bd_ext *txbd1;
 350         struct netdev_queue *txq;
 351         int i;
 352         dma_addr_t mapping;
 353         unsigned int length, pad = 0;
 354         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 355         u16 prod, last_frag;
 356         struct pci_dev *pdev = bp->pdev;
 357         struct bnxt_tx_ring_info *txr;
 358         struct bnxt_sw_tx_bd *tx_buf;
 359
 360         i = skb_get_queue_mapping(skb);
 361         if (unlikely(i >= bp->tx_nr_rings)) {
 362                 dev_kfree_skb_any(skb);
 363                 return NETDEV_TX_OK;
 364         }
 365
 366         txq = netdev_get_tx_queue(dev, i);
 367         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 368         prod = txr->tx_prod;
 369
 370         free_size = bnxt_tx_avail(bp, txr);
 371         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 372                 netif_tx_stop_queue(txq);
 373                 return NETDEV_TX_BUSY;
 374         }
 375
 376         length = skb->len;
 377         len = skb_headlen(skb);
 378         last_frag = skb_shinfo(skb)->nr_frags;
 379
 380         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381
 382         txbd->tx_bd_opaque = prod;
 383
 384         tx_buf = &txr->tx_buf_ring[prod];
 385         tx_buf->skb = skb;
 386         tx_buf->nr_frags = last_frag;
 387
 388         vlan_tag_flags = 0;
 389         cfa_action = bnxt_xmit_get_cfa_action(skb);
 390         if (skb_vlan_tag_present(skb)) {
 391                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 392                                  skb_vlan_tag_get(skb);
 393                 /* Currently supports 8021Q, 8021AD vlan offloads
 394                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 395                  */
 396                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 397                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 398         }
 399
 400         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 401                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 402                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 403                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 404                 void __iomem *db = txr->tx_db.doorbell;
 405                 void *pdata = tx_push_buf->data;
 406                 u64 *end;
 407                 int j, push_len;
 408
 409                 /* Set COAL_NOW to be ready quickly for the next push */
 410                 tx_push->tx_bd_len_flags_type =
 411                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 412                                         TX_BD_TYPE_LONG_TX_BD |
 413                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 414                                         TX_BD_FLAGS_COAL_NOW |
 415                                         TX_BD_FLAGS_PACKET_END |
 416                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 417
 418                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 419                         tx_push1->tx_bd_hsize_lflags =
 420                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 421                 else
 422                         tx_push1->tx_bd_hsize_lflags = 0;
 423
 424                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 425                 tx_push1->tx_bd_cfa_action =
 426                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 427
 428                 end = pdata + length;
 429                 end = PTR_ALIGN(end, 8) - 1;
 430                 *end = 0;
 431
 432                 skb_copy_from_linear_data(skb, pdata, len);
 433                 pdata += len;
 434                 for (j = 0; j < last_frag; j++) {
 435                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 436                         void *fptr;
 437
 438                         fptr = skb_frag_address_safe(frag);
 439                         if (!fptr)
 440                                 goto normal_tx;
 441
 442                         memcpy(pdata, fptr, skb_frag_size(frag));
 443                         pdata += skb_frag_size(frag);
 444                 }
 445
 446                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 447                 txbd->tx_bd_haddr = txr->data_mapping;
 448                 prod = NEXT_TX(prod);
 449                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 450                 memcpy(txbd, tx_push1, sizeof(*txbd));
 451                 prod = NEXT_TX(prod);
 452                 tx_push->doorbell =
 453                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 454                 txr->tx_prod = prod;
 455
 456                 tx_buf->is_push = 1;
 457                 netdev_tx_sent_queue(txq, skb->len);
 458                 wmb();  /* Sync is_push and byte queue before pushing data */
 459
 460                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 461                 if (push_len > 16) {
 462                         __iowrite64_copy(db, tx_push_buf, 16);
 463                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 464                                          (push_len - 16) << 1);
 465                 } else {
 466                         __iowrite64_copy(db, tx_push_buf, push_len);
 467                 }
 468
 469                 goto tx_done;
 470         }
 471
 472 normal_tx:
 473         if (length < BNXT_MIN_PKT_SIZE) {
 474                 pad = BNXT_MIN_PKT_SIZE - length;
 475                 if (skb_pad(skb, pad)) {
 476                         /* SKB already freed. */
 477                         tx_buf->skb = NULL;
 478                         return NETDEV_TX_OK;
 479                 }
 480                 length = BNXT_MIN_PKT_SIZE;
 481         }
 482
 483         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 484
 485         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 486                 dev_kfree_skb_any(skb);
 487                 tx_buf->skb = NULL;
 488                 return NETDEV_TX_OK;
 489         }
 490
 491         dma_unmap_addr_set(tx_buf, mapping, mapping);
 492         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 493                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 494
 495         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 496
 497         prod = NEXT_TX(prod);
 498         txbd1 = (struct tx_bd_ext *)
 499                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 500
 501         txbd1->tx_bd_hsize_lflags = 0;
 502         if (skb_is_gso(skb)) {
 503                 u32 hdr_len;
 504
 505                 if (skb->encapsulation)
 506                         hdr_len = skb_inner_network_offset(skb) +
 507                                 skb_inner_network_header_len(skb) +
 508                                 inner_tcp_hdrlen(skb);
 509                 else
 510                         hdr_len = skb_transport_offset(skb) +
 511                                 tcp_hdrlen(skb);
 512
 513                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 514                                         TX_BD_FLAGS_T_IPID |
 515                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 516                 length = skb_shinfo(skb)->gso_size;
 517                 txbd1->tx_bd_mss = cpu_to_le32(length);
 518                 length += hdr_len;
 519         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 520                 txbd1->tx_bd_hsize_lflags =
 521                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 522                 txbd1->tx_bd_mss = 0;
 523         }
 524
 525         length >>= 9;
 526         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 527                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 528                                      skb->len);
 529                 i = 0;
 530                 goto tx_dma_error;
 531         }
 532         flags |= bnxt_lhint_arr[length];
 533         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 534
 535         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 536         txbd1->tx_bd_cfa_action =
 537                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 538         for (i = 0; i < last_frag; i++) {
 539                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 540
 541                 prod = NEXT_TX(prod);
 542                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 543
 544                 len = skb_frag_size(frag);
 545                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 546                                            DMA_TO_DEVICE);
 547
 548                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 549                         goto tx_dma_error;
 550
 551                 tx_buf = &txr->tx_buf_ring[prod];
 552                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 553
 554                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 555
 556                 flags = len << TX_BD_LEN_SHIFT;
 557                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 558         }
 559
 560         flags &= ~TX_BD_LEN;
 561         txbd->tx_bd_len_flags_type =
 562                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 563                             TX_BD_FLAGS_PACKET_END);
 564
 565         netdev_tx_sent_queue(txq, skb->len);
 566
 567         /* Sync BD data before updating doorbell */
 568         wmb();
 569
 570         prod = NEXT_TX(prod);
 571         txr->tx_prod = prod;
 572
 573         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 574                 bnxt_db_write(bp, &txr->tx_db, prod);
 575
 576 tx_done:
 577
 578         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 579                 if (netdev_xmit_more() && !tx_buf->is_push)
 580                         bnxt_db_write(bp, &txr->tx_db, prod);
 581
 582                 netif_tx_stop_queue(txq);
 583
 584                 /* netif_tx_stop_queue() must be done before checking
 585                  * tx index in bnxt_tx_avail() below, because in
 586                  * bnxt_tx_int(), we update tx index before checking for
 587                  * netif_tx_queue_stopped().
 588                  */
 589                 smp_mb();
 590                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 591                         netif_tx_wake_queue(txq);
 592         }
 593         return NETDEV_TX_OK;
 594
 595 tx_dma_error:
 596         last_frag = i;
 597
 598         /* start back at beginning and unmap skb */
 599         prod = txr->tx_prod;
 600         tx_buf = &txr->tx_buf_ring[prod];
 601         tx_buf->skb = NULL;
 602         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 603                          skb_headlen(skb), PCI_DMA_TODEVICE);
 604         prod = NEXT_TX(prod);
 605
 606         /* unmap remaining mapped pages */
 607         for (i = 0; i < last_frag; i++) {
 608                 prod = NEXT_TX(prod);
 609                 tx_buf = &txr->tx_buf_ring[prod];
 610                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 611                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 612                                PCI_DMA_TODEVICE);
 613         }
 614
 615         dev_kfree_skb_any(skb);
 616         return NETDEV_TX_OK;
 617 }
 618
 619 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 620 {
 621         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 622         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 623         u16 cons = txr->tx_cons;
 624         struct pci_dev *pdev = bp->pdev;
 625         int i;
 626         unsigned int tx_bytes = 0;
 627
 628         for (i = 0; i < nr_pkts; i++) {
 629                 struct bnxt_sw_tx_bd *tx_buf;
 630                 struct sk_buff *skb;
 631                 int j, last;
 632
 633                 tx_buf = &txr->tx_buf_ring[cons];
 634                 cons = NEXT_TX(cons);
 635                 skb = tx_buf->skb;
 636                 tx_buf->skb = NULL;
 637
 638                 if (tx_buf->is_push) {
 639                         tx_buf->is_push = 0;
 640                         goto next_tx_int;
 641                 }
 642
 643                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 644                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 645                 last = tx_buf->nr_frags;
 646
 647                 for (j = 0; j < last; j++) {
 648                         cons = NEXT_TX(cons);
 649                         tx_buf = &txr->tx_buf_ring[cons];
 650                         dma_unmap_page(
 651                                 &pdev->dev,
 652                                 dma_unmap_addr(tx_buf, mapping),
 653                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 654                                 PCI_DMA_TODEVICE);
 655                 }
 656
 657 next_tx_int:
 658                 cons = NEXT_TX(cons);
 659
 660                 tx_bytes += skb->len;
 661                 dev_kfree_skb_any(skb);
 662         }
 663
 664         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 665         txr->tx_cons = cons;
 666
 667         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 668          * before checking for netif_tx_queue_stopped().  Without the
 669          * memory barrier, there is a small possibility that bnxt_start_xmit()
 670          * will miss it and cause the queue to be stopped forever.
 671          */
 672         smp_mb();
 673
 674         if (unlikely(netif_tx_queue_stopped(txq)) &&
 675             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 676                 __netif_tx_lock(txq, smp_processor_id());
 677                 if (netif_tx_queue_stopped(txq) &&
 678                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 679                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 680                         netif_tx_wake_queue(txq);
 681                 __netif_tx_unlock(txq);
 682         }
 683 }
 684
 685 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 686                                          struct bnxt_rx_ring_info *rxr,
 687                                          gfp_t gfp)
 688 {
 689         struct device *dev = &bp->pdev->dev;
 690         struct page *page;
 691
 692         page = page_pool_dev_alloc_pages(rxr->page_pool);
 693         if (!page)
 694                 return NULL;
 695
 696         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 697                                       DMA_ATTR_WEAK_ORDERING);
 698         if (dma_mapping_error(dev, *mapping)) {
 699                 page_pool_recycle_direct(rxr->page_pool, page);
 700                 return NULL;
 701         }
 702         *mapping += bp->rx_dma_offset;
 703         return page;
 704 }
 705
 706 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 707                                        gfp_t gfp)
 708 {
 709         u8 *data;
 710         struct pci_dev *pdev = bp->pdev;
 711
 712         data = kmalloc(bp->rx_buf_size, gfp);
 713         if (!data)
 714                 return NULL;
 715
 716         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 717                                         bp->rx_buf_use_size, bp->rx_dir,
 718                                         DMA_ATTR_WEAK_ORDERING);
 719
 720         if (dma_mapping_error(&pdev->dev, *mapping)) {
 721                 kfree(data);
 722                 data = NULL;
 723         }
 724         return data;
 725 }
 726
 727 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 728                        u16 prod, gfp_t gfp)
 729 {
 730         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 731         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 732         dma_addr_t mapping;
 733
 734         if (BNXT_RX_PAGE_MODE(bp)) {
 735                 struct page *page =
 736                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 737
 738                 if (!page)
 739                         return -ENOMEM;
 740
 741                 rx_buf->data = page;
 742                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 743         } else {
 744                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 745
 746                 if (!data)
 747                         return -ENOMEM;
 748
 749                 rx_buf->data = data;
 750                 rx_buf->data_ptr = data + bp->rx_offset;
 751         }
 752         rx_buf->mapping = mapping;
 753
 754         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 755         return 0;
 756 }
 757
 758 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 759 {
 760         u16 prod = rxr->rx_prod;
 761         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 762         struct rx_bd *cons_bd, *prod_bd;
 763
 764         prod_rx_buf = &rxr->rx_buf_ring[prod];
 765         cons_rx_buf = &rxr->rx_buf_ring[cons];
 766
 767         prod_rx_buf->data = data;
 768         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 769
 770         prod_rx_buf->mapping = cons_rx_buf->mapping;
 771
 772         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 773         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 774
 775         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 776 }
 777
 778 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 779 {
 780         u16 next, max = rxr->rx_agg_bmap_size;
 781
 782         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 783         if (next >= max)
 784                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 785         return next;
 786 }
 787
 788 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 789                                      struct bnxt_rx_ring_info *rxr,
 790                                      u16 prod, gfp_t gfp)
 791 {
 792         struct rx_bd *rxbd =
 793                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 794         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 795         struct pci_dev *pdev = bp->pdev;
 796         struct page *page;
 797         dma_addr_t mapping;
 798         u16 sw_prod = rxr->rx_sw_agg_prod;
 799         unsigned int offset = 0;
 800
 801         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 802                 page = rxr->rx_page;
 803                 if (!page) {
 804                         page = alloc_page(gfp);
 805                         if (!page)
 806                                 return -ENOMEM;
 807                         rxr->rx_page = page;
 808                         rxr->rx_page_offset = 0;
 809                 }
 810                 offset = rxr->rx_page_offset;
 811                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 812                 if (rxr->rx_page_offset == PAGE_SIZE)
 813                         rxr->rx_page = NULL;
 814                 else
 815                         get_page(page);
 816         } else {
 817                 page = alloc_page(gfp);
 818                 if (!page)
 819                         return -ENOMEM;
 820         }
 821
 822         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 823                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 824                                      DMA_ATTR_WEAK_ORDERING);
 825         if (dma_mapping_error(&pdev->dev, mapping)) {
 826                 __free_page(page);
 827                 return -EIO;
 828         }
 829
 830         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 831                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 832
 833         __set_bit(sw_prod, rxr->rx_agg_bmap);
 834         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 835         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 836
 837         rx_agg_buf->page = page;
 838         rx_agg_buf->offset = offset;
 839         rx_agg_buf->mapping = mapping;
 840         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 841         rxbd->rx_bd_opaque = sw_prod;
 842         return 0;
 843 }
 844
 845 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 846                                        struct bnxt_cp_ring_info *cpr,
 847                                        u16 cp_cons, u16 curr)
 848 {
 849         struct rx_agg_cmp *agg;
 850
 851         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 852         agg = (struct rx_agg_cmp *)
 853                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 854         return agg;
 855 }
 856
 857 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 858                                               struct bnxt_rx_ring_info *rxr,
 859                                               u16 agg_id, u16 curr)
 860 {
 861         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 862
 863         return &tpa_info->agg_arr[curr];
 864 }
 865
 866 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 867                                    u16 start, u32 agg_bufs, bool tpa)
 868 {
 869         struct bnxt_napi *bnapi = cpr->bnapi;
 870         struct bnxt *bp = bnapi->bp;
 871         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 872         u16 prod = rxr->rx_agg_prod;
 873         u16 sw_prod = rxr->rx_sw_agg_prod;
 874         bool p5_tpa = false;
 875         u32 i;
 876
 877         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 878                 p5_tpa = true;
 879
 880         for (i = 0; i < agg_bufs; i++) {
 881                 u16 cons;
 882                 struct rx_agg_cmp *agg;
 883                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 884                 struct rx_bd *prod_bd;
 885                 struct page *page;
 886
 887                 if (p5_tpa)
 888                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 889                 else
 890                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 891                 cons = agg->rx_agg_cmp_opaque;
 892                 __clear_bit(cons, rxr->rx_agg_bmap);
 893
 894                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 895                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 896
 897                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 898                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 899                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 900
 901                 /* It is possible for sw_prod to be equal to cons, so
 902                  * set cons_rx_buf->page to NULL first.
 903                  */
 904                 page = cons_rx_buf->page;
 905                 cons_rx_buf->page = NULL;
 906                 prod_rx_buf->page = page;
 907                 prod_rx_buf->offset = cons_rx_buf->offset;
 908
 909                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 910
 911                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 912
 913                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 914                 prod_bd->rx_bd_opaque = sw_prod;
 915
 916                 prod = NEXT_RX_AGG(prod);
 917                 sw_prod = NEXT_RX_AGG(sw_prod);
 918         }
 919         rxr->rx_agg_prod = prod;
 920         rxr->rx_sw_agg_prod = sw_prod;
 921 }
 922
 923 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 924                                         struct bnxt_rx_ring_info *rxr,
 925                                         u16 cons, void *data, u8 *data_ptr,
 926                                         dma_addr_t dma_addr,
 927                                         unsigned int offset_and_len)
 928 {
 929         unsigned int payload = offset_and_len >> 16;
 930         unsigned int len = offset_and_len & 0xffff;
 931         skb_frag_t *frag;
 932         struct page *page = data;
 933         u16 prod = rxr->rx_prod;
 934         struct sk_buff *skb;
 935         int off, err;
 936
 937         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 938         if (unlikely(err)) {
 939                 bnxt_reuse_rx_data(rxr, cons, data);
 940                 return NULL;
 941         }
 942         dma_addr -= bp->rx_dma_offset;
 943         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 944                              DMA_ATTR_WEAK_ORDERING);
 945         page_pool_release_page(rxr->page_pool, page);
 946
 947         if (unlikely(!payload))
 948                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 949
 950         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 951         if (!skb) {
 952                 __free_page(page);
 953                 return NULL;
 954         }
 955
 956         off = (void *)data_ptr - page_address(page);
 957         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 958         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 959                payload + NET_IP_ALIGN);
 960
 961         frag = &skb_shinfo(skb)->frags[0];
 962         skb_frag_size_sub(frag, payload);
 963         skb_frag_off_add(frag, payload);
 964         skb->data_len -= payload;
 965         skb->tail += payload;
 966
 967         return skb;
 968 }
 969
 970 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 971                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 972                                    void *data, u8 *data_ptr,
 973                                    dma_addr_t dma_addr,
 974                                    unsigned int offset_and_len)
 975 {
 976         u16 prod = rxr->rx_prod;
 977         struct sk_buff *skb;
 978         int err;
 979
 980         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 981         if (unlikely(err)) {
 982                 bnxt_reuse_rx_data(rxr, cons, data);
 983                 return NULL;
 984         }
 985
 986         skb = build_skb(data, 0);
 987         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 988                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 989         if (!skb) {
 990                 kfree(data);
 991                 return NULL;
 992         }
 993
 994         skb_reserve(skb, bp->rx_offset);
 995         skb_put(skb, offset_and_len & 0xffff);
 996         return skb;
 997 }
 998
 999 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1000                                      struct bnxt_cp_ring_info *cpr,
1001                                      struct sk_buff *skb, u16 idx,
1002                                      u32 agg_bufs, bool tpa)
1003 {
1004         struct bnxt_napi *bnapi = cpr->bnapi;
1005         struct pci_dev *pdev = bp->pdev;
1006         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1007         u16 prod = rxr->rx_agg_prod;
1008         bool p5_tpa = false;
1009         u32 i;
1010
1011         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1012                 p5_tpa = true;
1013
1014         for (i = 0; i < agg_bufs; i++) {
1015                 u16 cons, frag_len;
1016                 struct rx_agg_cmp *agg;
1017                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1018                 struct page *page;
1019                 dma_addr_t mapping;
1020
1021                 if (p5_tpa)
1022                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1023                 else
1024                         agg = bnxt_get_agg(bp, cpr, idx, i);
1025                 cons = agg->rx_agg_cmp_opaque;
1026                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1027                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1028
1029                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1030                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1031                                    cons_rx_buf->offset, frag_len);
1032                 __clear_bit(cons, rxr->rx_agg_bmap);
1033
1034                 /* It is possible for bnxt_alloc_rx_page() to allocate
1035                  * a sw_prod index that equals the cons index, so we
1036                  * need to clear the cons entry now.
1037                  */
1038                 mapping = cons_rx_buf->mapping;
1039                 page = cons_rx_buf->page;
1040                 cons_rx_buf->page = NULL;
1041
1042                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1043                         struct skb_shared_info *shinfo;
1044                         unsigned int nr_frags;
1045
1046                         shinfo = skb_shinfo(skb);
1047                         nr_frags = --shinfo->nr_frags;
1048                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1049
1050                         dev_kfree_skb(skb);
1051
1052                         cons_rx_buf->page = page;
1053
1054                         /* Update prod since possibly some pages have been
1055                          * allocated already.
1056                          */
1057                         rxr->rx_agg_prod = prod;
1058                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1059                         return NULL;
1060                 }
1061
1062                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1063                                      PCI_DMA_FROMDEVICE,
1064                                      DMA_ATTR_WEAK_ORDERING);
1065
1066                 skb->data_len += frag_len;
1067                 skb->len += frag_len;
1068                 skb->truesize += PAGE_SIZE;
1069
1070                 prod = NEXT_RX_AGG(prod);
1071         }
1072         rxr->rx_agg_prod = prod;
1073         return skb;
1074 }
1075
1076 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1077                                u8 agg_bufs, u32 *raw_cons)
1078 {
1079         u16 last;
1080         struct rx_agg_cmp *agg;
1081
1082         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1083         last = RING_CMP(*raw_cons);
1084         agg = (struct rx_agg_cmp *)
1085                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1086         return RX_AGG_CMP_VALID(agg, *raw_cons);
1087 }
1088
1089 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1090                                             unsigned int len,
1091                                             dma_addr_t mapping)
1092 {
1093         struct bnxt *bp = bnapi->bp;
1094         struct pci_dev *pdev = bp->pdev;
1095         struct sk_buff *skb;
1096
1097         skb = napi_alloc_skb(&bnapi->napi, len);
1098         if (!skb)
1099                 return NULL;
1100
1101         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1102                                 bp->rx_dir);
1103
1104         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1105                len + NET_IP_ALIGN);
1106
1107         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1108                                    bp->rx_dir);
1109
1110         skb_put(skb, len);
1111         return skb;
1112 }
1113
1114 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1115                            u32 *raw_cons, void *cmp)
1116 {
1117         struct rx_cmp *rxcmp = cmp;
1118         u32 tmp_raw_cons = *raw_cons;
1119         u8 cmp_type, agg_bufs = 0;
1120
1121         cmp_type = RX_CMP_TYPE(rxcmp);
1122
1123         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1124                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1125                             RX_CMP_AGG_BUFS) >>
1126                            RX_CMP_AGG_BUFS_SHIFT;
1127         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1128                 struct rx_tpa_end_cmp *tpa_end = cmp;
1129
1130                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1131                         return 0;
1132
1133                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1134         }
1135
1136         if (agg_bufs) {
1137                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1138                         return -EBUSY;
1139         }
1140         *raw_cons = tmp_raw_cons;
1141         return 0;
1142 }
1143
1144 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1145 {
1146         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1147                 return;
1148
1149         if (BNXT_PF(bp))
1150                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1151         else
1152                 schedule_delayed_work(&bp->fw_reset_task, delay);
1153 }
1154
1155 static void bnxt_queue_sp_work(struct bnxt *bp)
1156 {
1157         if (BNXT_PF(bp))
1158                 queue_work(bnxt_pf_wq, &bp->sp_task);
1159         else
1160                 schedule_work(&bp->sp_task);
1161 }
1162
1163 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1164 {
1165         if (!rxr->bnapi->in_reset) {
1166                 rxr->bnapi->in_reset = true;
1167                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1168                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1169                 else
1170                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1171                 bnxt_queue_sp_work(bp);
1172         }
1173         rxr->rx_next_cons = 0xffff;
1174 }
1175
1176 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1177 {
1178         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1179         u16 idx = agg_id & MAX_TPA_P5_MASK;
1180
1181         if (test_bit(idx, map->agg_idx_bmap))
1182                 idx = find_first_zero_bit(map->agg_idx_bmap,
1183                                           BNXT_AGG_IDX_BMAP_SIZE);
1184         __set_bit(idx, map->agg_idx_bmap);
1185         map->agg_id_tbl[agg_id] = idx;
1186         return idx;
1187 }
1188
1189 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1190 {
1191         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1192
1193         __clear_bit(idx, map->agg_idx_bmap);
1194 }
1195
1196 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1197 {
1198         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1199
1200         return map->agg_id_tbl[agg_id];
1201 }
1202
1203 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1204                            struct rx_tpa_start_cmp *tpa_start,
1205                            struct rx_tpa_start_cmp_ext *tpa_start1)
1206 {
1207         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1208         struct bnxt_tpa_info *tpa_info;
1209         u16 cons, prod, agg_id;
1210         struct rx_bd *prod_bd;
1211         dma_addr_t mapping;
1212
1213         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1214                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1215                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1216         } else {
1217                 agg_id = TPA_START_AGG_ID(tpa_start);
1218         }
1219         cons = tpa_start->rx_tpa_start_cmp_opaque;
1220         prod = rxr->rx_prod;
1221         cons_rx_buf = &rxr->rx_buf_ring[cons];
1222         prod_rx_buf = &rxr->rx_buf_ring[prod];
1223         tpa_info = &rxr->rx_tpa[agg_id];
1224
1225         if (unlikely(cons != rxr->rx_next_cons ||
1226                      TPA_START_ERROR(tpa_start))) {
1227                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1228                             cons, rxr->rx_next_cons,
1229                             TPA_START_ERROR_CODE(tpa_start1));
1230                 bnxt_sched_reset(bp, rxr);
1231                 return;
1232         }
1233         /* Store cfa_code in tpa_info to use in tpa_end
1234          * completion processing.
1235          */
1236         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1237         prod_rx_buf->data = tpa_info->data;
1238         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1239
1240         mapping = tpa_info->mapping;
1241         prod_rx_buf->mapping = mapping;
1242
1243         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1244
1245         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1246
1247         tpa_info->data = cons_rx_buf->data;
1248         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1249         cons_rx_buf->data = NULL;
1250         tpa_info->mapping = cons_rx_buf->mapping;
1251
1252         tpa_info->len =
1253                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1254                                 RX_TPA_START_CMP_LEN_SHIFT;
1255         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1256                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1257
1258                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1259                 tpa_info->gso_type = SKB_GSO_TCPV4;
1260                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1261                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1262                         tpa_info->gso_type = SKB_GSO_TCPV6;
1263                 tpa_info->rss_hash =
1264                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1265         } else {
1266                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1267                 tpa_info->gso_type = 0;
1268                 if (netif_msg_rx_err(bp))
1269                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1270         }
1271         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1272         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1273         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1274         tpa_info->agg_count = 0;
1275
1276         rxr->rx_prod = NEXT_RX(prod);
1277         cons = NEXT_RX(cons);
1278         rxr->rx_next_cons = NEXT_RX(cons);
1279         cons_rx_buf = &rxr->rx_buf_ring[cons];
1280
1281         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1282         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1283         cons_rx_buf->data = NULL;
1284 }
1285
1286 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1287 {
1288         if (agg_bufs)
1289                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1290 }
1291
1292 #ifdef CONFIG_INET
1293 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1294 {
1295         struct udphdr *uh = NULL;
1296
1297         if (ip_proto == htons(ETH_P_IP)) {
1298                 struct iphdr *iph = (struct iphdr *)skb->data;
1299
1300                 if (iph->protocol == IPPROTO_UDP)
1301                         uh = (struct udphdr *)(iph + 1);
1302         } else {
1303                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1304
1305                 if (iph->nexthdr == IPPROTO_UDP)
1306                         uh = (struct udphdr *)(iph + 1);
1307         }
1308         if (uh) {
1309                 if (uh->check)
1310                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1311                 else
1312                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1313         }
1314 }
1315 #endif
1316
1317 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1318                                            int payload_off, int tcp_ts,
1319                                            struct sk_buff *skb)
1320 {
1321 #ifdef CONFIG_INET
1322         struct tcphdr *th;
1323         int len, nw_off;
1324         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1325         u32 hdr_info = tpa_info->hdr_info;
1326         bool loopback = false;
1327
1328         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1329         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1330         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1331
1332         /* If the packet is an internal loopback packet, the offsets will
1333          * have an extra 4 bytes.
1334          */
1335         if (inner_mac_off == 4) {
1336                 loopback = true;
1337         } else if (inner_mac_off > 4) {
1338                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1339                                             ETH_HLEN - 2));
1340
1341                 /* We only support inner iPv4/ipv6.  If we don't see the
1342                  * correct protocol ID, it must be a loopback packet where
1343                  * the offsets are off by 4.
1344                  */
1345                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1346                         loopback = true;
1347         }
1348         if (loopback) {
1349                 /* internal loopback packet, subtract all offsets by 4 */
1350                 inner_ip_off -= 4;
1351                 inner_mac_off -= 4;
1352                 outer_ip_off -= 4;
1353         }
1354
1355         nw_off = inner_ip_off - ETH_HLEN;
1356         skb_set_network_header(skb, nw_off);
1357         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1358                 struct ipv6hdr *iph = ipv6_hdr(skb);
1359
1360                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1361                 len = skb->len - skb_transport_offset(skb);
1362                 th = tcp_hdr(skb);
1363                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1364         } else {
1365                 struct iphdr *iph = ip_hdr(skb);
1366
1367                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1368                 len = skb->len - skb_transport_offset(skb);
1369                 th = tcp_hdr(skb);
1370                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1371         }
1372
1373         if (inner_mac_off) { /* tunnel */
1374                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1375                                             ETH_HLEN - 2));
1376
1377                 bnxt_gro_tunnel(skb, proto);
1378         }
1379 #endif
1380         return skb;
1381 }
1382
1383 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1384                                            int payload_off, int tcp_ts,
1385                                            struct sk_buff *skb)
1386 {
1387 #ifdef CONFIG_INET
1388         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1389         u32 hdr_info = tpa_info->hdr_info;
1390         int iphdr_len, nw_off;
1391
1392         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1393         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1394         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1395
1396         nw_off = inner_ip_off - ETH_HLEN;
1397         skb_set_network_header(skb, nw_off);
1398         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1399                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1400         skb_set_transport_header(skb, nw_off + iphdr_len);
1401
1402         if (inner_mac_off) { /* tunnel */
1403                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1404                                             ETH_HLEN - 2));
1405
1406                 bnxt_gro_tunnel(skb, proto);
1407         }
1408 #endif
1409         return skb;
1410 }
1411
1412 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1413 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1414
1415 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1416                                            int payload_off, int tcp_ts,
1417                                            struct sk_buff *skb)
1418 {
1419 #ifdef CONFIG_INET
1420         struct tcphdr *th;
1421         int len, nw_off, tcp_opt_len = 0;
1422
1423         if (tcp_ts)
1424                 tcp_opt_len = 12;
1425
1426         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1427                 struct iphdr *iph;
1428
1429                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1430                          ETH_HLEN;
1431                 skb_set_network_header(skb, nw_off);
1432                 iph = ip_hdr(skb);
1433                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1434                 len = skb->len - skb_transport_offset(skb);
1435                 th = tcp_hdr(skb);
1436                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1437         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1438                 struct ipv6hdr *iph;
1439
1440                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1441                          ETH_HLEN;
1442                 skb_set_network_header(skb, nw_off);
1443                 iph = ipv6_hdr(skb);
1444                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1445                 len = skb->len - skb_transport_offset(skb);
1446                 th = tcp_hdr(skb);
1447                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1448         } else {
1449                 dev_kfree_skb_any(skb);
1450                 return NULL;
1451         }
1452
1453         if (nw_off) /* tunnel */
1454                 bnxt_gro_tunnel(skb, skb->protocol);
1455 #endif
1456         return skb;
1457 }
1458
1459 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1460                                            struct bnxt_tpa_info *tpa_info,
1461                                            struct rx_tpa_end_cmp *tpa_end,
1462                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1463                                            struct sk_buff *skb)
1464 {
1465 #ifdef CONFIG_INET
1466         int payload_off;
1467         u16 segs;
1468
1469         segs = TPA_END_TPA_SEGS(tpa_end);
1470         if (segs == 1)
1471                 return skb;
1472
1473         NAPI_GRO_CB(skb)->count = segs;
1474         skb_shinfo(skb)->gso_size =
1475                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1476         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1477         if (bp->flags & BNXT_FLAG_CHIP_P5)
1478                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1479         else
1480                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1481         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1482         if (likely(skb))
1483                 tcp_gro_complete(skb);
1484 #endif
1485         return skb;
1486 }
1487
1488 /* Given the cfa_code of a received packet determine which
1489  * netdev (vf-rep or PF) the packet is destined to.
1490  */
1491 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1492 {
1493         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1494
1495         /* if vf-rep dev is NULL, the must belongs to the PF */
1496         return dev ? dev : bp->dev;
1497 }
1498
1499 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1500                                            struct bnxt_cp_ring_info *cpr,
1501                                            u32 *raw_cons,
1502                                            struct rx_tpa_end_cmp *tpa_end,
1503                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1504                                            u8 *event)
1505 {
1506         struct bnxt_napi *bnapi = cpr->bnapi;
1507         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1508         u8 *data_ptr, agg_bufs;
1509         unsigned int len;
1510         struct bnxt_tpa_info *tpa_info;
1511         dma_addr_t mapping;
1512         struct sk_buff *skb;
1513         u16 idx = 0, agg_id;
1514         void *data;
1515         bool gro;
1516
1517         if (unlikely(bnapi->in_reset)) {
1518                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1519
1520                 if (rc < 0)
1521                         return ERR_PTR(-EBUSY);
1522                 return NULL;
1523         }
1524
1525         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1526                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1527                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1528                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1529                 tpa_info = &rxr->rx_tpa[agg_id];
1530                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1531                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1532                                     agg_bufs, tpa_info->agg_count);
1533                         agg_bufs = tpa_info->agg_count;
1534                 }
1535                 tpa_info->agg_count = 0;
1536                 *event |= BNXT_AGG_EVENT;
1537                 bnxt_free_agg_idx(rxr, agg_id);
1538                 idx = agg_id;
1539                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1540         } else {
1541                 agg_id = TPA_END_AGG_ID(tpa_end);
1542                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1543                 tpa_info = &rxr->rx_tpa[agg_id];
1544                 idx = RING_CMP(*raw_cons);
1545                 if (agg_bufs) {
1546                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1547                                 return ERR_PTR(-EBUSY);
1548
1549                         *event |= BNXT_AGG_EVENT;
1550                         idx = NEXT_CMP(idx);
1551                 }
1552                 gro = !!TPA_END_GRO(tpa_end);
1553         }
1554         data = tpa_info->data;
1555         data_ptr = tpa_info->data_ptr;
1556         prefetch(data_ptr);
1557         len = tpa_info->len;
1558         mapping = tpa_info->mapping;
1559
1560         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1561                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1562                 if (agg_bufs > MAX_SKB_FRAGS)
1563                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1564                                     agg_bufs, (int)MAX_SKB_FRAGS);
1565                 return NULL;
1566         }
1567
1568         if (len <= bp->rx_copy_thresh) {
1569                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1570                 if (!skb) {
1571                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1572                         return NULL;
1573                 }
1574         } else {
1575                 u8 *new_data;
1576                 dma_addr_t new_mapping;
1577
1578                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1579                 if (!new_data) {
1580                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1581                         return NULL;
1582                 }
1583
1584                 tpa_info->data = new_data;
1585                 tpa_info->data_ptr = new_data + bp->rx_offset;
1586                 tpa_info->mapping = new_mapping;
1587
1588                 skb = build_skb(data, 0);
1589                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1590                                        bp->rx_buf_use_size, bp->rx_dir,
1591                                        DMA_ATTR_WEAK_ORDERING);
1592
1593                 if (!skb) {
1594                         kfree(data);
1595                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1596                         return NULL;
1597                 }
1598                 skb_reserve(skb, bp->rx_offset);
1599                 skb_put(skb, len);
1600         }
1601
1602         if (agg_bufs) {
1603                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1604                 if (!skb) {
1605                         /* Page reuse already handled by bnxt_rx_pages(). */
1606                         return NULL;
1607                 }
1608         }
1609
1610         skb->protocol =
1611                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1612
1613         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1614                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1615
1616         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1617             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1618                 u16 vlan_proto = tpa_info->metadata >>
1619                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1620                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1621
1622                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1623         }
1624
1625         skb_checksum_none_assert(skb);
1626         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1627                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1628                 skb->csum_level =
1629                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1630         }
1631
1632         if (gro)
1633                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1634
1635         return skb;
1636 }
1637
1638 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1639                          struct rx_agg_cmp *rx_agg)
1640 {
1641         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1642         struct bnxt_tpa_info *tpa_info;
1643
1644         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1645         tpa_info = &rxr->rx_tpa[agg_id];
1646         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1647         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1648 }
1649
1650 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1651                              struct sk_buff *skb)
1652 {
1653         if (skb->dev != bp->dev) {
1654                 /* this packet belongs to a vf-rep */
1655                 bnxt_vf_rep_rx(bp, skb);
1656                 return;
1657         }
1658         skb_record_rx_queue(skb, bnapi->index);
1659         napi_gro_receive(&bnapi->napi, skb);
1660 }
1661
1662 /* returns the following:
1663  * 1       - 1 packet successfully received
1664  * 0       - successful TPA_START, packet not completed yet
1665  * -EBUSY  - completion ring does not have all the agg buffers yet
1666  * -ENOMEM - packet aborted due to out of memory
1667  * -EIO    - packet aborted due to hw error indicated in BD
1668  */
1669 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1670                        u32 *raw_cons, u8 *event)
1671 {
1672         struct bnxt_napi *bnapi = cpr->bnapi;
1673         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1674         struct net_device *dev = bp->dev;
1675         struct rx_cmp *rxcmp;
1676         struct rx_cmp_ext *rxcmp1;
1677         u32 tmp_raw_cons = *raw_cons;
1678         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1679         struct bnxt_sw_rx_bd *rx_buf;
1680         unsigned int len;
1681         u8 *data_ptr, agg_bufs, cmp_type;
1682         dma_addr_t dma_addr;
1683         struct sk_buff *skb;
1684         void *data;
1685         int rc = 0;
1686         u32 misc;
1687
1688         rxcmp = (struct rx_cmp *)
1689                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1690
1691         cmp_type = RX_CMP_TYPE(rxcmp);
1692
1693         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1694                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1695                 goto next_rx_no_prod_no_len;
1696         }
1697
1698         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1699         cp_cons = RING_CMP(tmp_raw_cons);
1700         rxcmp1 = (struct rx_cmp_ext *)
1701                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1702
1703         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1704                 return -EBUSY;
1705
1706         prod = rxr->rx_prod;
1707
1708         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1709                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1710                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1711
1712                 *event |= BNXT_RX_EVENT;
1713                 goto next_rx_no_prod_no_len;
1714
1715         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1716                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1717                                    (struct rx_tpa_end_cmp *)rxcmp,
1718                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1719
1720                 if (IS_ERR(skb))
1721                         return -EBUSY;
1722
1723                 rc = -ENOMEM;
1724                 if (likely(skb)) {
1725                         bnxt_deliver_skb(bp, bnapi, skb);
1726                         rc = 1;
1727                 }
1728                 *event |= BNXT_RX_EVENT;
1729                 goto next_rx_no_prod_no_len;
1730         }
1731
1732         cons = rxcmp->rx_cmp_opaque;
1733         if (unlikely(cons != rxr->rx_next_cons)) {
1734                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1735
1736                 /* 0xffff is forced error, don't print it */
1737                 if (rxr->rx_next_cons != 0xffff)
1738                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1739                                     cons, rxr->rx_next_cons);
1740                 bnxt_sched_reset(bp, rxr);
1741                 return rc1;
1742         }
1743         rx_buf = &rxr->rx_buf_ring[cons];
1744         data = rx_buf->data;
1745         data_ptr = rx_buf->data_ptr;
1746         prefetch(data_ptr);
1747
1748         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1749         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1750
1751         if (agg_bufs) {
1752                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1753                         return -EBUSY;
1754
1755                 cp_cons = NEXT_CMP(cp_cons);
1756                 *event |= BNXT_AGG_EVENT;
1757         }
1758         *event |= BNXT_RX_EVENT;
1759
1760         rx_buf->data = NULL;
1761         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1762                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1763
1764                 bnxt_reuse_rx_data(rxr, cons, data);
1765                 if (agg_bufs)
1766                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1767                                                false);
1768
1769                 rc = -EIO;
1770                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1771                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1772                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1773                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1774                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1775                                                  rx_err);
1776                                 bnxt_sched_reset(bp, rxr);
1777                         }
1778                 }
1779                 goto next_rx_no_len;
1780         }
1781
1782         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1783         dma_addr = rx_buf->mapping;
1784
1785         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1786                 rc = 1;
1787                 goto next_rx;
1788         }
1789
1790         if (len <= bp->rx_copy_thresh) {
1791                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1792                 bnxt_reuse_rx_data(rxr, cons, data);
1793                 if (!skb) {
1794                         if (agg_bufs)
1795                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1796                                                        agg_bufs, false);
1797                         rc = -ENOMEM;
1798                         goto next_rx;
1799                 }
1800         } else {
1801                 u32 payload;
1802
1803                 if (rx_buf->data_ptr == data_ptr)
1804                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1805                 else
1806                         payload = 0;
1807                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1808                                       payload | len);
1809                 if (!skb) {
1810                         rc = -ENOMEM;
1811                         goto next_rx;
1812                 }
1813         }
1814
1815         if (agg_bufs) {
1816                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1817                 if (!skb) {
1818                         rc = -ENOMEM;
1819                         goto next_rx;
1820                 }
1821         }
1822
1823         if (RX_CMP_HASH_VALID(rxcmp)) {
1824                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1825                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1826
1827                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1828                 if (hash_type != 1 && hash_type != 3)
1829                         type = PKT_HASH_TYPE_L3;
1830                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1831         }
1832
1833         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1834         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1835
1836         if ((rxcmp1->rx_cmp_flags2 &
1837              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1838             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1839                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1840                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1841                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1842
1843                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1844         }
1845
1846         skb_checksum_none_assert(skb);
1847         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1848                 if (dev->features & NETIF_F_RXCSUM) {
1849                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1850                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1851                 }
1852         } else {
1853                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1854                         if (dev->features & NETIF_F_RXCSUM)
1855                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1856                 }
1857         }
1858
1859         bnxt_deliver_skb(bp, bnapi, skb);
1860         rc = 1;
1861
1862 next_rx:
1863         cpr->rx_packets += 1;
1864         cpr->rx_bytes += len;
1865
1866 next_rx_no_len:
1867         rxr->rx_prod = NEXT_RX(prod);
1868         rxr->rx_next_cons = NEXT_RX(cons);
1869
1870 next_rx_no_prod_no_len:
1871         *raw_cons = tmp_raw_cons;
1872
1873         return rc;
1874 }
1875
1876 /* In netpoll mode, if we are using a combined completion ring, we need to
1877  * discard the rx packets and recycle the buffers.
1878  */
1879 static int bnxt_force_rx_discard(struct bnxt *bp,
1880                                  struct bnxt_cp_ring_info *cpr,
1881                                  u32 *raw_cons, u8 *event)
1882 {
1883         u32 tmp_raw_cons = *raw_cons;
1884         struct rx_cmp_ext *rxcmp1;
1885         struct rx_cmp *rxcmp;
1886         u16 cp_cons;
1887         u8 cmp_type;
1888
1889         cp_cons = RING_CMP(tmp_raw_cons);
1890         rxcmp = (struct rx_cmp *)
1891                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1892
1893         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1894         cp_cons = RING_CMP(tmp_raw_cons);
1895         rxcmp1 = (struct rx_cmp_ext *)
1896                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1897
1898         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1899                 return -EBUSY;
1900
1901         cmp_type = RX_CMP_TYPE(rxcmp);
1902         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1903                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1904                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1905         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1906                 struct rx_tpa_end_cmp_ext *tpa_end1;
1907
1908                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1909                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1910                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1911         }
1912         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1913 }
1914
1915 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1916 {
1917         struct bnxt_fw_health *fw_health = bp->fw_health;
1918         u32 reg = fw_health->regs[reg_idx];
1919         u32 reg_type, reg_off, val = 0;
1920
1921         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1922         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1923         switch (reg_type) {
1924         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1925                 pci_read_config_dword(bp->pdev, reg_off, &val);
1926                 break;
1927         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1928                 reg_off = fw_health->mapped_regs[reg_idx];
1929                 fallthrough;
1930         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1931                 val = readl(bp->bar0 + reg_off);
1932                 break;
1933         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1934                 val = readl(bp->bar1 + reg_off);
1935                 break;
1936         }
1937         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1938                 val &= fw_health->fw_reset_inprog_reg_mask;
1939         return val;
1940 }
1941
1942 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
1943 {
1944         int i;
1945
1946         for (i = 0; i < bp->rx_nr_rings; i++) {
1947                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
1948                 struct bnxt_ring_grp_info *grp_info;
1949
1950                 grp_info = &bp->grp_info[grp_idx];
1951                 if (grp_info->agg_fw_ring_id == ring_id)
1952                         return grp_idx;
1953         }
1954         return INVALID_HW_RING_ID;
1955 }
1956
1957 #define BNXT_GET_EVENT_PORT(data)       \
1958         ((data) &                       \
1959          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1960
1961 #define BNXT_EVENT_RING_TYPE(data2)     \
1962         ((data2) &                      \
1963          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
1964
1965 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
1966         (BNXT_EVENT_RING_TYPE(data2) == \
1967          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
1968
1969 static int bnxt_async_event_process(struct bnxt *bp,
1970                                     struct hwrm_async_event_cmpl *cmpl)
1971 {
1972         u16 event_id = le16_to_cpu(cmpl->event_id);
1973         u32 data1 = le32_to_cpu(cmpl->event_data1);
1974         u32 data2 = le32_to_cpu(cmpl->event_data2);
1975
1976         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1977         switch (event_id) {
1978         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1979                 struct bnxt_link_info *link_info = &bp->link_info;
1980
1981                 if (BNXT_VF(bp))
1982                         goto async_event_process_exit;
1983
1984                 /* print unsupported speed warning in forced speed mode only */
1985                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1986                     (data1 & 0x20000)) {
1987                         u16 fw_speed = link_info->force_link_speed;
1988                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1989
1990                         if (speed != SPEED_UNKNOWN)
1991                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1992                                             speed);
1993                 }
1994                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1995         }
1996                 fallthrough;
1997         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1998         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1999                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2000                 fallthrough;
2001         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2002                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2003                 break;
2004         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2005                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2006                 break;
2007         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2008                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2009
2010                 if (BNXT_VF(bp))
2011                         break;
2012
2013                 if (bp->pf.port_id != port_id)
2014                         break;
2015
2016                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2017                 break;
2018         }
2019         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2020                 if (BNXT_PF(bp))
2021                         goto async_event_process_exit;
2022                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2023                 break;
2024         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
2025                 if (netif_msg_hw(bp))
2026                         netdev_warn(bp->dev, "Received RESET_NOTIFY event, data1: 0x%x, data2: 0x%x\n",
2027                                     data1, data2);
2028                 if (!bp->fw_health)
2029                         goto async_event_process_exit;
2030
2031                 bp->fw_reset_timestamp = jiffies;
2032                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2033                 if (!bp->fw_reset_min_dsecs)
2034                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2035                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2036                 if (!bp->fw_reset_max_dsecs)
2037                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2038                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2039                         netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2040                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2041                 } else {
2042                         netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2043                                     bp->fw_reset_max_dsecs * 100);
2044                 }
2045                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2046                 break;
2047         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2048                 struct bnxt_fw_health *fw_health = bp->fw_health;
2049
2050                 if (!fw_health)
2051                         goto async_event_process_exit;
2052
2053                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2054                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2055                 if (!fw_health->enabled)
2056                         break;
2057
2058                 if (netif_msg_drv(bp))
2059                         netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2060                                     fw_health->enabled, fw_health->master,
2061                                     bnxt_fw_health_readl(bp,
2062                                                          BNXT_FW_RESET_CNT_REG),
2063                                     bnxt_fw_health_readl(bp,
2064                                                          BNXT_FW_HEALTH_REG));
2065                 fw_health->tmr_multiplier =
2066                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2067                                      bp->current_interval * 10);
2068                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2069                 fw_health->last_fw_heartbeat =
2070                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2071                 fw_health->last_fw_reset_cnt =
2072                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2073                 goto async_event_process_exit;
2074         }
2075         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2076                 struct bnxt_rx_ring_info *rxr;
2077                 u16 grp_idx;
2078
2079                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2080                         goto async_event_process_exit;
2081
2082                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2083                             BNXT_EVENT_RING_TYPE(data2), data1);
2084                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2085                         goto async_event_process_exit;
2086
2087                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2088                 if (grp_idx == INVALID_HW_RING_ID) {
2089                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2090                                     data1);
2091                         goto async_event_process_exit;
2092                 }
2093                 rxr = bp->bnapi[grp_idx]->rx_ring;
2094                 bnxt_sched_reset(bp, rxr);
2095                 goto async_event_process_exit;
2096         }
2097         default:
2098                 goto async_event_process_exit;
2099         }
2100         bnxt_queue_sp_work(bp);
2101 async_event_process_exit:
2102         bnxt_ulp_async_events(bp, cmpl);
2103         return 0;
2104 }
2105
2106 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2107 {
2108         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2109         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2110         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2111                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2112
2113         switch (cmpl_type) {
2114         case CMPL_BASE_TYPE_HWRM_DONE:
2115                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2116                 if (seq_id == bp->hwrm_intr_seq_id)
2117                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2118                 else
2119                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2120                 break;
2121
2122         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2123                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2124
2125                 if ((vf_id < bp->pf.first_vf_id) ||
2126                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2127                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2128                                    vf_id);
2129                         return -EINVAL;
2130                 }
2131
2132                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2133                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2134                 bnxt_queue_sp_work(bp);
2135                 break;
2136
2137         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2138                 bnxt_async_event_process(bp,
2139                                          (struct hwrm_async_event_cmpl *)txcmp);
2140
2141         default:
2142                 break;
2143         }
2144
2145         return 0;
2146 }
2147
2148 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2149 {
2150         struct bnxt_napi *bnapi = dev_instance;
2151         struct bnxt *bp = bnapi->bp;
2152         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2153         u32 cons = RING_CMP(cpr->cp_raw_cons);
2154
2155         cpr->event_ctr++;
2156         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2157         napi_schedule(&bnapi->napi);
2158         return IRQ_HANDLED;
2159 }
2160
2161 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2162 {
2163         u32 raw_cons = cpr->cp_raw_cons;
2164         u16 cons = RING_CMP(raw_cons);
2165         struct tx_cmp *txcmp;
2166
2167         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2168
2169         return TX_CMP_VALID(txcmp, raw_cons);
2170 }
2171
2172 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2173 {
2174         struct bnxt_napi *bnapi = dev_instance;
2175         struct bnxt *bp = bnapi->bp;
2176         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2177         u32 cons = RING_CMP(cpr->cp_raw_cons);
2178         u32 int_status;
2179
2180         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2181
2182         if (!bnxt_has_work(bp, cpr)) {
2183                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2184                 /* return if erroneous interrupt */
2185                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2186                         return IRQ_NONE;
2187         }
2188
2189         /* disable ring IRQ */
2190         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2191
2192         /* Return here if interrupt is shared and is disabled. */
2193         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2194                 return IRQ_HANDLED;
2195
2196         napi_schedule(&bnapi->napi);
2197         return IRQ_HANDLED;
2198 }
2199
2200 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2201                             int budget)
2202 {
2203         struct bnxt_napi *bnapi = cpr->bnapi;
2204         u32 raw_cons = cpr->cp_raw_cons;
2205         u32 cons;
2206         int tx_pkts = 0;
2207         int rx_pkts = 0;
2208         u8 event = 0;
2209         struct tx_cmp *txcmp;
2210
2211         cpr->has_more_work = 0;
2212         cpr->had_work_done = 1;
2213         while (1) {
2214                 int rc;
2215
2216                 cons = RING_CMP(raw_cons);
2217                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2218
2219                 if (!TX_CMP_VALID(txcmp, raw_cons))
2220                         break;
2221
2222                 /* The valid test of the entry must be done first before
2223                  * reading any further.
2224                  */
2225                 dma_rmb();
2226                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2227                         tx_pkts++;
2228                         /* return full budget so NAPI will complete. */
2229                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2230                                 rx_pkts = budget;
2231                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2232                                 if (budget)
2233                                         cpr->has_more_work = 1;
2234                                 break;
2235                         }
2236                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2237                         if (likely(budget))
2238                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2239                         else
2240                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2241                                                            &event);
2242                         if (likely(rc >= 0))
2243                                 rx_pkts += rc;
2244                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2245                          * the NAPI budget.  Otherwise, we may potentially loop
2246                          * here forever if we consistently cannot allocate
2247                          * buffers.
2248                          */
2249                         else if (rc == -ENOMEM && budget)
2250                                 rx_pkts++;
2251                         else if (rc == -EBUSY)  /* partial completion */
2252                                 break;
2253                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2254                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2255                                     (TX_CMP_TYPE(txcmp) ==
2256                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2257                                     (TX_CMP_TYPE(txcmp) ==
2258                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2259                         bnxt_hwrm_handler(bp, txcmp);
2260                 }
2261                 raw_cons = NEXT_RAW_CMP(raw_cons);
2262
2263                 if (rx_pkts && rx_pkts == budget) {
2264                         cpr->has_more_work = 1;
2265                         break;
2266                 }
2267         }
2268
2269         if (event & BNXT_REDIRECT_EVENT)
2270                 xdp_do_flush_map();
2271
2272         if (event & BNXT_TX_EVENT) {
2273                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2274                 u16 prod = txr->tx_prod;
2275
2276                 /* Sync BD data before updating doorbell */
2277                 wmb();
2278
2279                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2280         }
2281
2282         cpr->cp_raw_cons = raw_cons;
2283         bnapi->tx_pkts += tx_pkts;
2284         bnapi->events |= event;
2285         return rx_pkts;
2286 }
2287
2288 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2289 {
2290         if (bnapi->tx_pkts) {
2291                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2292                 bnapi->tx_pkts = 0;
2293         }
2294
2295         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2296                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2297
2298                 if (bnapi->events & BNXT_AGG_EVENT)
2299                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2300                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2301         }
2302         bnapi->events = 0;
2303 }
2304
2305 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2306                           int budget)
2307 {
2308         struct bnxt_napi *bnapi = cpr->bnapi;
2309         int rx_pkts;
2310
2311         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2312
2313         /* ACK completion ring before freeing tx ring and producing new
2314          * buffers in rx/agg rings to prevent overflowing the completion
2315          * ring.
2316          */
2317         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2318
2319         __bnxt_poll_work_done(bp, bnapi);
2320         return rx_pkts;
2321 }
2322
2323 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2324 {
2325         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2326         struct bnxt *bp = bnapi->bp;
2327         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2328         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2329         struct tx_cmp *txcmp;
2330         struct rx_cmp_ext *rxcmp1;
2331         u32 cp_cons, tmp_raw_cons;
2332         u32 raw_cons = cpr->cp_raw_cons;
2333         u32 rx_pkts = 0;
2334         u8 event = 0;
2335
2336         while (1) {
2337                 int rc;
2338
2339                 cp_cons = RING_CMP(raw_cons);
2340                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2341
2342                 if (!TX_CMP_VALID(txcmp, raw_cons))
2343                         break;
2344
2345                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2346                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2347                         cp_cons = RING_CMP(tmp_raw_cons);
2348                         rxcmp1 = (struct rx_cmp_ext *)
2349                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2350
2351                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2352                                 break;
2353
2354                         /* force an error to recycle the buffer */
2355                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2356                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2357
2358                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2359                         if (likely(rc == -EIO) && budget)
2360                                 rx_pkts++;
2361                         else if (rc == -EBUSY)  /* partial completion */
2362                                 break;
2363                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2364                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2365                         bnxt_hwrm_handler(bp, txcmp);
2366                 } else {
2367                         netdev_err(bp->dev,
2368                                    "Invalid completion received on special ring\n");
2369                 }
2370                 raw_cons = NEXT_RAW_CMP(raw_cons);
2371
2372                 if (rx_pkts == budget)
2373                         break;
2374         }
2375
2376         cpr->cp_raw_cons = raw_cons;
2377         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2378         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2379
2380         if (event & BNXT_AGG_EVENT)
2381                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2382
2383         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2384                 napi_complete_done(napi, rx_pkts);
2385                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2386         }
2387         return rx_pkts;
2388 }
2389
2390 static int bnxt_poll(struct napi_struct *napi, int budget)
2391 {
2392         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2393         struct bnxt *bp = bnapi->bp;
2394         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2395         int work_done = 0;
2396
2397         while (1) {
2398                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2399
2400                 if (work_done >= budget) {
2401                         if (!budget)
2402                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2403                         break;
2404                 }
2405
2406                 if (!bnxt_has_work(bp, cpr)) {
2407                         if (napi_complete_done(napi, work_done))
2408                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2409                         break;
2410                 }
2411         }
2412         if (bp->flags & BNXT_FLAG_DIM) {
2413                 struct dim_sample dim_sample = {};
2414
2415                 dim_update_sample(cpr->event_ctr,
2416                                   cpr->rx_packets,
2417                                   cpr->rx_bytes,
2418                                   &dim_sample);
2419                 net_dim(&cpr->dim, dim_sample);
2420         }
2421         return work_done;
2422 }
2423
2424 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2425 {
2426         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2427         int i, work_done = 0;
2428
2429         for (i = 0; i < 2; i++) {
2430                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2431
2432                 if (cpr2) {
2433                         work_done += __bnxt_poll_work(bp, cpr2,
2434                                                       budget - work_done);
2435                         cpr->has_more_work |= cpr2->has_more_work;
2436                 }
2437         }
2438         return work_done;
2439 }
2440
2441 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2442                                  u64 dbr_type)
2443 {
2444         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2445         int i;
2446
2447         for (i = 0; i < 2; i++) {
2448                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2449                 struct bnxt_db_info *db;
2450
2451                 if (cpr2 && cpr2->had_work_done) {
2452                         db = &cpr2->cp_db;
2453                         writeq(db->db_key64 | dbr_type |
2454                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2455                         cpr2->had_work_done = 0;
2456                 }
2457         }
2458         __bnxt_poll_work_done(bp, bnapi);
2459 }
2460
2461 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2462 {
2463         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2464         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2465         u32 raw_cons = cpr->cp_raw_cons;
2466         struct bnxt *bp = bnapi->bp;
2467         struct nqe_cn *nqcmp;
2468         int work_done = 0;
2469         u32 cons;
2470
2471         if (cpr->has_more_work) {
2472                 cpr->has_more_work = 0;
2473                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2474         }
2475         while (1) {
2476                 cons = RING_CMP(raw_cons);
2477                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2478
2479                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2480                         if (cpr->has_more_work)
2481                                 break;
2482
2483                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2484                         cpr->cp_raw_cons = raw_cons;
2485                         if (napi_complete_done(napi, work_done))
2486                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2487                                                   cpr->cp_raw_cons);
2488                         return work_done;
2489                 }
2490
2491                 /* The valid test of the entry must be done first before
2492                  * reading any further.
2493                  */
2494                 dma_rmb();
2495
2496                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2497                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2498                         struct bnxt_cp_ring_info *cpr2;
2499
2500                         cpr2 = cpr->cp_ring_arr[idx];
2501                         work_done += __bnxt_poll_work(bp, cpr2,
2502                                                       budget - work_done);
2503                         cpr->has_more_work |= cpr2->has_more_work;
2504                 } else {
2505                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2506                 }
2507                 raw_cons = NEXT_RAW_CMP(raw_cons);
2508         }
2509         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2510         if (raw_cons != cpr->cp_raw_cons) {
2511                 cpr->cp_raw_cons = raw_cons;
2512                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2513         }
2514         return work_done;
2515 }
2516
2517 static void bnxt_free_tx_skbs(struct bnxt *bp)
2518 {
2519         int i, max_idx;
2520         struct pci_dev *pdev = bp->pdev;
2521
2522         if (!bp->tx_ring)
2523                 return;
2524
2525         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2526         for (i = 0; i < bp->tx_nr_rings; i++) {
2527                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2528                 int j;
2529
2530                 for (j = 0; j < max_idx;) {
2531                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2532                         struct sk_buff *skb;
2533                         int k, last;
2534
2535                         if (i < bp->tx_nr_rings_xdp &&
2536                             tx_buf->action == XDP_REDIRECT) {
2537                                 dma_unmap_single(&pdev->dev,
2538                                         dma_unmap_addr(tx_buf, mapping),
2539                                         dma_unmap_len(tx_buf, len),
2540                                         PCI_DMA_TODEVICE);
2541                                 xdp_return_frame(tx_buf->xdpf);
2542                                 tx_buf->action = 0;
2543                                 tx_buf->xdpf = NULL;
2544                                 j++;
2545                                 continue;
2546                         }
2547
2548                         skb = tx_buf->skb;
2549                         if (!skb) {
2550                                 j++;
2551                                 continue;
2552                         }
2553
2554                         tx_buf->skb = NULL;
2555
2556                         if (tx_buf->is_push) {
2557                                 dev_kfree_skb(skb);
2558                                 j += 2;
2559                                 continue;
2560                         }
2561
2562                         dma_unmap_single(&pdev->dev,
2563                                          dma_unmap_addr(tx_buf, mapping),
2564                                          skb_headlen(skb),
2565                                          PCI_DMA_TODEVICE);
2566
2567                         last = tx_buf->nr_frags;
2568                         j += 2;
2569                         for (k = 0; k < last; k++, j++) {
2570                                 int ring_idx = j & bp->tx_ring_mask;
2571                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2572
2573                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2574                                 dma_unmap_page(
2575                                         &pdev->dev,
2576                                         dma_unmap_addr(tx_buf, mapping),
2577                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2578                         }
2579                         dev_kfree_skb(skb);
2580                 }
2581                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2582         }
2583 }
2584
2585 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2586 {
2587         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2588         struct pci_dev *pdev = bp->pdev;
2589         struct bnxt_tpa_idx_map *map;
2590         int i, max_idx, max_agg_idx;
2591
2592         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2593         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2594         if (!rxr->rx_tpa)
2595                 goto skip_rx_tpa_free;
2596
2597         for (i = 0; i < bp->max_tpa; i++) {
2598                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2599                 u8 *data = tpa_info->data;
2600
2601                 if (!data)
2602                         continue;
2603
2604                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2605                                        bp->rx_buf_use_size, bp->rx_dir,
2606                                        DMA_ATTR_WEAK_ORDERING);
2607
2608                 tpa_info->data = NULL;
2609
2610                 kfree(data);
2611         }
2612
2613 skip_rx_tpa_free:
2614         for (i = 0; i < max_idx; i++) {
2615                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2616                 dma_addr_t mapping = rx_buf->mapping;
2617                 void *data = rx_buf->data;
2618
2619                 if (!data)
2620                         continue;
2621
2622                 rx_buf->data = NULL;
2623                 if (BNXT_RX_PAGE_MODE(bp)) {
2624                         mapping -= bp->rx_dma_offset;
2625                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2626                                              bp->rx_dir,
2627                                              DMA_ATTR_WEAK_ORDERING);
2628                         page_pool_recycle_direct(rxr->page_pool, data);
2629                 } else {
2630                         dma_unmap_single_attrs(&pdev->dev, mapping,
2631                                                bp->rx_buf_use_size, bp->rx_dir,
2632                                                DMA_ATTR_WEAK_ORDERING);
2633                         kfree(data);
2634                 }
2635         }
2636         for (i = 0; i < max_agg_idx; i++) {
2637                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2638                 struct page *page = rx_agg_buf->page;
2639
2640                 if (!page)
2641                         continue;
2642
2643                 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2644                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2645                                      DMA_ATTR_WEAK_ORDERING);
2646
2647                 rx_agg_buf->page = NULL;
2648                 __clear_bit(i, rxr->rx_agg_bmap);
2649
2650                 __free_page(page);
2651         }
2652         if (rxr->rx_page) {
2653                 __free_page(rxr->rx_page);
2654                 rxr->rx_page = NULL;
2655         }
2656         map = rxr->rx_tpa_idx_map;
2657         if (map)
2658                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2659 }
2660
2661 static void bnxt_free_rx_skbs(struct bnxt *bp)
2662 {
2663         int i;
2664
2665         if (!bp->rx_ring)
2666                 return;
2667
2668         for (i = 0; i < bp->rx_nr_rings; i++)
2669                 bnxt_free_one_rx_ring_skbs(bp, i);
2670 }
2671
2672 static void bnxt_free_skbs(struct bnxt *bp)
2673 {
2674         bnxt_free_tx_skbs(bp);
2675         bnxt_free_rx_skbs(bp);
2676 }
2677
2678 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2679 {
2680         struct pci_dev *pdev = bp->pdev;
2681         int i;
2682
2683         for (i = 0; i < rmem->nr_pages; i++) {
2684                 if (!rmem->pg_arr[i])
2685                         continue;
2686
2687                 dma_free_coherent(&pdev->dev, rmem->page_size,
2688                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2689
2690                 rmem->pg_arr[i] = NULL;
2691         }
2692         if (rmem->pg_tbl) {
2693                 size_t pg_tbl_size = rmem->nr_pages * 8;
2694
2695                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2696                         pg_tbl_size = rmem->page_size;
2697                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2698                                   rmem->pg_tbl, rmem->pg_tbl_map);
2699                 rmem->pg_tbl = NULL;
2700         }
2701         if (rmem->vmem_size && *rmem->vmem) {
2702                 vfree(*rmem->vmem);
2703                 *rmem->vmem = NULL;
2704         }
2705 }
2706
2707 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2708 {
2709         struct pci_dev *pdev = bp->pdev;
2710         u64 valid_bit = 0;
2711         int i;
2712
2713         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2714                 valid_bit = PTU_PTE_VALID;
2715         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2716                 size_t pg_tbl_size = rmem->nr_pages * 8;
2717
2718                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2719                         pg_tbl_size = rmem->page_size;
2720                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2721                                                   &rmem->pg_tbl_map,
2722                                                   GFP_KERNEL);
2723                 if (!rmem->pg_tbl)
2724                         return -ENOMEM;
2725         }
2726
2727         for (i = 0; i < rmem->nr_pages; i++) {
2728                 u64 extra_bits = valid_bit;
2729
2730                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2731                                                      rmem->page_size,
2732                                                      &rmem->dma_arr[i],
2733                                                      GFP_KERNEL);
2734                 if (!rmem->pg_arr[i])
2735                         return -ENOMEM;
2736
2737                 if (rmem->init_val)
2738                         memset(rmem->pg_arr[i], rmem->init_val,
2739                                rmem->page_size);
2740                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2741                         if (i == rmem->nr_pages - 2 &&
2742                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2743                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2744                         else if (i == rmem->nr_pages - 1 &&
2745                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2746                                 extra_bits |= PTU_PTE_LAST;
2747                         rmem->pg_tbl[i] =
2748                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2749                 }
2750         }
2751
2752         if (rmem->vmem_size) {
2753                 *rmem->vmem = vzalloc(rmem->vmem_size);
2754                 if (!(*rmem->vmem))
2755                         return -ENOMEM;
2756         }
2757         return 0;
2758 }
2759
2760 static void bnxt_free_tpa_info(struct bnxt *bp)
2761 {
2762         int i;
2763
2764         for (i = 0; i < bp->rx_nr_rings; i++) {
2765                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2766
2767                 kfree(rxr->rx_tpa_idx_map);
2768                 rxr->rx_tpa_idx_map = NULL;
2769                 if (rxr->rx_tpa) {
2770                         kfree(rxr->rx_tpa[0].agg_arr);
2771                         rxr->rx_tpa[0].agg_arr = NULL;
2772                 }
2773                 kfree(rxr->rx_tpa);
2774                 rxr->rx_tpa = NULL;
2775         }
2776 }
2777
2778 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2779 {
2780         int i, j, total_aggs = 0;
2781
2782         bp->max_tpa = MAX_TPA;
2783         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2784                 if (!bp->max_tpa_v2)
2785                         return 0;
2786                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2787                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2788         }
2789
2790         for (i = 0; i < bp->rx_nr_rings; i++) {
2791                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2792                 struct rx_agg_cmp *agg;
2793
2794                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2795                                       GFP_KERNEL);
2796                 if (!rxr->rx_tpa)
2797                         return -ENOMEM;
2798
2799                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2800                         continue;
2801                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2802                 rxr->rx_tpa[0].agg_arr = agg;
2803                 if (!agg)
2804                         return -ENOMEM;
2805                 for (j = 1; j < bp->max_tpa; j++)
2806                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2807                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2808                                               GFP_KERNEL);
2809                 if (!rxr->rx_tpa_idx_map)
2810                         return -ENOMEM;
2811         }
2812         return 0;
2813 }
2814
2815 static void bnxt_free_rx_rings(struct bnxt *bp)
2816 {
2817         int i;
2818
2819         if (!bp->rx_ring)
2820                 return;
2821
2822         bnxt_free_tpa_info(bp);
2823         for (i = 0; i < bp->rx_nr_rings; i++) {
2824                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2825                 struct bnxt_ring_struct *ring;
2826
2827                 if (rxr->xdp_prog)
2828                         bpf_prog_put(rxr->xdp_prog);
2829
2830                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2831                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2832
2833                 page_pool_destroy(rxr->page_pool);
2834                 rxr->page_pool = NULL;
2835
2836                 kfree(rxr->rx_agg_bmap);
2837                 rxr->rx_agg_bmap = NULL;
2838
2839                 ring = &rxr->rx_ring_struct;
2840                 bnxt_free_ring(bp, &ring->ring_mem);
2841
2842                 ring = &rxr->rx_agg_ring_struct;
2843                 bnxt_free_ring(bp, &ring->ring_mem);
2844         }
2845 }
2846
2847 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2848                                    struct bnxt_rx_ring_info *rxr)
2849 {
2850         struct page_pool_params pp = { 0 };
2851
2852         pp.pool_size = bp->rx_ring_size;
2853         pp.nid = dev_to_node(&bp->pdev->dev);
2854         pp.dev = &bp->pdev->dev;
2855         pp.dma_dir = DMA_BIDIRECTIONAL;
2856
2857         rxr->page_pool = page_pool_create(&pp);
2858         if (IS_ERR(rxr->page_pool)) {
2859                 int err = PTR_ERR(rxr->page_pool);
2860
2861                 rxr->page_pool = NULL;
2862                 return err;
2863         }
2864         return 0;
2865 }
2866
2867 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2868 {
2869         int i, rc = 0, agg_rings = 0;
2870
2871         if (!bp->rx_ring)
2872                 return -ENOMEM;
2873
2874         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2875                 agg_rings = 1;
2876
2877         for (i = 0; i < bp->rx_nr_rings; i++) {
2878                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2879                 struct bnxt_ring_struct *ring;
2880
2881                 ring = &rxr->rx_ring_struct;
2882
2883                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2884                 if (rc)
2885                         return rc;
2886
2887                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
2888                 if (rc < 0)
2889                         return rc;
2890
2891                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2892                                                 MEM_TYPE_PAGE_POOL,
2893                                                 rxr->page_pool);
2894                 if (rc) {
2895                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2896                         return rc;
2897                 }
2898
2899                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2900                 if (rc)
2901                         return rc;
2902
2903                 ring->grp_idx = i;
2904                 if (agg_rings) {
2905                         u16 mem_size;
2906
2907                         ring = &rxr->rx_agg_ring_struct;
2908                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2909                         if (rc)
2910                                 return rc;
2911
2912                         ring->grp_idx = i;
2913                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2914                         mem_size = rxr->rx_agg_bmap_size / 8;
2915                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2916                         if (!rxr->rx_agg_bmap)
2917                                 return -ENOMEM;
2918                 }
2919         }
2920         if (bp->flags & BNXT_FLAG_TPA)
2921                 rc = bnxt_alloc_tpa_info(bp);
2922         return rc;
2923 }
2924
2925 static void bnxt_free_tx_rings(struct bnxt *bp)
2926 {
2927         int i;
2928         struct pci_dev *pdev = bp->pdev;
2929
2930         if (!bp->tx_ring)
2931                 return;
2932
2933         for (i = 0; i < bp->tx_nr_rings; i++) {
2934                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2935                 struct bnxt_ring_struct *ring;
2936
2937                 if (txr->tx_push) {
2938                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2939                                           txr->tx_push, txr->tx_push_mapping);
2940                         txr->tx_push = NULL;
2941                 }
2942
2943                 ring = &txr->tx_ring_struct;
2944
2945                 bnxt_free_ring(bp, &ring->ring_mem);
2946         }
2947 }
2948
2949 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2950 {
2951         int i, j, rc;
2952         struct pci_dev *pdev = bp->pdev;
2953
2954         bp->tx_push_size = 0;
2955         if (bp->tx_push_thresh) {
2956                 int push_size;
2957
2958                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2959                                         bp->tx_push_thresh);
2960
2961                 if (push_size > 256) {
2962                         push_size = 0;
2963                         bp->tx_push_thresh = 0;
2964                 }
2965
2966                 bp->tx_push_size = push_size;
2967         }
2968
2969         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2970                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2971                 struct bnxt_ring_struct *ring;
2972                 u8 qidx;
2973
2974                 ring = &txr->tx_ring_struct;
2975
2976                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2977                 if (rc)
2978                         return rc;
2979
2980                 ring->grp_idx = txr->bnapi->index;
2981                 if (bp->tx_push_size) {
2982                         dma_addr_t mapping;
2983
2984                         /* One pre-allocated DMA buffer to backup
2985                          * TX push operation
2986                          */
2987                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2988                                                 bp->tx_push_size,
2989                                                 &txr->tx_push_mapping,
2990                                                 GFP_KERNEL);
2991
2992                         if (!txr->tx_push)
2993                                 return -ENOMEM;
2994
2995                         mapping = txr->tx_push_mapping +
2996                                 sizeof(struct tx_push_bd);
2997                         txr->data_mapping = cpu_to_le64(mapping);
2998                 }
2999                 qidx = bp->tc_to_qidx[j];
3000                 ring->queue_id = bp->q_info[qidx].queue_id;
3001                 if (i < bp->tx_nr_rings_xdp)
3002                         continue;
3003                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3004                         j++;
3005         }
3006         return 0;
3007 }
3008
3009 static void bnxt_free_cp_rings(struct bnxt *bp)
3010 {
3011         int i;
3012
3013         if (!bp->bnapi)
3014                 return;
3015
3016         for (i = 0; i < bp->cp_nr_rings; i++) {
3017                 struct bnxt_napi *bnapi = bp->bnapi[i];
3018                 struct bnxt_cp_ring_info *cpr;
3019                 struct bnxt_ring_struct *ring;
3020                 int j;
3021
3022                 if (!bnapi)
3023                         continue;
3024
3025                 cpr = &bnapi->cp_ring;
3026                 ring = &cpr->cp_ring_struct;
3027
3028                 bnxt_free_ring(bp, &ring->ring_mem);
3029
3030                 for (j = 0; j < 2; j++) {
3031                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3032
3033                         if (cpr2) {
3034                                 ring = &cpr2->cp_ring_struct;
3035                                 bnxt_free_ring(bp, &ring->ring_mem);
3036                                 kfree(cpr2);
3037                                 cpr->cp_ring_arr[j] = NULL;
3038                         }
3039                 }
3040         }
3041 }
3042
3043 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3044 {
3045         struct bnxt_ring_mem_info *rmem;
3046         struct bnxt_ring_struct *ring;
3047         struct bnxt_cp_ring_info *cpr;
3048         int rc;
3049
3050         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3051         if (!cpr)
3052                 return NULL;
3053
3054         ring = &cpr->cp_ring_struct;
3055         rmem = &ring->ring_mem;
3056         rmem->nr_pages = bp->cp_nr_pages;
3057         rmem->page_size = HW_CMPD_RING_SIZE;
3058         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3059         rmem->dma_arr = cpr->cp_desc_mapping;
3060         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3061         rc = bnxt_alloc_ring(bp, rmem);
3062         if (rc) {
3063                 bnxt_free_ring(bp, rmem);
3064                 kfree(cpr);
3065                 cpr = NULL;
3066         }
3067         return cpr;
3068 }
3069
3070 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3071 {
3072         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3073         int i, rc, ulp_base_vec, ulp_msix;
3074
3075         ulp_msix = bnxt_get_ulp_msix_num(bp);
3076         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3077         for (i = 0; i < bp->cp_nr_rings; i++) {
3078                 struct bnxt_napi *bnapi = bp->bnapi[i];
3079                 struct bnxt_cp_ring_info *cpr;
3080                 struct bnxt_ring_struct *ring;
3081
3082                 if (!bnapi)
3083                         continue;
3084
3085                 cpr = &bnapi->cp_ring;
3086                 cpr->bnapi = bnapi;
3087                 ring = &cpr->cp_ring_struct;
3088
3089                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3090                 if (rc)
3091                         return rc;
3092
3093                 if (ulp_msix && i >= ulp_base_vec)
3094                         ring->map_idx = i + ulp_msix;
3095                 else
3096                         ring->map_idx = i;
3097
3098                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3099                         continue;
3100
3101                 if (i < bp->rx_nr_rings) {
3102                         struct bnxt_cp_ring_info *cpr2 =
3103                                 bnxt_alloc_cp_sub_ring(bp);
3104
3105                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3106                         if (!cpr2)
3107                                 return -ENOMEM;
3108                         cpr2->bnapi = bnapi;
3109                 }
3110                 if ((sh && i < bp->tx_nr_rings) ||
3111                     (!sh && i >= bp->rx_nr_rings)) {
3112                         struct bnxt_cp_ring_info *cpr2 =
3113                                 bnxt_alloc_cp_sub_ring(bp);
3114
3115                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3116                         if (!cpr2)
3117                                 return -ENOMEM;
3118                         cpr2->bnapi = bnapi;
3119                 }
3120         }
3121         return 0;
3122 }
3123
3124 static void bnxt_init_ring_struct(struct bnxt *bp)
3125 {
3126         int i;
3127
3128         for (i = 0; i < bp->cp_nr_rings; i++) {
3129                 struct bnxt_napi *bnapi = bp->bnapi[i];
3130                 struct bnxt_ring_mem_info *rmem;
3131                 struct bnxt_cp_ring_info *cpr;
3132                 struct bnxt_rx_ring_info *rxr;
3133                 struct bnxt_tx_ring_info *txr;
3134                 struct bnxt_ring_struct *ring;
3135
3136                 if (!bnapi)
3137                         continue;
3138
3139                 cpr = &bnapi->cp_ring;
3140                 ring = &cpr->cp_ring_struct;
3141                 rmem = &ring->ring_mem;
3142                 rmem->nr_pages = bp->cp_nr_pages;
3143                 rmem->page_size = HW_CMPD_RING_SIZE;
3144                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3145                 rmem->dma_arr = cpr->cp_desc_mapping;
3146                 rmem->vmem_size = 0;
3147
3148                 rxr = bnapi->rx_ring;
3149                 if (!rxr)
3150                         goto skip_rx;
3151
3152                 ring = &rxr->rx_ring_struct;
3153                 rmem = &ring->ring_mem;
3154                 rmem->nr_pages = bp->rx_nr_pages;
3155                 rmem->page_size = HW_RXBD_RING_SIZE;
3156                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3157                 rmem->dma_arr = rxr->rx_desc_mapping;
3158                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3159                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3160
3161                 ring = &rxr->rx_agg_ring_struct;
3162                 rmem = &ring->ring_mem;
3163                 rmem->nr_pages = bp->rx_agg_nr_pages;
3164                 rmem->page_size = HW_RXBD_RING_SIZE;
3165                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3166                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3167                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3168                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3169
3170 skip_rx:
3171                 txr = bnapi->tx_ring;
3172                 if (!txr)
3173                         continue;
3174
3175                 ring = &txr->tx_ring_struct;
3176                 rmem = &ring->ring_mem;
3177                 rmem->nr_pages = bp->tx_nr_pages;
3178                 rmem->page_size = HW_RXBD_RING_SIZE;
3179                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3180                 rmem->dma_arr = txr->tx_desc_mapping;
3181                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3182                 rmem->vmem = (void **)&txr->tx_buf_ring;
3183         }
3184 }
3185
3186 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3187 {
3188         int i;
3189         u32 prod;
3190         struct rx_bd **rx_buf_ring;
3191
3192         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3193         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3194                 int j;
3195                 struct rx_bd *rxbd;
3196
3197                 rxbd = rx_buf_ring[i];
3198                 if (!rxbd)
3199                         continue;
3200
3201                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3202                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3203                         rxbd->rx_bd_opaque = prod;
3204                 }
3205         }
3206 }
3207
3208 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3209 {
3210         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3211         struct net_device *dev = bp->dev;
3212         u32 prod;
3213         int i;
3214
3215         prod = rxr->rx_prod;
3216         for (i = 0; i < bp->rx_ring_size; i++) {
3217                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3218                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3219                                     ring_nr, i, bp->rx_ring_size);
3220                         break;
3221                 }
3222                 prod = NEXT_RX(prod);
3223         }
3224         rxr->rx_prod = prod;
3225
3226         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3227                 return 0;
3228
3229         prod = rxr->rx_agg_prod;
3230         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3231                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3232                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3233                                     ring_nr, i, bp->rx_ring_size);
3234                         break;
3235                 }
3236                 prod = NEXT_RX_AGG(prod);
3237         }
3238         rxr->rx_agg_prod = prod;
3239
3240         if (rxr->rx_tpa) {
3241                 dma_addr_t mapping;
3242                 u8 *data;
3243
3244                 for (i = 0; i < bp->max_tpa; i++) {
3245                         data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3246                         if (!data)
3247                                 return -ENOMEM;
3248
3249                         rxr->rx_tpa[i].data = data;
3250                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3251                         rxr->rx_tpa[i].mapping = mapping;
3252                 }
3253         }
3254         return 0;
3255 }
3256
3257 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3258 {
3259         struct bnxt_rx_ring_info *rxr;
3260         struct bnxt_ring_struct *ring;
3261         u32 type;
3262
3263         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3264                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3265
3266         if (NET_IP_ALIGN == 2)
3267                 type |= RX_BD_FLAGS_SOP;
3268
3269         rxr = &bp->rx_ring[ring_nr];
3270         ring = &rxr->rx_ring_struct;
3271         bnxt_init_rxbd_pages(ring, type);
3272
3273         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3274                 bpf_prog_add(bp->xdp_prog, 1);
3275                 rxr->xdp_prog = bp->xdp_prog;
3276         }
3277         ring->fw_ring_id = INVALID_HW_RING_ID;
3278
3279         ring = &rxr->rx_agg_ring_struct;
3280         ring->fw_ring_id = INVALID_HW_RING_ID;
3281
3282         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3283                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3284                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3285
3286                 bnxt_init_rxbd_pages(ring, type);
3287         }
3288
3289         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3290 }
3291
3292 static void bnxt_init_cp_rings(struct bnxt *bp)
3293 {
3294         int i, j;
3295
3296         for (i = 0; i < bp->cp_nr_rings; i++) {
3297                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3298                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3299
3300                 ring->fw_ring_id = INVALID_HW_RING_ID;
3301                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3302                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3303                 for (j = 0; j < 2; j++) {
3304                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3305
3306                         if (!cpr2)
3307                                 continue;
3308
3309                         ring = &cpr2->cp_ring_struct;
3310                         ring->fw_ring_id = INVALID_HW_RING_ID;
3311                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3312                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3313                 }
3314         }
3315 }
3316
3317 static int bnxt_init_rx_rings(struct bnxt *bp)
3318 {
3319         int i, rc = 0;
3320
3321         if (BNXT_RX_PAGE_MODE(bp)) {
3322                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3323                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3324         } else {
3325                 bp->rx_offset = BNXT_RX_OFFSET;
3326                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3327         }
3328
3329         for (i = 0; i < bp->rx_nr_rings; i++) {
3330                 rc = bnxt_init_one_rx_ring(bp, i);
3331                 if (rc)
3332                         break;
3333         }
3334
3335         return rc;
3336 }
3337
3338 static int bnxt_init_tx_rings(struct bnxt *bp)
3339 {
3340         u16 i;
3341
3342         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3343                                    MAX_SKB_FRAGS + 1);
3344
3345         for (i = 0; i < bp->tx_nr_rings; i++) {
3346                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3347                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3348
3349                 ring->fw_ring_id = INVALID_HW_RING_ID;
3350         }
3351
3352         return 0;
3353 }
3354
3355 static void bnxt_free_ring_grps(struct bnxt *bp)
3356 {
3357         kfree(bp->grp_info);
3358         bp->grp_info = NULL;
3359 }
3360
3361 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3362 {
3363         int i;
3364
3365         if (irq_re_init) {
3366                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3367                                        sizeof(struct bnxt_ring_grp_info),
3368                                        GFP_KERNEL);
3369                 if (!bp->grp_info)
3370                         return -ENOMEM;
3371         }
3372         for (i = 0; i < bp->cp_nr_rings; i++) {
3373                 if (irq_re_init)
3374                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3375                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3376                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3377                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3378                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3379         }
3380         return 0;
3381 }
3382
3383 static void bnxt_free_vnics(struct bnxt *bp)
3384 {
3385         kfree(bp->vnic_info);
3386         bp->vnic_info = NULL;
3387         bp->nr_vnics = 0;
3388 }
3389
3390 static int bnxt_alloc_vnics(struct bnxt *bp)
3391 {
3392         int num_vnics = 1;
3393
3394 #ifdef CONFIG_RFS_ACCEL
3395         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3396                 num_vnics += bp->rx_nr_rings;
3397 #endif
3398
3399         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3400                 num_vnics++;
3401
3402         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3403                                 GFP_KERNEL);
3404         if (!bp->vnic_info)
3405                 return -ENOMEM;
3406
3407         bp->nr_vnics = num_vnics;
3408         return 0;
3409 }
3410
3411 static void bnxt_init_vnics(struct bnxt *bp)
3412 {
3413         int i;
3414
3415         for (i = 0; i < bp->nr_vnics; i++) {
3416                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3417                 int j;
3418
3419                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3420                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3421                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3422
3423                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3424
3425                 if (bp->vnic_info[i].rss_hash_key) {
3426                         if (i == 0)
3427                                 prandom_bytes(vnic->rss_hash_key,
3428                                               HW_HASH_KEY_SIZE);
3429                         else
3430                                 memcpy(vnic->rss_hash_key,
3431                                        bp->vnic_info[0].rss_hash_key,
3432                                        HW_HASH_KEY_SIZE);
3433                 }
3434         }
3435 }
3436
3437 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3438 {
3439         int pages;
3440
3441         pages = ring_size / desc_per_pg;
3442
3443         if (!pages)
3444                 return 1;
3445
3446         pages++;
3447
3448         while (pages & (pages - 1))
3449                 pages++;
3450
3451         return pages;
3452 }
3453
3454 void bnxt_set_tpa_flags(struct bnxt *bp)
3455 {
3456         bp->flags &= ~BNXT_FLAG_TPA;
3457         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3458                 return;
3459         if (bp->dev->features & NETIF_F_LRO)
3460                 bp->flags |= BNXT_FLAG_LRO;
3461         else if (bp->dev->features & NETIF_F_GRO_HW)
3462                 bp->flags |= BNXT_FLAG_GRO;
3463 }
3464
3465 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3466  * be set on entry.
3467  */
3468 void bnxt_set_ring_params(struct bnxt *bp)
3469 {
3470         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3471         u32 agg_factor = 0, agg_ring_size = 0;
3472
3473         /* 8 for CRC and VLAN */
3474         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3475
3476         rx_space = rx_size + NET_SKB_PAD +
3477                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3478
3479         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3480         ring_size = bp->rx_ring_size;
3481         bp->rx_agg_ring_size = 0;
3482         bp->rx_agg_nr_pages = 0;
3483
3484         if (bp->flags & BNXT_FLAG_TPA)
3485                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3486
3487         bp->flags &= ~BNXT_FLAG_JUMBO;
3488         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3489                 u32 jumbo_factor;
3490
3491                 bp->flags |= BNXT_FLAG_JUMBO;
3492                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3493                 if (jumbo_factor > agg_factor)
3494                         agg_factor = jumbo_factor;
3495         }
3496         agg_ring_size = ring_size * agg_factor;
3497
3498         if (agg_ring_size) {
3499                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3500                                                         RX_DESC_CNT);
3501                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3502                         u32 tmp = agg_ring_size;
3503
3504                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3505                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3506                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3507                                     tmp, agg_ring_size);
3508                 }
3509                 bp->rx_agg_ring_size = agg_ring_size;
3510                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3511                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3512                 rx_space = rx_size + NET_SKB_PAD +
3513                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3514         }
3515
3516         bp->rx_buf_use_size = rx_size;
3517         bp->rx_buf_size = rx_space;
3518
3519         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3520         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3521
3522         ring_size = bp->tx_ring_size;
3523         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3524         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3525
3526         max_rx_cmpl = bp->rx_ring_size;
3527         /* MAX TPA needs to be added because TPA_START completions are
3528          * immediately recycled, so the TPA completions are not bound by
3529          * the RX ring size.
3530          */
3531         if (bp->flags & BNXT_FLAG_TPA)
3532                 max_rx_cmpl += bp->max_tpa;
3533         /* RX and TPA completions are 32-byte, all others are 16-byte */
3534         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3535         bp->cp_ring_size = ring_size;
3536
3537         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3538         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3539                 bp->cp_nr_pages = MAX_CP_PAGES;
3540                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3541                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3542                             ring_size, bp->cp_ring_size);
3543         }
3544         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3545         bp->cp_ring_mask = bp->cp_bit - 1;
3546 }
3547
3548 /* Changing allocation mode of RX rings.
3549  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3550  */
3551 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3552 {
3553         if (page_mode) {
3554                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3555                         return -EOPNOTSUPP;
3556                 bp->dev->max_mtu =
3557                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3558                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3559                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3560                 bp->rx_dir = DMA_BIDIRECTIONAL;
3561                 bp->rx_skb_func = bnxt_rx_page_skb;
3562                 /* Disable LRO or GRO_HW */
3563                 netdev_update_features(bp->dev);
3564         } else {
3565                 bp->dev->max_mtu = bp->max_mtu;
3566                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3567                 bp->rx_dir = DMA_FROM_DEVICE;
3568                 bp->rx_skb_func = bnxt_rx_skb;
3569         }
3570         return 0;
3571 }
3572
3573 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3574 {
3575         int i;
3576         struct bnxt_vnic_info *vnic;
3577         struct pci_dev *pdev = bp->pdev;
3578
3579         if (!bp->vnic_info)
3580                 return;
3581
3582         for (i = 0; i < bp->nr_vnics; i++) {
3583                 vnic = &bp->vnic_info[i];
3584
3585                 kfree(vnic->fw_grp_ids);
3586                 vnic->fw_grp_ids = NULL;
3587
3588                 kfree(vnic->uc_list);
3589                 vnic->uc_list = NULL;
3590
3591                 if (vnic->mc_list) {
3592                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3593                                           vnic->mc_list, vnic->mc_list_mapping);
3594                         vnic->mc_list = NULL;
3595                 }
3596
3597                 if (vnic->rss_table) {
3598                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3599                                           vnic->rss_table,
3600                                           vnic->rss_table_dma_addr);
3601                         vnic->rss_table = NULL;
3602                 }
3603
3604                 vnic->rss_hash_key = NULL;
3605                 vnic->flags = 0;
3606         }
3607 }
3608
3609 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3610 {
3611         int i, rc = 0, size;
3612         struct bnxt_vnic_info *vnic;
3613         struct pci_dev *pdev = bp->pdev;
3614         int max_rings;
3615
3616         for (i = 0; i < bp->nr_vnics; i++) {
3617                 vnic = &bp->vnic_info[i];
3618
3619                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3620                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3621
3622                         if (mem_size > 0) {
3623                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3624                                 if (!vnic->uc_list) {
3625                                         rc = -ENOMEM;
3626                                         goto out;
3627                                 }
3628                         }
3629                 }
3630
3631                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3632                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3633                         vnic->mc_list =
3634                                 dma_alloc_coherent(&pdev->dev,
3635                                                    vnic->mc_list_size,
3636                                                    &vnic->mc_list_mapping,
3637                                                    GFP_KERNEL);
3638                         if (!vnic->mc_list) {
3639                                 rc = -ENOMEM;
3640                                 goto out;
3641                         }
3642                 }
3643
3644                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3645                         goto vnic_skip_grps;
3646
3647                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3648                         max_rings = bp->rx_nr_rings;
3649                 else
3650                         max_rings = 1;
3651
3652                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3653                 if (!vnic->fw_grp_ids) {
3654                         rc = -ENOMEM;
3655                         goto out;
3656                 }
3657 vnic_skip_grps:
3658                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3659                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3660                         continue;
3661
3662                 /* Allocate rss table and hash key */
3663                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3664                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3665                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3666
3667                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3668                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3669                                                      vnic->rss_table_size,
3670                                                      &vnic->rss_table_dma_addr,
3671                                                      GFP_KERNEL);
3672                 if (!vnic->rss_table) {
3673                         rc = -ENOMEM;
3674                         goto out;
3675                 }
3676
3677                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3678                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3679         }
3680         return 0;
3681
3682 out:
3683         return rc;
3684 }
3685
3686 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3687 {
3688         struct pci_dev *pdev = bp->pdev;
3689
3690         if (bp->hwrm_cmd_resp_addr) {
3691                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3692                                   bp->hwrm_cmd_resp_dma_addr);
3693                 bp->hwrm_cmd_resp_addr = NULL;
3694         }
3695
3696         if (bp->hwrm_cmd_kong_resp_addr) {
3697                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3698                                   bp->hwrm_cmd_kong_resp_addr,
3699                                   bp->hwrm_cmd_kong_resp_dma_addr);
3700                 bp->hwrm_cmd_kong_resp_addr = NULL;
3701         }
3702 }
3703
3704 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3705 {
3706         struct pci_dev *pdev = bp->pdev;
3707
3708         if (bp->hwrm_cmd_kong_resp_addr)
3709                 return 0;
3710
3711         bp->hwrm_cmd_kong_resp_addr =
3712                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3713                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3714                                    GFP_KERNEL);
3715         if (!bp->hwrm_cmd_kong_resp_addr)
3716                 return -ENOMEM;
3717
3718         return 0;
3719 }
3720
3721 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3722 {
3723         struct pci_dev *pdev = bp->pdev;
3724
3725         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3726                                                    &bp->hwrm_cmd_resp_dma_addr,
3727                                                    GFP_KERNEL);
3728         if (!bp->hwrm_cmd_resp_addr)
3729                 return -ENOMEM;
3730
3731         return 0;
3732 }
3733
3734 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3735 {
3736         if (bp->hwrm_short_cmd_req_addr) {
3737                 struct pci_dev *pdev = bp->pdev;
3738
3739                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3740                                   bp->hwrm_short_cmd_req_addr,
3741                                   bp->hwrm_short_cmd_req_dma_addr);
3742                 bp->hwrm_short_cmd_req_addr = NULL;
3743         }
3744 }
3745
3746 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3747 {
3748         struct pci_dev *pdev = bp->pdev;
3749
3750         if (bp->hwrm_short_cmd_req_addr)
3751                 return 0;
3752
3753         bp->hwrm_short_cmd_req_addr =
3754                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3755                                    &bp->hwrm_short_cmd_req_dma_addr,
3756                                    GFP_KERNEL);
3757         if (!bp->hwrm_short_cmd_req_addr)
3758                 return -ENOMEM;
3759
3760         return 0;
3761 }
3762
3763 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3764 {
3765         kfree(stats->hw_masks);
3766         stats->hw_masks = NULL;
3767         kfree(stats->sw_stats);
3768         stats->sw_stats = NULL;
3769         if (stats->hw_stats) {
3770                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3771                                   stats->hw_stats_map);
3772                 stats->hw_stats = NULL;
3773         }
3774 }
3775
3776 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3777                                 bool alloc_masks)
3778 {
3779         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3780                                              &stats->hw_stats_map, GFP_KERNEL);
3781         if (!stats->hw_stats)
3782                 return -ENOMEM;
3783
3784         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3785         if (!stats->sw_stats)
3786                 goto stats_mem_err;
3787
3788         if (alloc_masks) {
3789                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3790                 if (!stats->hw_masks)
3791                         goto stats_mem_err;
3792         }
3793         return 0;
3794
3795 stats_mem_err:
3796         bnxt_free_stats_mem(bp, stats);
3797         return -ENOMEM;
3798 }
3799
3800 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3801 {
3802         int i;
3803
3804         for (i = 0; i < count; i++)
3805                 mask_arr[i] = mask;
3806 }
3807
3808 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3809 {
3810         int i;
3811
3812         for (i = 0; i < count; i++)
3813                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3814 }
3815
3816 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3817                                     struct bnxt_stats_mem *stats)
3818 {
3819         struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3820         struct hwrm_func_qstats_ext_input req = {0};
3821         __le64 *hw_masks;
3822         int rc;
3823
3824         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3825             !(bp->flags & BNXT_FLAG_CHIP_P5))
3826                 return -EOPNOTSUPP;
3827
3828         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3829         req.fid = cpu_to_le16(0xffff);
3830         req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3831         mutex_lock(&bp->hwrm_cmd_lock);
3832         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3833         if (rc)
3834                 goto qstat_exit;
3835
3836         hw_masks = &resp->rx_ucast_pkts;
3837         bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3838
3839 qstat_exit:
3840         mutex_unlock(&bp->hwrm_cmd_lock);
3841         return rc;
3842 }
3843
3844 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3845 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3846
3847 static void bnxt_init_stats(struct bnxt *bp)
3848 {
3849         struct bnxt_napi *bnapi = bp->bnapi[0];
3850         struct bnxt_cp_ring_info *cpr;
3851         struct bnxt_stats_mem *stats;
3852         __le64 *rx_stats, *tx_stats;
3853         int rc, rx_count, tx_count;
3854         u64 *rx_masks, *tx_masks;
3855         u64 mask;
3856         u8 flags;
3857
3858         cpr = &bnapi->cp_ring;
3859         stats = &cpr->stats;
3860         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3861         if (rc) {
3862                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3863                         mask = (1ULL << 48) - 1;
3864                 else
3865                         mask = -1ULL;
3866                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3867         }
3868         if (bp->flags & BNXT_FLAG_PORT_STATS) {
3869                 stats = &bp->port_stats;
3870                 rx_stats = stats->hw_stats;
3871                 rx_masks = stats->hw_masks;
3872                 rx_count = sizeof(struct rx_port_stats) / 8;
3873                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3874                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3875                 tx_count = sizeof(struct tx_port_stats) / 8;
3876
3877                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3878                 rc = bnxt_hwrm_port_qstats(bp, flags);
3879                 if (rc) {
3880                         mask = (1ULL << 40) - 1;
3881
3882                         bnxt_fill_masks(rx_masks, mask, rx_count);
3883                         bnxt_fill_masks(tx_masks, mask, tx_count);
3884                 } else {
3885                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3886                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3887                         bnxt_hwrm_port_qstats(bp, 0);
3888                 }
3889         }
3890         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3891                 stats = &bp->rx_port_stats_ext;
3892                 rx_stats = stats->hw_stats;
3893                 rx_masks = stats->hw_masks;
3894                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3895                 stats = &bp->tx_port_stats_ext;
3896                 tx_stats = stats->hw_stats;
3897                 tx_masks = stats->hw_masks;
3898                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3899
3900                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3901                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3902                 if (rc) {
3903                         mask = (1ULL << 40) - 1;
3904
3905                         bnxt_fill_masks(rx_masks, mask, rx_count);
3906                         if (tx_stats)
3907                                 bnxt_fill_masks(tx_masks, mask, tx_count);
3908                 } else {
3909                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3910                         if (tx_stats)
3911                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
3912                                                    tx_count);
3913                         bnxt_hwrm_port_qstats_ext(bp, 0);
3914                 }
3915         }
3916 }
3917
3918 static void bnxt_free_port_stats(struct bnxt *bp)
3919 {
3920         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3921         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3922
3923         bnxt_free_stats_mem(bp, &bp->port_stats);
3924         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3925         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3926 }
3927
3928 static void bnxt_free_ring_stats(struct bnxt *bp)
3929 {
3930         int i;
3931
3932         if (!bp->bnapi)
3933                 return;
3934
3935         for (i = 0; i < bp->cp_nr_rings; i++) {
3936                 struct bnxt_napi *bnapi = bp->bnapi[i];
3937                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3938
3939                 bnxt_free_stats_mem(bp, &cpr->stats);
3940         }
3941 }
3942
3943 static int bnxt_alloc_stats(struct bnxt *bp)
3944 {
3945         u32 size, i;
3946         int rc;
3947
3948         size = bp->hw_ring_stats_size;
3949
3950         for (i = 0; i < bp->cp_nr_rings; i++) {
3951                 struct bnxt_napi *bnapi = bp->bnapi[i];
3952                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3953
3954                 cpr->stats.len = size;
3955                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3956                 if (rc)
3957                         return rc;
3958
3959                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3960         }
3961
3962         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3963                 return 0;
3964
3965         if (bp->port_stats.hw_stats)
3966                 goto alloc_ext_stats;
3967
3968         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3969         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3970         if (rc)
3971                 return rc;
3972
3973         bp->flags |= BNXT_FLAG_PORT_STATS;
3974
3975 alloc_ext_stats:
3976         /* Display extended statistics only if FW supports it */
3977         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3978                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3979                         return 0;
3980
3981         if (bp->rx_port_stats_ext.hw_stats)
3982                 goto alloc_tx_ext_stats;
3983
3984         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3985         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3986         /* Extended stats are optional */
3987         if (rc)
3988                 return 0;
3989
3990 alloc_tx_ext_stats:
3991         if (bp->tx_port_stats_ext.hw_stats)
3992                 return 0;
3993
3994         if (bp->hwrm_spec_code >= 0x10902 ||
3995             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3996                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3997                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3998                 /* Extended stats are optional */
3999                 if (rc)
4000                         return 0;
4001         }
4002         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4003         return 0;
4004 }
4005
4006 static void bnxt_clear_ring_indices(struct bnxt *bp)
4007 {
4008         int i;
4009
4010         if (!bp->bnapi)
4011                 return;
4012
4013         for (i = 0; i < bp->cp_nr_rings; i++) {
4014                 struct bnxt_napi *bnapi = bp->bnapi[i];
4015                 struct bnxt_cp_ring_info *cpr;
4016                 struct bnxt_rx_ring_info *rxr;
4017                 struct bnxt_tx_ring_info *txr;
4018
4019                 if (!bnapi)
4020                         continue;
4021
4022                 cpr = &bnapi->cp_ring;
4023                 cpr->cp_raw_cons = 0;
4024
4025                 txr = bnapi->tx_ring;
4026                 if (txr) {
4027                         txr->tx_prod = 0;
4028                         txr->tx_cons = 0;
4029                 }
4030
4031                 rxr = bnapi->rx_ring;
4032                 if (rxr) {
4033                         rxr->rx_prod = 0;
4034                         rxr->rx_agg_prod = 0;
4035                         rxr->rx_sw_agg_prod = 0;
4036                         rxr->rx_next_cons = 0;
4037                 }
4038         }
4039 }
4040
4041 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4042 {
4043 #ifdef CONFIG_RFS_ACCEL
4044         int i;
4045
4046         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4047          * safe to delete the hash table.
4048          */
4049         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4050                 struct hlist_head *head;
4051                 struct hlist_node *tmp;
4052                 struct bnxt_ntuple_filter *fltr;
4053
4054                 head = &bp->ntp_fltr_hash_tbl[i];
4055                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4056                         hlist_del(&fltr->hash);
4057                         kfree(fltr);
4058                 }
4059         }
4060         if (irq_reinit) {
4061                 kfree(bp->ntp_fltr_bmap);
4062                 bp->ntp_fltr_bmap = NULL;
4063         }
4064         bp->ntp_fltr_count = 0;
4065 #endif
4066 }
4067
4068 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4069 {
4070 #ifdef CONFIG_RFS_ACCEL
4071         int i, rc = 0;
4072
4073         if (!(bp->flags & BNXT_FLAG_RFS))
4074                 return 0;
4075
4076         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4077                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4078
4079         bp->ntp_fltr_count = 0;
4080         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4081                                     sizeof(long),
4082                                     GFP_KERNEL);
4083
4084         if (!bp->ntp_fltr_bmap)
4085                 rc = -ENOMEM;
4086
4087         return rc;
4088 #else
4089         return 0;
4090 #endif
4091 }
4092
4093 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4094 {
4095         bnxt_free_vnic_attributes(bp);
4096         bnxt_free_tx_rings(bp);
4097         bnxt_free_rx_rings(bp);
4098         bnxt_free_cp_rings(bp);
4099         bnxt_free_ntp_fltrs(bp, irq_re_init);
4100         if (irq_re_init) {
4101                 bnxt_free_ring_stats(bp);
4102                 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET))
4103                         bnxt_free_port_stats(bp);
4104                 bnxt_free_ring_grps(bp);
4105                 bnxt_free_vnics(bp);
4106                 kfree(bp->tx_ring_map);
4107                 bp->tx_ring_map = NULL;
4108                 kfree(bp->tx_ring);
4109                 bp->tx_ring = NULL;
4110                 kfree(bp->rx_ring);
4111                 bp->rx_ring = NULL;
4112                 kfree(bp->bnapi);
4113                 bp->bnapi = NULL;
4114         } else {
4115                 bnxt_clear_ring_indices(bp);
4116         }
4117 }
4118
4119 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4120 {
4121         int i, j, rc, size, arr_size;
4122         void *bnapi;
4123
4124         if (irq_re_init) {
4125                 /* Allocate bnapi mem pointer array and mem block for
4126                  * all queues
4127                  */
4128                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4129                                 bp->cp_nr_rings);
4130                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4131                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4132                 if (!bnapi)
4133                         return -ENOMEM;
4134
4135                 bp->bnapi = bnapi;
4136                 bnapi += arr_size;
4137                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4138                         bp->bnapi[i] = bnapi;
4139                         bp->bnapi[i]->index = i;
4140                         bp->bnapi[i]->bp = bp;
4141                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4142                                 struct bnxt_cp_ring_info *cpr =
4143                                         &bp->bnapi[i]->cp_ring;
4144
4145                                 cpr->cp_ring_struct.ring_mem.flags =
4146                                         BNXT_RMEM_RING_PTE_FLAG;
4147                         }
4148                 }
4149
4150                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4151                                       sizeof(struct bnxt_rx_ring_info),
4152                                       GFP_KERNEL);
4153                 if (!bp->rx_ring)
4154                         return -ENOMEM;
4155
4156                 for (i = 0; i < bp->rx_nr_rings; i++) {
4157                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4158
4159                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4160                                 rxr->rx_ring_struct.ring_mem.flags =
4161                                         BNXT_RMEM_RING_PTE_FLAG;
4162                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4163                                         BNXT_RMEM_RING_PTE_FLAG;
4164                         }
4165                         rxr->bnapi = bp->bnapi[i];
4166                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4167                 }
4168
4169                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4170                                       sizeof(struct bnxt_tx_ring_info),
4171                                       GFP_KERNEL);
4172                 if (!bp->tx_ring)
4173                         return -ENOMEM;
4174
4175                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4176                                           GFP_KERNEL);
4177
4178                 if (!bp->tx_ring_map)
4179                         return -ENOMEM;
4180
4181                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4182                         j = 0;
4183                 else
4184                         j = bp->rx_nr_rings;
4185
4186                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4187                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4188
4189                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4190                                 txr->tx_ring_struct.ring_mem.flags =
4191                                         BNXT_RMEM_RING_PTE_FLAG;
4192                         txr->bnapi = bp->bnapi[j];
4193                         bp->bnapi[j]->tx_ring = txr;
4194                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4195                         if (i >= bp->tx_nr_rings_xdp) {
4196                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4197                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4198                         } else {
4199                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4200                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4201                         }
4202                 }
4203
4204                 rc = bnxt_alloc_stats(bp);
4205                 if (rc)
4206                         goto alloc_mem_err;
4207                 bnxt_init_stats(bp);
4208
4209                 rc = bnxt_alloc_ntp_fltrs(bp);
4210                 if (rc)
4211                         goto alloc_mem_err;
4212
4213                 rc = bnxt_alloc_vnics(bp);
4214                 if (rc)
4215                         goto alloc_mem_err;
4216         }
4217
4218         bnxt_init_ring_struct(bp);
4219
4220         rc = bnxt_alloc_rx_rings(bp);
4221         if (rc)
4222                 goto alloc_mem_err;
4223
4224         rc = bnxt_alloc_tx_rings(bp);
4225         if (rc)
4226                 goto alloc_mem_err;
4227
4228         rc = bnxt_alloc_cp_rings(bp);
4229         if (rc)
4230                 goto alloc_mem_err;
4231
4232         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4233                                   BNXT_VNIC_UCAST_FLAG;
4234         rc = bnxt_alloc_vnic_attributes(bp);
4235         if (rc)
4236                 goto alloc_mem_err;
4237         return 0;
4238
4239 alloc_mem_err:
4240         bnxt_free_mem(bp, true);
4241         return rc;
4242 }
4243
4244 static void bnxt_disable_int(struct bnxt *bp)
4245 {
4246         int i;
4247
4248         if (!bp->bnapi)
4249                 return;
4250
4251         for (i = 0; i < bp->cp_nr_rings; i++) {
4252                 struct bnxt_napi *bnapi = bp->bnapi[i];
4253                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4254                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4255
4256                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4257                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4258         }
4259 }
4260
4261 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4262 {
4263         struct bnxt_napi *bnapi = bp->bnapi[n];
4264         struct bnxt_cp_ring_info *cpr;
4265
4266         cpr = &bnapi->cp_ring;
4267         return cpr->cp_ring_struct.map_idx;
4268 }
4269
4270 static void bnxt_disable_int_sync(struct bnxt *bp)
4271 {
4272         int i;
4273
4274         atomic_inc(&bp->intr_sem);
4275
4276         bnxt_disable_int(bp);
4277         for (i = 0; i < bp->cp_nr_rings; i++) {
4278                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4279
4280                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4281         }
4282 }
4283
4284 static void bnxt_enable_int(struct bnxt *bp)
4285 {
4286         int i;
4287
4288         atomic_set(&bp->intr_sem, 0);
4289         for (i = 0; i < bp->cp_nr_rings; i++) {
4290                 struct bnxt_napi *bnapi = bp->bnapi[i];
4291                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4292
4293                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4294         }
4295 }
4296
4297 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4298                             u16 cmpl_ring, u16 target_id)
4299 {
4300         struct input *req = request;
4301
4302         req->req_type = cpu_to_le16(req_type);
4303         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4304         req->target_id = cpu_to_le16(target_id);
4305         if (bnxt_kong_hwrm_message(bp, req))
4306                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4307         else
4308                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4309 }
4310
4311 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4312 {
4313         switch (hwrm_err) {
4314         case HWRM_ERR_CODE_SUCCESS:
4315                 return 0;
4316         case HWRM_ERR_CODE_RESOURCE_LOCKED:
4317                 return -EROFS;
4318         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4319                 return -EACCES;
4320         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4321                 return -ENOSPC;
4322         case HWRM_ERR_CODE_INVALID_PARAMS:
4323         case HWRM_ERR_CODE_INVALID_FLAGS:
4324         case HWRM_ERR_CODE_INVALID_ENABLES:
4325         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4326         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4327                 return -EINVAL;
4328         case HWRM_ERR_CODE_NO_BUFFER:
4329                 return -ENOMEM;
4330         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4331         case HWRM_ERR_CODE_BUSY:
4332                 return -EAGAIN;
4333         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4334                 return -EOPNOTSUPP;
4335         default:
4336                 return -EIO;
4337         }
4338 }
4339
4340 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4341                                  int timeout, bool silent)
4342 {
4343         int i, intr_process, rc, tmo_count;
4344         struct input *req = msg;
4345         u32 *data = msg;
4346         u8 *valid;
4347         u16 cp_ring_id, len = 0;
4348         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4349         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4350         struct hwrm_short_input short_input = {0};
4351         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4352         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4353         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4354
4355         if (BNXT_NO_FW_ACCESS(bp) &&
4356             le16_to_cpu(req->req_type) != HWRM_FUNC_RESET)
4357                 return -EBUSY;
4358
4359         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4360                 if (msg_len > bp->hwrm_max_ext_req_len ||
4361                     !bp->hwrm_short_cmd_req_addr)
4362                         return -EINVAL;
4363         }
4364
4365         if (bnxt_hwrm_kong_chnl(bp, req)) {
4366                 dst = BNXT_HWRM_CHNL_KONG;
4367                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4368                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4369                 resp = bp->hwrm_cmd_kong_resp_addr;
4370         }
4371
4372         memset(resp, 0, PAGE_SIZE);
4373         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4374         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4375
4376         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4377         /* currently supports only one outstanding message */
4378         if (intr_process)
4379                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4380
4381         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4382             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4383                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4384                 u16 max_msg_len;
4385
4386                 /* Set boundary for maximum extended request length for short
4387                  * cmd format. If passed up from device use the max supported
4388                  * internal req length.
4389                  */
4390                 max_msg_len = bp->hwrm_max_ext_req_len;
4391
4392                 memcpy(short_cmd_req, req, msg_len);
4393                 if (msg_len < max_msg_len)
4394                         memset(short_cmd_req + msg_len, 0,
4395                                max_msg_len - msg_len);
4396
4397                 short_input.req_type = req->req_type;
4398                 short_input.signature =
4399                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4400                 short_input.size = cpu_to_le16(msg_len);
4401                 short_input.req_addr =
4402                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4403
4404                 data = (u32 *)&short_input;
4405                 msg_len = sizeof(short_input);
4406
4407                 /* Sync memory write before updating doorbell */
4408                 wmb();
4409
4410                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4411         }
4412
4413         /* Write request msg to hwrm channel */
4414         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4415
4416         for (i = msg_len; i < max_req_len; i += 4)
4417                 writel(0, bp->bar0 + bar_offset + i);
4418
4419         /* Ring channel doorbell */
4420         writel(1, bp->bar0 + doorbell_offset);
4421
4422         if (!pci_is_enabled(bp->pdev))
4423                 return 0;
4424
4425         if (!timeout)
4426                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4427         /* convert timeout to usec */
4428         timeout *= 1000;
4429
4430         i = 0;
4431         /* Short timeout for the first few iterations:
4432          * number of loops = number of loops for short timeout +
4433          * number of loops for standard timeout.
4434          */
4435         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4436         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4437         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4438
4439         if (intr_process) {
4440                 u16 seq_id = bp->hwrm_intr_seq_id;
4441
4442                 /* Wait until hwrm response cmpl interrupt is processed */
4443                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4444                        i++ < tmo_count) {
4445                         /* Abort the wait for completion if the FW health
4446                          * check has failed.
4447                          */
4448                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4449                                 return -EBUSY;
4450                         /* on first few passes, just barely sleep */
4451                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4452                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4453                                              HWRM_SHORT_MAX_TIMEOUT);
4454                         else
4455                                 usleep_range(HWRM_MIN_TIMEOUT,
4456                                              HWRM_MAX_TIMEOUT);
4457                 }
4458
4459                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4460                         if (!silent)
4461                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4462                                            le16_to_cpu(req->req_type));
4463                         return -EBUSY;
4464                 }
4465                 len = le16_to_cpu(resp->resp_len);
4466                 valid = ((u8 *)resp) + len - 1;
4467         } else {
4468                 int j;
4469
4470                 /* Check if response len is updated */
4471                 for (i = 0; i < tmo_count; i++) {
4472                         /* Abort the wait for completion if the FW health
4473                          * check has failed.
4474                          */
4475                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4476                                 return -EBUSY;
4477                         len = le16_to_cpu(resp->resp_len);
4478                         if (len)
4479                                 break;
4480                         /* on first few passes, just barely sleep */
4481                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4482                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4483                                              HWRM_SHORT_MAX_TIMEOUT);
4484                         else
4485                                 usleep_range(HWRM_MIN_TIMEOUT,
4486                                              HWRM_MAX_TIMEOUT);
4487                 }
4488
4489                 if (i >= tmo_count) {
4490                         if (!silent)
4491                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4492                                            HWRM_TOTAL_TIMEOUT(i),
4493                                            le16_to_cpu(req->req_type),
4494                                            le16_to_cpu(req->seq_id), len);
4495                         return -EBUSY;
4496                 }
4497
4498                 /* Last byte of resp contains valid bit */
4499                 valid = ((u8 *)resp) + len - 1;
4500                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4501                         /* make sure we read from updated DMA memory */
4502                         dma_rmb();
4503                         if (*valid)
4504                                 break;
4505                         usleep_range(1, 5);
4506                 }
4507
4508                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4509                         if (!silent)
4510                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4511                                            HWRM_TOTAL_TIMEOUT(i),
4512                                            le16_to_cpu(req->req_type),
4513                                            le16_to_cpu(req->seq_id), len,
4514                                            *valid);
4515                         return -EBUSY;
4516                 }
4517         }
4518
4519         /* Zero valid bit for compatibility.  Valid bit in an older spec
4520          * may become a new field in a newer spec.  We must make sure that
4521          * a new field not implemented by old spec will read zero.
4522          */
4523         *valid = 0;
4524         rc = le16_to_cpu(resp->error_code);
4525         if (rc && !silent)
4526                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4527                            le16_to_cpu(resp->req_type),
4528                            le16_to_cpu(resp->seq_id), rc);
4529         return bnxt_hwrm_to_stderr(rc);
4530 }
4531
4532 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4533 {
4534         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4535 }
4536
4537 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4538                               int timeout)
4539 {
4540         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4541 }
4542
4543 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4544 {
4545         int rc;
4546
4547         mutex_lock(&bp->hwrm_cmd_lock);
4548         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4549         mutex_unlock(&bp->hwrm_cmd_lock);
4550         return rc;
4551 }
4552
4553 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4554                              int timeout)
4555 {
4556         int rc;
4557
4558         mutex_lock(&bp->hwrm_cmd_lock);
4559         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4560         mutex_unlock(&bp->hwrm_cmd_lock);
4561         return rc;
4562 }
4563
4564 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4565                             bool async_only)
4566 {
4567         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4568         struct hwrm_func_drv_rgtr_input req = {0};
4569         DECLARE_BITMAP(async_events_bmap, 256);
4570         u32 *events = (u32 *)async_events_bmap;
4571         u32 flags;
4572         int rc, i;
4573
4574         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4575
4576         req.enables =
4577                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4578                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
4579                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4580
4581         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4582         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4583         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4584                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4585         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4586                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4587                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4588         req.flags = cpu_to_le32(flags);
4589         req.ver_maj_8b = DRV_VER_MAJ;
4590         req.ver_min_8b = DRV_VER_MIN;
4591         req.ver_upd_8b = DRV_VER_UPD;
4592         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4593         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4594         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4595
4596         if (BNXT_PF(bp)) {
4597                 u32 data[8];
4598                 int i;
4599
4600                 memset(data, 0, sizeof(data));
4601                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4602                         u16 cmd = bnxt_vf_req_snif[i];
4603                         unsigned int bit, idx;
4604
4605                         idx = cmd / 32;
4606                         bit = cmd % 32;
4607                         data[idx] |= 1 << bit;
4608                 }
4609
4610                 for (i = 0; i < 8; i++)
4611                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4612
4613                 req.enables |=
4614                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4615         }
4616
4617         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4618                 req.flags |= cpu_to_le32(
4619                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4620
4621         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4622         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4623                 u16 event_id = bnxt_async_events_arr[i];
4624
4625                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4626                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4627                         continue;
4628                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4629         }
4630         if (bmap && bmap_size) {
4631                 for (i = 0; i < bmap_size; i++) {
4632                         if (test_bit(i, bmap))
4633                                 __set_bit(i, async_events_bmap);
4634                 }
4635         }
4636         for (i = 0; i < 8; i++)
4637                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4638
4639         if (async_only)
4640                 req.enables =
4641                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4642
4643         mutex_lock(&bp->hwrm_cmd_lock);
4644         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4645         if (!rc) {
4646                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4647                 if (resp->flags &
4648                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4649                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4650         }
4651         mutex_unlock(&bp->hwrm_cmd_lock);
4652         return rc;
4653 }
4654
4655 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4656 {
4657         struct hwrm_func_drv_unrgtr_input req = {0};
4658
4659         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4660                 return 0;
4661
4662         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4663         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4664 }
4665
4666 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4667 {
4668         u32 rc = 0;
4669         struct hwrm_tunnel_dst_port_free_input req = {0};
4670
4671         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4672         req.tunnel_type = tunnel_type;
4673
4674         switch (tunnel_type) {
4675         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4676                 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4677                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4678                 break;
4679         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4680                 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4681                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4682                 break;
4683         default:
4684                 break;
4685         }
4686
4687         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4688         if (rc)
4689                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4690                            rc);
4691         return rc;
4692 }
4693
4694 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4695                                            u8 tunnel_type)
4696 {
4697         u32 rc = 0;
4698         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4699         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4700
4701         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4702
4703         req.tunnel_type = tunnel_type;
4704         req.tunnel_dst_port_val = port;
4705
4706         mutex_lock(&bp->hwrm_cmd_lock);
4707         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4708         if (rc) {
4709                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4710                            rc);
4711                 goto err_out;
4712         }
4713
4714         switch (tunnel_type) {
4715         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4716                 bp->vxlan_fw_dst_port_id =
4717                         le16_to_cpu(resp->tunnel_dst_port_id);
4718                 break;
4719         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4720                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4721                 break;
4722         default:
4723                 break;
4724         }
4725
4726 err_out:
4727         mutex_unlock(&bp->hwrm_cmd_lock);
4728         return rc;
4729 }
4730
4731 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4732 {
4733         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4734         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4735
4736         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4737         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4738
4739         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4740         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4741         req.mask = cpu_to_le32(vnic->rx_mask);
4742         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4743 }
4744
4745 #ifdef CONFIG_RFS_ACCEL
4746 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4747                                             struct bnxt_ntuple_filter *fltr)
4748 {
4749         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4750
4751         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4752         req.ntuple_filter_id = fltr->filter_id;
4753         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4754 }
4755
4756 #define BNXT_NTP_FLTR_FLAGS                                     \
4757         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4758          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4759          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4760          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4761          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4762          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4763          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4764          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4765          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4766          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4767          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4768          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4769          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4770          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4771
4772 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4773                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4774
4775 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4776                                              struct bnxt_ntuple_filter *fltr)
4777 {
4778         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4779         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4780         struct flow_keys *keys = &fltr->fkeys;
4781         struct bnxt_vnic_info *vnic;
4782         u32 flags = 0;
4783         int rc = 0;
4784
4785         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4786         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4787
4788         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4789                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4790                 req.dst_id = cpu_to_le16(fltr->rxq);
4791         } else {
4792                 vnic = &bp->vnic_info[fltr->rxq + 1];
4793                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4794         }
4795         req.flags = cpu_to_le32(flags);
4796         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4797
4798         req.ethertype = htons(ETH_P_IP);
4799         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4800         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4801         req.ip_protocol = keys->basic.ip_proto;
4802
4803         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4804                 int i;
4805
4806                 req.ethertype = htons(ETH_P_IPV6);
4807                 req.ip_addr_type =
4808                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4809                 *(struct in6_addr *)&req.src_ipaddr[0] =
4810                         keys->addrs.v6addrs.src;
4811                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4812                         keys->addrs.v6addrs.dst;
4813                 for (i = 0; i < 4; i++) {
4814                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4815                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4816                 }
4817         } else {
4818                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4819                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4820                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4821                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4822         }
4823         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4824                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4825                 req.tunnel_type =
4826                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4827         }
4828
4829         req.src_port = keys->ports.src;
4830         req.src_port_mask = cpu_to_be16(0xffff);
4831         req.dst_port = keys->ports.dst;
4832         req.dst_port_mask = cpu_to_be16(0xffff);
4833
4834         mutex_lock(&bp->hwrm_cmd_lock);
4835         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4836         if (!rc) {
4837                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4838                 fltr->filter_id = resp->ntuple_filter_id;
4839         }
4840         mutex_unlock(&bp->hwrm_cmd_lock);
4841         return rc;
4842 }
4843 #endif
4844
4845 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4846                                      u8 *mac_addr)
4847 {
4848         u32 rc = 0;
4849         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4850         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4851
4852         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4853         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4854         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4855                 req.flags |=
4856                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4857         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4858         req.enables =
4859                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4860                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4861                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4862         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4863         req.l2_addr_mask[0] = 0xff;
4864         req.l2_addr_mask[1] = 0xff;
4865         req.l2_addr_mask[2] = 0xff;
4866         req.l2_addr_mask[3] = 0xff;
4867         req.l2_addr_mask[4] = 0xff;
4868         req.l2_addr_mask[5] = 0xff;
4869
4870         mutex_lock(&bp->hwrm_cmd_lock);
4871         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4872         if (!rc)
4873                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4874                                                         resp->l2_filter_id;
4875         mutex_unlock(&bp->hwrm_cmd_lock);
4876         return rc;
4877 }
4878
4879 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4880 {
4881         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4882         int rc = 0;
4883
4884         /* Any associated ntuple filters will also be cleared by firmware. */
4885         mutex_lock(&bp->hwrm_cmd_lock);
4886         for (i = 0; i < num_of_vnics; i++) {
4887                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4888
4889                 for (j = 0; j < vnic->uc_filter_count; j++) {
4890                         struct hwrm_cfa_l2_filter_free_input req = {0};
4891
4892                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4893                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4894
4895                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4896
4897                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4898                                                 HWRM_CMD_TIMEOUT);
4899                 }
4900                 vnic->uc_filter_count = 0;
4901         }
4902         mutex_unlock(&bp->hwrm_cmd_lock);
4903
4904         return rc;
4905 }
4906
4907 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4908 {
4909         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4910         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4911         struct hwrm_vnic_tpa_cfg_input req = {0};
4912
4913         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4914                 return 0;
4915
4916         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4917
4918         if (tpa_flags) {
4919                 u16 mss = bp->dev->mtu - 40;
4920                 u32 nsegs, n, segs = 0, flags;
4921
4922                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4923                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4924                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4925                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4926                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4927                 if (tpa_flags & BNXT_FLAG_GRO)
4928                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4929
4930                 req.flags = cpu_to_le32(flags);
4931
4932                 req.enables =
4933                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4934                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4935                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4936
4937                 /* Number of segs are log2 units, and first packet is not
4938                  * included as part of this units.
4939                  */
4940                 if (mss <= BNXT_RX_PAGE_SIZE) {
4941                         n = BNXT_RX_PAGE_SIZE / mss;
4942                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4943                 } else {
4944                         n = mss / BNXT_RX_PAGE_SIZE;
4945                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4946                                 n++;
4947                         nsegs = (MAX_SKB_FRAGS - n) / n;
4948                 }
4949
4950                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4951                         segs = MAX_TPA_SEGS_P5;
4952                         max_aggs = bp->max_tpa;
4953                 } else {
4954                         segs = ilog2(nsegs);
4955                 }
4956                 req.max_agg_segs = cpu_to_le16(segs);
4957                 req.max_aggs = cpu_to_le16(max_aggs);
4958
4959                 req.min_agg_len = cpu_to_le32(512);
4960         }
4961         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4962
4963         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4964 }
4965
4966 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4967 {
4968         struct bnxt_ring_grp_info *grp_info;
4969
4970         grp_info = &bp->grp_info[ring->grp_idx];
4971         return grp_info->cp_fw_ring_id;
4972 }
4973
4974 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4975 {
4976         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4977                 struct bnxt_napi *bnapi = rxr->bnapi;
4978                 struct bnxt_cp_ring_info *cpr;
4979
4980                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4981                 return cpr->cp_ring_struct.fw_ring_id;
4982         } else {
4983                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4984         }
4985 }
4986
4987 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4988 {
4989         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4990                 struct bnxt_napi *bnapi = txr->bnapi;
4991                 struct bnxt_cp_ring_info *cpr;
4992
4993                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4994                 return cpr->cp_ring_struct.fw_ring_id;
4995         } else {
4996                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4997         }
4998 }
4999
5000 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5001 {
5002         int entries;
5003
5004         if (bp->flags & BNXT_FLAG_CHIP_P5)
5005                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5006         else
5007                 entries = HW_HASH_INDEX_SIZE;
5008
5009         bp->rss_indir_tbl_entries = entries;
5010         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5011                                           GFP_KERNEL);
5012         if (!bp->rss_indir_tbl)
5013                 return -ENOMEM;
5014         return 0;
5015 }
5016
5017 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5018 {
5019         u16 max_rings, max_entries, pad, i;
5020
5021         if (!bp->rx_nr_rings)
5022                 return;
5023
5024         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5025                 max_rings = bp->rx_nr_rings - 1;
5026         else
5027                 max_rings = bp->rx_nr_rings;
5028
5029         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5030
5031         for (i = 0; i < max_entries; i++)
5032                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5033
5034         pad = bp->rss_indir_tbl_entries - max_entries;
5035         if (pad)
5036                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5037 }
5038
5039 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5040 {
5041         u16 i, tbl_size, max_ring = 0;
5042
5043         if (!bp->rss_indir_tbl)
5044                 return 0;
5045
5046         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5047         for (i = 0; i < tbl_size; i++)
5048                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5049         return max_ring;
5050 }
5051
5052 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5053 {
5054         if (bp->flags & BNXT_FLAG_CHIP_P5)
5055                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5056         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5057                 return 2;
5058         return 1;
5059 }
5060
5061 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5062 {
5063         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5064         u16 i, j;
5065
5066         /* Fill the RSS indirection table with ring group ids */
5067         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5068                 if (!no_rss)
5069                         j = bp->rss_indir_tbl[i];
5070                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5071         }
5072 }
5073
5074 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5075                                       struct bnxt_vnic_info *vnic)
5076 {
5077         __le16 *ring_tbl = vnic->rss_table;
5078         struct bnxt_rx_ring_info *rxr;
5079         u16 tbl_size, i;
5080
5081         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5082
5083         for (i = 0; i < tbl_size; i++) {
5084                 u16 ring_id, j;
5085
5086                 j = bp->rss_indir_tbl[i];
5087                 rxr = &bp->rx_ring[j];
5088
5089                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5090                 *ring_tbl++ = cpu_to_le16(ring_id);
5091                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5092                 *ring_tbl++ = cpu_to_le16(ring_id);
5093         }
5094 }
5095
5096 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5097 {
5098         if (bp->flags & BNXT_FLAG_CHIP_P5)
5099                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5100         else
5101                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5102 }
5103
5104 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5105 {
5106         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5107         struct hwrm_vnic_rss_cfg_input req = {0};
5108
5109         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5110             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5111                 return 0;
5112
5113         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5114         if (set_rss) {
5115                 bnxt_fill_hw_rss_tbl(bp, vnic);
5116                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5117                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5118                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5119                 req.hash_key_tbl_addr =
5120                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5121         }
5122         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5123         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5124 }
5125
5126 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5127 {
5128         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5129         struct hwrm_vnic_rss_cfg_input req = {0};
5130         dma_addr_t ring_tbl_map;
5131         u32 i, nr_ctxs;
5132
5133         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5134         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5135         if (!set_rss) {
5136                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5137                 return 0;
5138         }
5139         bnxt_fill_hw_rss_tbl(bp, vnic);
5140         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5141         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5142         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5143         ring_tbl_map = vnic->rss_table_dma_addr;
5144         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5145         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5146                 int rc;
5147
5148                 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5149                 req.ring_table_pair_index = i;
5150                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5151                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5152                 if (rc)
5153                         return rc;
5154         }
5155         return 0;
5156 }
5157
5158 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5159 {
5160         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5161         struct hwrm_vnic_plcmodes_cfg_input req = {0};
5162
5163         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5164         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5165                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5166                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5167         req.enables =
5168                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5169                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5170         /* thresholds not implemented in firmware yet */
5171         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5172         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5173         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5174         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5175 }
5176
5177 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5178                                         u16 ctx_idx)
5179 {
5180         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5181
5182         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5183         req.rss_cos_lb_ctx_id =
5184                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5185
5186         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5187         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5188 }
5189
5190 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5191 {
5192         int i, j;
5193
5194         for (i = 0; i < bp->nr_vnics; i++) {
5195                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5196
5197                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5198                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5199                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5200                 }
5201         }
5202         bp->rsscos_nr_ctxs = 0;
5203 }
5204
5205 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5206 {
5207         int rc;
5208         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5209         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5210                                                 bp->hwrm_cmd_resp_addr;
5211
5212         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5213                                -1);
5214
5215         mutex_lock(&bp->hwrm_cmd_lock);
5216         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5217         if (!rc)
5218                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5219                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5220         mutex_unlock(&bp->hwrm_cmd_lock);
5221
5222         return rc;
5223 }
5224
5225 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5226 {
5227         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5228                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5229         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5230 }
5231
5232 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5233 {
5234         unsigned int ring = 0, grp_idx;
5235         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5236         struct hwrm_vnic_cfg_input req = {0};
5237         u16 def_vlan = 0;
5238
5239         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5240
5241         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5242                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5243
5244                 req.default_rx_ring_id =
5245                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5246                 req.default_cmpl_ring_id =
5247                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5248                 req.enables =
5249                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5250                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5251                 goto vnic_mru;
5252         }
5253         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5254         /* Only RSS support for now TBD: COS & LB */
5255         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5256                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5257                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5258                                            VNIC_CFG_REQ_ENABLES_MRU);
5259         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5260                 req.rss_rule =
5261                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5262                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5263                                            VNIC_CFG_REQ_ENABLES_MRU);
5264                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5265         } else {
5266                 req.rss_rule = cpu_to_le16(0xffff);
5267         }
5268
5269         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5270             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5271                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5272                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5273         } else {
5274                 req.cos_rule = cpu_to_le16(0xffff);
5275         }
5276
5277         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5278                 ring = 0;
5279         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5280                 ring = vnic_id - 1;
5281         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5282                 ring = bp->rx_nr_rings - 1;
5283
5284         grp_idx = bp->rx_ring[ring].bnapi->index;
5285         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5286         req.lb_rule = cpu_to_le16(0xffff);
5287 vnic_mru:
5288         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5289
5290         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5291 #ifdef CONFIG_BNXT_SRIOV
5292         if (BNXT_VF(bp))
5293                 def_vlan = bp->vf.vlan;
5294 #endif
5295         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5296                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5297         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5298                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5299
5300         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5301 }
5302
5303 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5304 {
5305         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5306                 struct hwrm_vnic_free_input req = {0};
5307
5308                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5309                 req.vnic_id =
5310                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5311
5312                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5313                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5314         }
5315 }
5316
5317 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5318 {
5319         u16 i;
5320
5321         for (i = 0; i < bp->nr_vnics; i++)
5322                 bnxt_hwrm_vnic_free_one(bp, i);
5323 }
5324
5325 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5326                                 unsigned int start_rx_ring_idx,
5327                                 unsigned int nr_rings)
5328 {
5329         int rc = 0;
5330         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5331         struct hwrm_vnic_alloc_input req = {0};
5332         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5333         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5334
5335         if (bp->flags & BNXT_FLAG_CHIP_P5)
5336                 goto vnic_no_ring_grps;
5337
5338         /* map ring groups to this vnic */
5339         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5340                 grp_idx = bp->rx_ring[i].bnapi->index;
5341                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5342                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5343                                    j, nr_rings);
5344                         break;
5345                 }
5346                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5347         }
5348
5349 vnic_no_ring_grps:
5350         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5351                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5352         if (vnic_id == 0)
5353                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5354
5355         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5356
5357         mutex_lock(&bp->hwrm_cmd_lock);
5358         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5359         if (!rc)
5360                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5361         mutex_unlock(&bp->hwrm_cmd_lock);
5362         return rc;
5363 }
5364
5365 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5366 {
5367         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5368         struct hwrm_vnic_qcaps_input req = {0};
5369         int rc;
5370
5371         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5372         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5373         if (bp->hwrm_spec_code < 0x10600)
5374                 return 0;
5375
5376         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5377         mutex_lock(&bp->hwrm_cmd_lock);
5378         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5379         if (!rc) {
5380                 u32 flags = le32_to_cpu(resp->flags);
5381
5382                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5383                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5384                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5385                 if (flags &
5386                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5387                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5388
5389                 /* Older P5 fw before EXT_HW_STATS support did not set
5390                  * VLAN_STRIP_CAP properly.
5391                  */
5392                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5393                     (BNXT_CHIP_P5_THOR(bp) &&
5394                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5395                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5396                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5397                 if (bp->max_tpa_v2) {
5398                         if (BNXT_CHIP_P5_THOR(bp))
5399                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5400                         else
5401                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5402                 }
5403         }
5404         mutex_unlock(&bp->hwrm_cmd_lock);
5405         return rc;
5406 }
5407
5408 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5409 {
5410         u16 i;
5411         u32 rc = 0;
5412
5413         if (bp->flags & BNXT_FLAG_CHIP_P5)
5414                 return 0;
5415
5416         mutex_lock(&bp->hwrm_cmd_lock);
5417         for (i = 0; i < bp->rx_nr_rings; i++) {
5418                 struct hwrm_ring_grp_alloc_input req = {0};
5419                 struct hwrm_ring_grp_alloc_output *resp =
5420                                         bp->hwrm_cmd_resp_addr;
5421                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5422
5423                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5424
5425                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5426                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5427                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5428                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5429
5430                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5431                                         HWRM_CMD_TIMEOUT);
5432                 if (rc)
5433                         break;
5434
5435                 bp->grp_info[grp_idx].fw_grp_id =
5436                         le32_to_cpu(resp->ring_group_id);
5437         }
5438         mutex_unlock(&bp->hwrm_cmd_lock);
5439         return rc;
5440 }
5441
5442 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5443 {
5444         u16 i;
5445         struct hwrm_ring_grp_free_input req = {0};
5446
5447         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5448                 return;
5449
5450         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5451
5452         mutex_lock(&bp->hwrm_cmd_lock);
5453         for (i = 0; i < bp->cp_nr_rings; i++) {
5454                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5455                         continue;
5456                 req.ring_group_id =
5457                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5458
5459                 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5460                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5461         }
5462         mutex_unlock(&bp->hwrm_cmd_lock);
5463 }
5464
5465 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5466                                     struct bnxt_ring_struct *ring,
5467                                     u32 ring_type, u32 map_index)
5468 {
5469         int rc = 0, err = 0;
5470         struct hwrm_ring_alloc_input req = {0};
5471         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5472         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5473         struct bnxt_ring_grp_info *grp_info;
5474         u16 ring_id;
5475
5476         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5477
5478         req.enables = 0;
5479         if (rmem->nr_pages > 1) {
5480                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5481                 /* Page size is in log2 units */
5482                 req.page_size = BNXT_PAGE_SHIFT;
5483                 req.page_tbl_depth = 1;
5484         } else {
5485                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5486         }
5487         req.fbo = 0;
5488         /* Association of ring index with doorbell index and MSIX number */
5489         req.logical_id = cpu_to_le16(map_index);
5490
5491         switch (ring_type) {
5492         case HWRM_RING_ALLOC_TX: {
5493                 struct bnxt_tx_ring_info *txr;
5494
5495                 txr = container_of(ring, struct bnxt_tx_ring_info,
5496                                    tx_ring_struct);
5497                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5498                 /* Association of transmit ring with completion ring */
5499                 grp_info = &bp->grp_info[ring->grp_idx];
5500                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5501                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5502                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5503                 req.queue_id = cpu_to_le16(ring->queue_id);
5504                 break;
5505         }
5506         case HWRM_RING_ALLOC_RX:
5507                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5508                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5509                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5510                         u16 flags = 0;
5511
5512                         /* Association of rx ring with stats context */
5513                         grp_info = &bp->grp_info[ring->grp_idx];
5514                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5515                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5516                         req.enables |= cpu_to_le32(
5517                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5518                         if (NET_IP_ALIGN == 2)
5519                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5520                         req.flags = cpu_to_le16(flags);
5521                 }
5522                 break;
5523         case HWRM_RING_ALLOC_AGG:
5524                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5525                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5526                         /* Association of agg ring with rx ring */
5527                         grp_info = &bp->grp_info[ring->grp_idx];
5528                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5529                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5530                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5531                         req.enables |= cpu_to_le32(
5532                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5533                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5534                 } else {
5535                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5536                 }
5537                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5538                 break;
5539         case HWRM_RING_ALLOC_CMPL:
5540                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5541                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5542                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5543                         /* Association of cp ring with nq */
5544                         grp_info = &bp->grp_info[map_index];
5545                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5546                         req.cq_handle = cpu_to_le64(ring->handle);
5547                         req.enables |= cpu_to_le32(
5548                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5549                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5550                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5551                 }
5552                 break;
5553         case HWRM_RING_ALLOC_NQ:
5554                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5555                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5556                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5557                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5558                 break;
5559         default:
5560                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5561                            ring_type);
5562                 return -1;
5563         }
5564
5565         mutex_lock(&bp->hwrm_cmd_lock);
5566         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5567         err = le16_to_cpu(resp->error_code);
5568         ring_id = le16_to_cpu(resp->ring_id);
5569         mutex_unlock(&bp->hwrm_cmd_lock);
5570
5571         if (rc || err) {
5572                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5573                            ring_type, rc, err);
5574                 return -EIO;
5575         }
5576         ring->fw_ring_id = ring_id;
5577         return rc;
5578 }
5579
5580 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5581 {
5582         int rc;
5583
5584         if (BNXT_PF(bp)) {
5585                 struct hwrm_func_cfg_input req = {0};
5586
5587                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5588                 req.fid = cpu_to_le16(0xffff);
5589                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5590                 req.async_event_cr = cpu_to_le16(idx);
5591                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5592         } else {
5593                 struct hwrm_func_vf_cfg_input req = {0};
5594
5595                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5596                 req.enables =
5597                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5598                 req.async_event_cr = cpu_to_le16(idx);
5599                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5600         }
5601         return rc;
5602 }
5603
5604 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5605                         u32 map_idx, u32 xid)
5606 {
5607         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5608                 if (BNXT_PF(bp))
5609                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5610                 else
5611                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5612                 switch (ring_type) {
5613                 case HWRM_RING_ALLOC_TX:
5614                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5615                         break;
5616                 case HWRM_RING_ALLOC_RX:
5617                 case HWRM_RING_ALLOC_AGG:
5618                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5619                         break;
5620                 case HWRM_RING_ALLOC_CMPL:
5621                         db->db_key64 = DBR_PATH_L2;
5622                         break;
5623                 case HWRM_RING_ALLOC_NQ:
5624                         db->db_key64 = DBR_PATH_L2;
5625                         break;
5626                 }
5627                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5628         } else {
5629                 db->doorbell = bp->bar1 + map_idx * 0x80;
5630                 switch (ring_type) {
5631                 case HWRM_RING_ALLOC_TX:
5632                         db->db_key32 = DB_KEY_TX;
5633                         break;
5634                 case HWRM_RING_ALLOC_RX:
5635                 case HWRM_RING_ALLOC_AGG:
5636                         db->db_key32 = DB_KEY_RX;
5637                         break;
5638                 case HWRM_RING_ALLOC_CMPL:
5639                         db->db_key32 = DB_KEY_CP;
5640                         break;
5641                 }
5642         }
5643 }
5644
5645 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5646 {
5647         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5648         int i, rc = 0;
5649         u32 type;
5650
5651         if (bp->flags & BNXT_FLAG_CHIP_P5)
5652                 type = HWRM_RING_ALLOC_NQ;
5653         else
5654                 type = HWRM_RING_ALLOC_CMPL;
5655         for (i = 0; i < bp->cp_nr_rings; i++) {
5656                 struct bnxt_napi *bnapi = bp->bnapi[i];
5657                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5658                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5659                 u32 map_idx = ring->map_idx;
5660                 unsigned int vector;
5661
5662                 vector = bp->irq_tbl[map_idx].vector;
5663                 disable_irq_nosync(vector);
5664                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5665                 if (rc) {
5666                         enable_irq(vector);
5667                         goto err_out;
5668                 }
5669                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5670                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5671                 enable_irq(vector);
5672                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5673
5674                 if (!i) {
5675                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5676                         if (rc)
5677                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5678                 }
5679         }
5680
5681         type = HWRM_RING_ALLOC_TX;
5682         for (i = 0; i < bp->tx_nr_rings; i++) {
5683                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5684                 struct bnxt_ring_struct *ring;
5685                 u32 map_idx;
5686
5687                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5688                         struct bnxt_napi *bnapi = txr->bnapi;
5689                         struct bnxt_cp_ring_info *cpr, *cpr2;
5690                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5691
5692                         cpr = &bnapi->cp_ring;
5693                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5694                         ring = &cpr2->cp_ring_struct;
5695                         ring->handle = BNXT_TX_HDL;
5696                         map_idx = bnapi->index;
5697                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5698                         if (rc)
5699                                 goto err_out;
5700                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5701                                     ring->fw_ring_id);
5702                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5703                 }
5704                 ring = &txr->tx_ring_struct;
5705                 map_idx = i;
5706                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5707                 if (rc)
5708                         goto err_out;
5709                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5710         }
5711
5712         type = HWRM_RING_ALLOC_RX;
5713         for (i = 0; i < bp->rx_nr_rings; i++) {
5714                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5715                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5716                 struct bnxt_napi *bnapi = rxr->bnapi;
5717                 u32 map_idx = bnapi->index;
5718
5719                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5720                 if (rc)
5721                         goto err_out;
5722                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5723                 /* If we have agg rings, post agg buffers first. */
5724                 if (!agg_rings)
5725                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5726                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5727                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5728                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5729                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5730                         struct bnxt_cp_ring_info *cpr2;
5731
5732                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5733                         ring = &cpr2->cp_ring_struct;
5734                         ring->handle = BNXT_RX_HDL;
5735                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5736                         if (rc)
5737                                 goto err_out;
5738                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5739                                     ring->fw_ring_id);
5740                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5741                 }
5742         }
5743
5744         if (agg_rings) {
5745                 type = HWRM_RING_ALLOC_AGG;
5746                 for (i = 0; i < bp->rx_nr_rings; i++) {
5747                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5748                         struct bnxt_ring_struct *ring =
5749                                                 &rxr->rx_agg_ring_struct;
5750                         u32 grp_idx = ring->grp_idx;
5751                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5752
5753                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5754                         if (rc)
5755                                 goto err_out;
5756
5757                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5758                                     ring->fw_ring_id);
5759                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5760                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5761                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5762                 }
5763         }
5764 err_out:
5765         return rc;
5766 }
5767
5768 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5769                                    struct bnxt_ring_struct *ring,
5770                                    u32 ring_type, int cmpl_ring_id)
5771 {
5772         int rc;
5773         struct hwrm_ring_free_input req = {0};
5774         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5775         u16 error_code;
5776
5777         if (BNXT_NO_FW_ACCESS(bp))
5778                 return 0;
5779
5780         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5781         req.ring_type = ring_type;
5782         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5783
5784         mutex_lock(&bp->hwrm_cmd_lock);
5785         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5786         error_code = le16_to_cpu(resp->error_code);
5787         mutex_unlock(&bp->hwrm_cmd_lock);
5788
5789         if (rc || error_code) {
5790                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5791                            ring_type, rc, error_code);
5792                 return -EIO;
5793         }
5794         return 0;
5795 }
5796
5797 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5798 {
5799         u32 type;
5800         int i;
5801
5802         if (!bp->bnapi)
5803                 return;
5804
5805         for (i = 0; i < bp->tx_nr_rings; i++) {
5806                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5807                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5808
5809                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5810                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5811
5812                         hwrm_ring_free_send_msg(bp, ring,
5813                                                 RING_FREE_REQ_RING_TYPE_TX,
5814                                                 close_path ? cmpl_ring_id :
5815                                                 INVALID_HW_RING_ID);
5816                         ring->fw_ring_id = INVALID_HW_RING_ID;
5817                 }
5818         }
5819
5820         for (i = 0; i < bp->rx_nr_rings; i++) {
5821                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5822                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5823                 u32 grp_idx = rxr->bnapi->index;
5824
5825                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5826                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5827
5828                         hwrm_ring_free_send_msg(bp, ring,
5829                                                 RING_FREE_REQ_RING_TYPE_RX,
5830                                                 close_path ? cmpl_ring_id :
5831                                                 INVALID_HW_RING_ID);
5832                         ring->fw_ring_id = INVALID_HW_RING_ID;
5833                         bp->grp_info[grp_idx].rx_fw_ring_id =
5834                                 INVALID_HW_RING_ID;
5835                 }
5836         }
5837
5838         if (bp->flags & BNXT_FLAG_CHIP_P5)
5839                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5840         else
5841                 type = RING_FREE_REQ_RING_TYPE_RX;
5842         for (i = 0; i < bp->rx_nr_rings; i++) {
5843                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5844                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5845                 u32 grp_idx = rxr->bnapi->index;
5846
5847                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5848                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5849
5850                         hwrm_ring_free_send_msg(bp, ring, type,
5851                                                 close_path ? cmpl_ring_id :
5852                                                 INVALID_HW_RING_ID);
5853                         ring->fw_ring_id = INVALID_HW_RING_ID;
5854                         bp->grp_info[grp_idx].agg_fw_ring_id =
5855                                 INVALID_HW_RING_ID;
5856                 }
5857         }
5858
5859         /* The completion rings are about to be freed.  After that the
5860          * IRQ doorbell will not work anymore.  So we need to disable
5861          * IRQ here.
5862          */
5863         bnxt_disable_int_sync(bp);
5864
5865         if (bp->flags & BNXT_FLAG_CHIP_P5)
5866                 type = RING_FREE_REQ_RING_TYPE_NQ;
5867         else
5868                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5869         for (i = 0; i < bp->cp_nr_rings; i++) {
5870                 struct bnxt_napi *bnapi = bp->bnapi[i];
5871                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5872                 struct bnxt_ring_struct *ring;
5873                 int j;
5874
5875                 for (j = 0; j < 2; j++) {
5876                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5877
5878                         if (cpr2) {
5879                                 ring = &cpr2->cp_ring_struct;
5880                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5881                                         continue;
5882                                 hwrm_ring_free_send_msg(bp, ring,
5883                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5884                                         INVALID_HW_RING_ID);
5885                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5886                         }
5887                 }
5888                 ring = &cpr->cp_ring_struct;
5889                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5890                         hwrm_ring_free_send_msg(bp, ring, type,
5891                                                 INVALID_HW_RING_ID);
5892                         ring->fw_ring_id = INVALID_HW_RING_ID;
5893                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5894                 }
5895         }
5896 }
5897
5898 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5899                            bool shared);
5900
5901 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5902 {
5903         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5904         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5905         struct hwrm_func_qcfg_input req = {0};
5906         int rc;
5907
5908         if (bp->hwrm_spec_code < 0x10601)
5909                 return 0;
5910
5911         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5912         req.fid = cpu_to_le16(0xffff);
5913         mutex_lock(&bp->hwrm_cmd_lock);
5914         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5915         if (rc) {
5916                 mutex_unlock(&bp->hwrm_cmd_lock);
5917                 return rc;
5918         }
5919
5920         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5921         if (BNXT_NEW_RM(bp)) {
5922                 u16 cp, stats;
5923
5924                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5925                 hw_resc->resv_hw_ring_grps =
5926                         le32_to_cpu(resp->alloc_hw_ring_grps);
5927                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5928                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5929                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5930                 hw_resc->resv_irqs = cp;
5931                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5932                         int rx = hw_resc->resv_rx_rings;
5933                         int tx = hw_resc->resv_tx_rings;
5934
5935                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5936                                 rx >>= 1;
5937                         if (cp < (rx + tx)) {
5938                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5939                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5940                                         rx <<= 1;
5941                                 hw_resc->resv_rx_rings = rx;
5942                                 hw_resc->resv_tx_rings = tx;
5943                         }
5944                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5945                         hw_resc->resv_hw_ring_grps = rx;
5946                 }
5947                 hw_resc->resv_cp_rings = cp;
5948                 hw_resc->resv_stat_ctxs = stats;
5949         }
5950         mutex_unlock(&bp->hwrm_cmd_lock);
5951         return 0;
5952 }
5953
5954 /* Caller must hold bp->hwrm_cmd_lock */
5955 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5956 {
5957         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5958         struct hwrm_func_qcfg_input req = {0};
5959         int rc;
5960
5961         if (bp->hwrm_spec_code < 0x10601)
5962                 return 0;
5963
5964         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5965         req.fid = cpu_to_le16(fid);
5966         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5967         if (!rc)
5968                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5969
5970         return rc;
5971 }
5972
5973 static bool bnxt_rfs_supported(struct bnxt *bp);
5974
5975 static void
5976 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5977                              int tx_rings, int rx_rings, int ring_grps,
5978                              int cp_rings, int stats, int vnics)
5979 {
5980         u32 enables = 0;
5981
5982         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5983         req->fid = cpu_to_le16(0xffff);
5984         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5985         req->num_tx_rings = cpu_to_le16(tx_rings);
5986         if (BNXT_NEW_RM(bp)) {
5987                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5988                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5989                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5990                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5991                         enables |= tx_rings + ring_grps ?
5992                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5993                         enables |= rx_rings ?
5994                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5995                 } else {
5996                         enables |= cp_rings ?
5997                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5998                         enables |= ring_grps ?
5999                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6000                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6001                 }
6002                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6003
6004                 req->num_rx_rings = cpu_to_le16(rx_rings);
6005                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6006                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6007                         req->num_msix = cpu_to_le16(cp_rings);
6008                         req->num_rsscos_ctxs =
6009                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6010                 } else {
6011                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6012                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6013                         req->num_rsscos_ctxs = cpu_to_le16(1);
6014                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6015                             bnxt_rfs_supported(bp))
6016                                 req->num_rsscos_ctxs =
6017                                         cpu_to_le16(ring_grps + 1);
6018                 }
6019                 req->num_stat_ctxs = cpu_to_le16(stats);
6020                 req->num_vnics = cpu_to_le16(vnics);
6021         }
6022         req->enables = cpu_to_le32(enables);
6023 }
6024
6025 static void
6026 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
6027                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
6028                              int rx_rings, int ring_grps, int cp_rings,
6029                              int stats, int vnics)
6030 {
6031         u32 enables = 0;
6032
6033         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
6034         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6035         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6036                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6037         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6038         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6039                 enables |= tx_rings + ring_grps ?
6040                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6041         } else {
6042                 enables |= cp_rings ?
6043                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6044                 enables |= ring_grps ?
6045                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6046         }
6047         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6048         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6049
6050         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6051         req->num_tx_rings = cpu_to_le16(tx_rings);
6052         req->num_rx_rings = cpu_to_le16(rx_rings);
6053         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6054                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6055                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6056         } else {
6057                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6058                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6059                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6060         }
6061         req->num_stat_ctxs = cpu_to_le16(stats);
6062         req->num_vnics = cpu_to_le16(vnics);
6063
6064         req->enables = cpu_to_le32(enables);
6065 }
6066
6067 static int
6068 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6069                            int ring_grps, int cp_rings, int stats, int vnics)
6070 {
6071         struct hwrm_func_cfg_input req = {0};
6072         int rc;
6073
6074         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6075                                      cp_rings, stats, vnics);
6076         if (!req.enables)
6077                 return 0;
6078
6079         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6080         if (rc)
6081                 return rc;
6082
6083         if (bp->hwrm_spec_code < 0x10601)
6084                 bp->hw_resc.resv_tx_rings = tx_rings;
6085
6086         return bnxt_hwrm_get_rings(bp);
6087 }
6088
6089 static int
6090 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6091                            int ring_grps, int cp_rings, int stats, int vnics)
6092 {
6093         struct hwrm_func_vf_cfg_input req = {0};
6094         int rc;
6095
6096         if (!BNXT_NEW_RM(bp)) {
6097                 bp->hw_resc.resv_tx_rings = tx_rings;
6098                 return 0;
6099         }
6100
6101         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6102                                      cp_rings, stats, vnics);
6103         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6104         if (rc)
6105                 return rc;
6106
6107         return bnxt_hwrm_get_rings(bp);
6108 }
6109
6110 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6111                                    int cp, int stat, int vnic)
6112 {
6113         if (BNXT_PF(bp))
6114                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6115                                                   vnic);
6116         else
6117                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6118                                                   vnic);
6119 }
6120
6121 int bnxt_nq_rings_in_use(struct bnxt *bp)
6122 {
6123         int cp = bp->cp_nr_rings;
6124         int ulp_msix, ulp_base;
6125
6126         ulp_msix = bnxt_get_ulp_msix_num(bp);
6127         if (ulp_msix) {
6128                 ulp_base = bnxt_get_ulp_msix_base(bp);
6129                 cp += ulp_msix;
6130                 if ((ulp_base + ulp_msix) > cp)
6131                         cp = ulp_base + ulp_msix;
6132         }
6133         return cp;
6134 }
6135
6136 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6137 {
6138         int cp;
6139
6140         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6141                 return bnxt_nq_rings_in_use(bp);
6142
6143         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6144         return cp;
6145 }
6146
6147 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6148 {
6149         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6150         int cp = bp->cp_nr_rings;
6151
6152         if (!ulp_stat)
6153                 return cp;
6154
6155         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6156                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6157
6158         return cp + ulp_stat;
6159 }
6160
6161 /* Check if a default RSS map needs to be setup.  This function is only
6162  * used on older firmware that does not require reserving RX rings.
6163  */
6164 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6165 {
6166         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6167
6168         /* The RSS map is valid for RX rings set to resv_rx_rings */
6169         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6170                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6171                 if (!netif_is_rxfh_configured(bp->dev))
6172                         bnxt_set_dflt_rss_indir_tbl(bp);
6173         }
6174 }
6175
6176 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6177 {
6178         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6179         int cp = bnxt_cp_rings_in_use(bp);
6180         int nq = bnxt_nq_rings_in_use(bp);
6181         int rx = bp->rx_nr_rings, stat;
6182         int vnic = 1, grp = rx;
6183
6184         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6185             bp->hwrm_spec_code >= 0x10601)
6186                 return true;
6187
6188         /* Old firmware does not need RX ring reservations but we still
6189          * need to setup a default RSS map when needed.  With new firmware
6190          * we go through RX ring reservations first and then set up the
6191          * RSS map for the successfully reserved RX rings when needed.
6192          */
6193         if (!BNXT_NEW_RM(bp)) {
6194                 bnxt_check_rss_tbl_no_rmgr(bp);
6195                 return false;
6196         }
6197         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6198                 vnic = rx + 1;
6199         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6200                 rx <<= 1;
6201         stat = bnxt_get_func_stat_ctxs(bp);
6202         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6203             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6204             (hw_resc->resv_hw_ring_grps != grp &&
6205              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6206                 return true;
6207         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6208             hw_resc->resv_irqs != nq)
6209                 return true;
6210         return false;
6211 }
6212
6213 static int __bnxt_reserve_rings(struct bnxt *bp)
6214 {
6215         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6216         int cp = bnxt_nq_rings_in_use(bp);
6217         int tx = bp->tx_nr_rings;
6218         int rx = bp->rx_nr_rings;
6219         int grp, rx_rings, rc;
6220         int vnic = 1, stat;
6221         bool sh = false;
6222
6223         if (!bnxt_need_reserve_rings(bp))
6224                 return 0;
6225
6226         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6227                 sh = true;
6228         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6229                 vnic = rx + 1;
6230         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6231                 rx <<= 1;
6232         grp = bp->rx_nr_rings;
6233         stat = bnxt_get_func_stat_ctxs(bp);
6234
6235         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6236         if (rc)
6237                 return rc;
6238
6239         tx = hw_resc->resv_tx_rings;
6240         if (BNXT_NEW_RM(bp)) {
6241                 rx = hw_resc->resv_rx_rings;
6242                 cp = hw_resc->resv_irqs;
6243                 grp = hw_resc->resv_hw_ring_grps;
6244                 vnic = hw_resc->resv_vnics;
6245                 stat = hw_resc->resv_stat_ctxs;
6246         }
6247
6248         rx_rings = rx;
6249         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6250                 if (rx >= 2) {
6251                         rx_rings = rx >> 1;
6252                 } else {
6253                         if (netif_running(bp->dev))
6254                                 return -ENOMEM;
6255
6256                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6257                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6258                         bp->dev->hw_features &= ~NETIF_F_LRO;
6259                         bp->dev->features &= ~NETIF_F_LRO;
6260                         bnxt_set_ring_params(bp);
6261                 }
6262         }
6263         rx_rings = min_t(int, rx_rings, grp);
6264         cp = min_t(int, cp, bp->cp_nr_rings);
6265         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6266                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6267         cp = min_t(int, cp, stat);
6268         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6269         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6270                 rx = rx_rings << 1;
6271         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6272         bp->tx_nr_rings = tx;
6273
6274         /* If we cannot reserve all the RX rings, reset the RSS map only
6275          * if absolutely necessary
6276          */
6277         if (rx_rings != bp->rx_nr_rings) {
6278                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6279                             rx_rings, bp->rx_nr_rings);
6280                 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6281                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6282                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6283                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6284                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6285                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6286                 }
6287         }
6288         bp->rx_nr_rings = rx_rings;
6289         bp->cp_nr_rings = cp;
6290
6291         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6292                 return -ENOMEM;
6293
6294         if (!netif_is_rxfh_configured(bp->dev))
6295                 bnxt_set_dflt_rss_indir_tbl(bp);
6296
6297         return rc;
6298 }
6299
6300 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6301                                     int ring_grps, int cp_rings, int stats,
6302                                     int vnics)
6303 {
6304         struct hwrm_func_vf_cfg_input req = {0};
6305         u32 flags;
6306
6307         if (!BNXT_NEW_RM(bp))
6308                 return 0;
6309
6310         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6311                                      cp_rings, stats, vnics);
6312         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6313                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6314                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6315                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6316                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6317                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6318         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6319                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6320
6321         req.flags = cpu_to_le32(flags);
6322         return hwrm_send_message_silent(bp, &req, sizeof(req),
6323                                         HWRM_CMD_TIMEOUT);
6324 }
6325
6326 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6327                                     int ring_grps, int cp_rings, int stats,
6328                                     int vnics)
6329 {
6330         struct hwrm_func_cfg_input req = {0};
6331         u32 flags;
6332
6333         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6334                                      cp_rings, stats, vnics);
6335         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6336         if (BNXT_NEW_RM(bp)) {
6337                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6338                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6339                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6340                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6341                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6342                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6343                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6344                 else
6345                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6346         }
6347
6348         req.flags = cpu_to_le32(flags);
6349         return hwrm_send_message_silent(bp, &req, sizeof(req),
6350                                         HWRM_CMD_TIMEOUT);
6351 }
6352
6353 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6354                                  int ring_grps, int cp_rings, int stats,
6355                                  int vnics)
6356 {
6357         if (bp->hwrm_spec_code < 0x10801)
6358                 return 0;
6359
6360         if (BNXT_PF(bp))
6361                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6362                                                 ring_grps, cp_rings, stats,
6363                                                 vnics);
6364
6365         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6366                                         cp_rings, stats, vnics);
6367 }
6368
6369 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6370 {
6371         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6372         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6373         struct hwrm_ring_aggint_qcaps_input req = {0};
6374         int rc;
6375
6376         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6377         coal_cap->num_cmpl_dma_aggr_max = 63;
6378         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6379         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6380         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6381         coal_cap->int_lat_tmr_min_max = 65535;
6382         coal_cap->int_lat_tmr_max_max = 65535;
6383         coal_cap->num_cmpl_aggr_int_max = 65535;
6384         coal_cap->timer_units = 80;
6385
6386         if (bp->hwrm_spec_code < 0x10902)
6387                 return;
6388
6389         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6390         mutex_lock(&bp->hwrm_cmd_lock);
6391         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6392         if (!rc) {
6393                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6394                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6395                 coal_cap->num_cmpl_dma_aggr_max =
6396                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6397                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6398                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6399                 coal_cap->cmpl_aggr_dma_tmr_max =
6400                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6401                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6402                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6403                 coal_cap->int_lat_tmr_min_max =
6404                         le16_to_cpu(resp->int_lat_tmr_min_max);
6405                 coal_cap->int_lat_tmr_max_max =
6406                         le16_to_cpu(resp->int_lat_tmr_max_max);
6407                 coal_cap->num_cmpl_aggr_int_max =
6408                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6409                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6410         }
6411         mutex_unlock(&bp->hwrm_cmd_lock);
6412 }
6413
6414 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6415 {
6416         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6417
6418         return usec * 1000 / coal_cap->timer_units;
6419 }
6420
6421 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6422         struct bnxt_coal *hw_coal,
6423         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6424 {
6425         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6426         u32 cmpl_params = coal_cap->cmpl_params;
6427         u16 val, tmr, max, flags = 0;
6428
6429         max = hw_coal->bufs_per_record * 128;
6430         if (hw_coal->budget)
6431                 max = hw_coal->bufs_per_record * hw_coal->budget;
6432         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6433
6434         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6435         req->num_cmpl_aggr_int = cpu_to_le16(val);
6436
6437         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6438         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6439
6440         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6441                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6442         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6443
6444         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6445         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6446         req->int_lat_tmr_max = cpu_to_le16(tmr);
6447
6448         /* min timer set to 1/2 of interrupt timer */
6449         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6450                 val = tmr / 2;
6451                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6452                 req->int_lat_tmr_min = cpu_to_le16(val);
6453                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6454         }
6455
6456         /* buf timer set to 1/4 of interrupt timer */
6457         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6458         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6459
6460         if (cmpl_params &
6461             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6462                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6463                 val = clamp_t(u16, tmr, 1,
6464                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6465                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6466                 req->enables |=
6467                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6468         }
6469
6470         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6471                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6472         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6473             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6474                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6475         req->flags = cpu_to_le16(flags);
6476         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6477 }
6478
6479 /* Caller holds bp->hwrm_cmd_lock */
6480 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6481                                    struct bnxt_coal *hw_coal)
6482 {
6483         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6484         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6485         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6486         u32 nq_params = coal_cap->nq_params;
6487         u16 tmr;
6488
6489         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6490                 return 0;
6491
6492         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6493                                -1, -1);
6494         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6495         req.flags =
6496                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6497
6498         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6499         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6500         req.int_lat_tmr_min = cpu_to_le16(tmr);
6501         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6502         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6503 }
6504
6505 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6506 {
6507         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6508         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6509         struct bnxt_coal coal;
6510
6511         /* Tick values in micro seconds.
6512          * 1 coal_buf x bufs_per_record = 1 completion record.
6513          */
6514         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6515
6516         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6517         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6518
6519         if (!bnapi->rx_ring)
6520                 return -ENODEV;
6521
6522         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6523                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6524
6525         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6526
6527         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6528
6529         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6530                                  HWRM_CMD_TIMEOUT);
6531 }
6532
6533 int bnxt_hwrm_set_coal(struct bnxt *bp)
6534 {
6535         int i, rc = 0;
6536         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6537                                                            req_tx = {0}, *req;
6538
6539         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6540                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6541         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6542                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6543
6544         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6545         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6546
6547         mutex_lock(&bp->hwrm_cmd_lock);
6548         for (i = 0; i < bp->cp_nr_rings; i++) {
6549                 struct bnxt_napi *bnapi = bp->bnapi[i];
6550                 struct bnxt_coal *hw_coal;
6551                 u16 ring_id;
6552
6553                 req = &req_rx;
6554                 if (!bnapi->rx_ring) {
6555                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6556                         req = &req_tx;
6557                 } else {
6558                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6559                 }
6560                 req->ring_id = cpu_to_le16(ring_id);
6561
6562                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6563                                         HWRM_CMD_TIMEOUT);
6564                 if (rc)
6565                         break;
6566
6567                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6568                         continue;
6569
6570                 if (bnapi->rx_ring && bnapi->tx_ring) {
6571                         req = &req_tx;
6572                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6573                         req->ring_id = cpu_to_le16(ring_id);
6574                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6575                                                 HWRM_CMD_TIMEOUT);
6576                         if (rc)
6577                                 break;
6578                 }
6579                 if (bnapi->rx_ring)
6580                         hw_coal = &bp->rx_coal;
6581                 else
6582                         hw_coal = &bp->tx_coal;
6583                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6584         }
6585         mutex_unlock(&bp->hwrm_cmd_lock);
6586         return rc;
6587 }
6588
6589 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6590 {
6591         struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6592         struct hwrm_stat_ctx_free_input req = {0};
6593         int i;
6594
6595         if (!bp->bnapi)
6596                 return;
6597
6598         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6599                 return;
6600
6601         bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6602         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6603
6604         mutex_lock(&bp->hwrm_cmd_lock);
6605         for (i = 0; i < bp->cp_nr_rings; i++) {
6606                 struct bnxt_napi *bnapi = bp->bnapi[i];
6607                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6608
6609                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6610                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6611                         if (BNXT_FW_MAJ(bp) <= 20) {
6612                                 req0.stat_ctx_id = req.stat_ctx_id;
6613                                 _hwrm_send_message(bp, &req0, sizeof(req0),
6614                                                    HWRM_CMD_TIMEOUT);
6615                         }
6616                         _hwrm_send_message(bp, &req, sizeof(req),
6617                                            HWRM_CMD_TIMEOUT);
6618
6619                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6620                 }
6621         }
6622         mutex_unlock(&bp->hwrm_cmd_lock);
6623 }
6624
6625 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6626 {
6627         int rc = 0, i;
6628         struct hwrm_stat_ctx_alloc_input req = {0};
6629         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6630
6631         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6632                 return 0;
6633
6634         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6635
6636         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6637         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6638
6639         mutex_lock(&bp->hwrm_cmd_lock);
6640         for (i = 0; i < bp->cp_nr_rings; i++) {
6641                 struct bnxt_napi *bnapi = bp->bnapi[i];
6642                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6643
6644                 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6645
6646                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6647                                         HWRM_CMD_TIMEOUT);
6648                 if (rc)
6649                         break;
6650
6651                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6652
6653                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6654         }
6655         mutex_unlock(&bp->hwrm_cmd_lock);
6656         return rc;
6657 }
6658
6659 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6660 {
6661         struct hwrm_func_qcfg_input req = {0};
6662         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6663         u32 min_db_offset = 0;
6664         u16 flags;
6665         int rc;
6666
6667         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6668         req.fid = cpu_to_le16(0xffff);
6669         mutex_lock(&bp->hwrm_cmd_lock);
6670         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6671         if (rc)
6672                 goto func_qcfg_exit;
6673
6674 #ifdef CONFIG_BNXT_SRIOV
6675         if (BNXT_VF(bp)) {
6676                 struct bnxt_vf_info *vf = &bp->vf;
6677
6678                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6679         } else {
6680                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6681         }
6682 #endif
6683         flags = le16_to_cpu(resp->flags);
6684         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6685                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6686                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6687                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6688                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6689         }
6690         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6691                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6692         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6693                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6694
6695         switch (resp->port_partition_type) {
6696         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6697         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6698         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6699                 bp->port_partition_type = resp->port_partition_type;
6700                 break;
6701         }
6702         if (bp->hwrm_spec_code < 0x10707 ||
6703             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6704                 bp->br_mode = BRIDGE_MODE_VEB;
6705         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6706                 bp->br_mode = BRIDGE_MODE_VEPA;
6707         else
6708                 bp->br_mode = BRIDGE_MODE_UNDEF;
6709
6710         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6711         if (!bp->max_mtu)
6712                 bp->max_mtu = BNXT_MAX_MTU;
6713
6714         if (bp->db_size)
6715                 goto func_qcfg_exit;
6716
6717         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6718                 if (BNXT_PF(bp))
6719                         min_db_offset = DB_PF_OFFSET_P5;
6720                 else
6721                         min_db_offset = DB_VF_OFFSET_P5;
6722         }
6723         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6724                                  1024);
6725         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6726             bp->db_size <= min_db_offset)
6727                 bp->db_size = pci_resource_len(bp->pdev, 2);
6728
6729 func_qcfg_exit:
6730         mutex_unlock(&bp->hwrm_cmd_lock);
6731         return rc;
6732 }
6733
6734 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6735 {
6736         struct hwrm_func_backing_store_qcaps_input req = {0};
6737         struct hwrm_func_backing_store_qcaps_output *resp =
6738                 bp->hwrm_cmd_resp_addr;
6739         int rc;
6740
6741         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6742                 return 0;
6743
6744         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6745         mutex_lock(&bp->hwrm_cmd_lock);
6746         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6747         if (!rc) {
6748                 struct bnxt_ctx_pg_info *ctx_pg;
6749                 struct bnxt_ctx_mem_info *ctx;
6750                 int i, tqm_rings;
6751
6752                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6753                 if (!ctx) {
6754                         rc = -ENOMEM;
6755                         goto ctx_err;
6756                 }
6757                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6758                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6759                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6760                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6761                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6762                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6763                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6764                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6765                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6766                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6767                 ctx->vnic_max_vnic_entries =
6768                         le16_to_cpu(resp->vnic_max_vnic_entries);
6769                 ctx->vnic_max_ring_table_entries =
6770                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6771                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6772                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6773                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6774                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6775                 ctx->tqm_min_entries_per_ring =
6776                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6777                 ctx->tqm_max_entries_per_ring =
6778                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6779                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6780                 if (!ctx->tqm_entries_multiple)
6781                         ctx->tqm_entries_multiple = 1;
6782                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6783                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6784                 ctx->mrav_num_entries_units =
6785                         le16_to_cpu(resp->mrav_num_entries_units);
6786                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6787                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6788                 ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6789                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6790                 if (!ctx->tqm_fp_rings_count)
6791                         ctx->tqm_fp_rings_count = bp->max_q;
6792
6793                 tqm_rings = ctx->tqm_fp_rings_count + 1;
6794                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6795                 if (!ctx_pg) {
6796                         kfree(ctx);
6797                         rc = -ENOMEM;
6798                         goto ctx_err;
6799                 }
6800                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6801                         ctx->tqm_mem[i] = ctx_pg;
6802                 bp->ctx = ctx;
6803         } else {
6804                 rc = 0;
6805         }
6806 ctx_err:
6807         mutex_unlock(&bp->hwrm_cmd_lock);
6808         return rc;
6809 }
6810
6811 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6812                                   __le64 *pg_dir)
6813 {
6814         u8 pg_size = 0;
6815
6816         if (BNXT_PAGE_SHIFT == 13)
6817                 pg_size = 1 << 4;
6818         else if (BNXT_PAGE_SIZE == 16)
6819                 pg_size = 2 << 4;
6820
6821         *pg_attr = pg_size;
6822         if (rmem->depth >= 1) {
6823                 if (rmem->depth == 2)
6824                         *pg_attr |= 2;
6825                 else
6826                         *pg_attr |= 1;
6827                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6828         } else {
6829                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6830         }
6831 }
6832
6833 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6834         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6835          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6836          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6837          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6838          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6839
6840 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6841 {
6842         struct hwrm_func_backing_store_cfg_input req = {0};
6843         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6844         struct bnxt_ctx_pg_info *ctx_pg;
6845         __le32 *num_entries;
6846         __le64 *pg_dir;
6847         u32 flags = 0;
6848         u8 *pg_attr;
6849         u32 ena;
6850         int i;
6851
6852         if (!ctx)
6853                 return 0;
6854
6855         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6856         req.enables = cpu_to_le32(enables);
6857
6858         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6859                 ctx_pg = &ctx->qp_mem;
6860                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6861                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6862                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6863                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6864                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6865                                       &req.qpc_pg_size_qpc_lvl,
6866                                       &req.qpc_page_dir);
6867         }
6868         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6869                 ctx_pg = &ctx->srq_mem;
6870                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6871                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6872                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6873                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6874                                       &req.srq_pg_size_srq_lvl,
6875                                       &req.srq_page_dir);
6876         }
6877         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6878                 ctx_pg = &ctx->cq_mem;
6879                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6880                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6881                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6882                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6883                                       &req.cq_page_dir);
6884         }
6885         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6886                 ctx_pg = &ctx->vnic_mem;
6887                 req.vnic_num_vnic_entries =
6888                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6889                 req.vnic_num_ring_table_entries =
6890                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6891                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6892                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6893                                       &req.vnic_pg_size_vnic_lvl,
6894                                       &req.vnic_page_dir);
6895         }
6896         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6897                 ctx_pg = &ctx->stat_mem;
6898                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6899                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6900                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6901                                       &req.stat_pg_size_stat_lvl,
6902                                       &req.stat_page_dir);
6903         }
6904         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6905                 ctx_pg = &ctx->mrav_mem;
6906                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6907                 if (ctx->mrav_num_entries_units)
6908                         flags |=
6909                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6910                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6911                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6912                                       &req.mrav_pg_size_mrav_lvl,
6913                                       &req.mrav_page_dir);
6914         }
6915         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6916                 ctx_pg = &ctx->tim_mem;
6917                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6918                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6919                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6920                                       &req.tim_pg_size_tim_lvl,
6921                                       &req.tim_page_dir);
6922         }
6923         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6924              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6925              pg_dir = &req.tqm_sp_page_dir,
6926              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6927              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6928                 if (!(enables & ena))
6929                         continue;
6930
6931                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6932                 ctx_pg = ctx->tqm_mem[i];
6933                 *num_entries = cpu_to_le32(ctx_pg->entries);
6934                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6935         }
6936         req.flags = cpu_to_le32(flags);
6937         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6938 }
6939
6940 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6941                                   struct bnxt_ctx_pg_info *ctx_pg)
6942 {
6943         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6944
6945         rmem->page_size = BNXT_PAGE_SIZE;
6946         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6947         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6948         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6949         if (rmem->depth >= 1)
6950                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6951         return bnxt_alloc_ring(bp, rmem);
6952 }
6953
6954 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6955                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6956                                   u8 depth, bool use_init_val)
6957 {
6958         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6959         int rc;
6960
6961         if (!mem_size)
6962                 return -EINVAL;
6963
6964         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6965         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6966                 ctx_pg->nr_pages = 0;
6967                 return -EINVAL;
6968         }
6969         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6970                 int nr_tbls, i;
6971
6972                 rmem->depth = 2;
6973                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6974                                              GFP_KERNEL);
6975                 if (!ctx_pg->ctx_pg_tbl)
6976                         return -ENOMEM;
6977                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6978                 rmem->nr_pages = nr_tbls;
6979                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6980                 if (rc)
6981                         return rc;
6982                 for (i = 0; i < nr_tbls; i++) {
6983                         struct bnxt_ctx_pg_info *pg_tbl;
6984
6985                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6986                         if (!pg_tbl)
6987                                 return -ENOMEM;
6988                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6989                         rmem = &pg_tbl->ring_mem;
6990                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6991                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6992                         rmem->depth = 1;
6993                         rmem->nr_pages = MAX_CTX_PAGES;
6994                         if (use_init_val)
6995                                 rmem->init_val = bp->ctx->ctx_kind_initializer;
6996                         if (i == (nr_tbls - 1)) {
6997                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6998
6999                                 if (rem)
7000                                         rmem->nr_pages = rem;
7001                         }
7002                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7003                         if (rc)
7004                                 break;
7005                 }
7006         } else {
7007                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7008                 if (rmem->nr_pages > 1 || depth)
7009                         rmem->depth = 1;
7010                 if (use_init_val)
7011                         rmem->init_val = bp->ctx->ctx_kind_initializer;
7012                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7013         }
7014         return rc;
7015 }
7016
7017 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7018                                   struct bnxt_ctx_pg_info *ctx_pg)
7019 {
7020         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7021
7022         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7023             ctx_pg->ctx_pg_tbl) {
7024                 int i, nr_tbls = rmem->nr_pages;
7025
7026                 for (i = 0; i < nr_tbls; i++) {
7027                         struct bnxt_ctx_pg_info *pg_tbl;
7028                         struct bnxt_ring_mem_info *rmem2;
7029
7030                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7031                         if (!pg_tbl)
7032                                 continue;
7033                         rmem2 = &pg_tbl->ring_mem;
7034                         bnxt_free_ring(bp, rmem2);
7035                         ctx_pg->ctx_pg_arr[i] = NULL;
7036                         kfree(pg_tbl);
7037                         ctx_pg->ctx_pg_tbl[i] = NULL;
7038                 }
7039                 kfree(ctx_pg->ctx_pg_tbl);
7040                 ctx_pg->ctx_pg_tbl = NULL;
7041         }
7042         bnxt_free_ring(bp, rmem);
7043         ctx_pg->nr_pages = 0;
7044 }
7045
7046 static void bnxt_free_ctx_mem(struct bnxt *bp)
7047 {
7048         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7049         int i;
7050
7051         if (!ctx)
7052                 return;
7053
7054         if (ctx->tqm_mem[0]) {
7055                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7056                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7057                 kfree(ctx->tqm_mem[0]);
7058                 ctx->tqm_mem[0] = NULL;
7059         }
7060
7061         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7062         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7063         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7064         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7065         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7066         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7067         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7068         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7069 }
7070
7071 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7072 {
7073         struct bnxt_ctx_pg_info *ctx_pg;
7074         struct bnxt_ctx_mem_info *ctx;
7075         u32 mem_size, ena, entries;
7076         u32 entries_sp, min;
7077         u32 num_mr, num_ah;
7078         u32 extra_srqs = 0;
7079         u32 extra_qps = 0;
7080         u8 pg_lvl = 1;
7081         int i, rc;
7082
7083         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7084         if (rc) {
7085                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7086                            rc);
7087                 return rc;
7088         }
7089         ctx = bp->ctx;
7090         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7091                 return 0;
7092
7093         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7094                 pg_lvl = 2;
7095                 extra_qps = 65536;
7096                 extra_srqs = 8192;
7097         }
7098
7099         ctx_pg = &ctx->qp_mem;
7100         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7101                           extra_qps;
7102         mem_size = ctx->qp_entry_size * ctx_pg->entries;
7103         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7104         if (rc)
7105                 return rc;
7106
7107         ctx_pg = &ctx->srq_mem;
7108         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7109         mem_size = ctx->srq_entry_size * ctx_pg->entries;
7110         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7111         if (rc)
7112                 return rc;
7113
7114         ctx_pg = &ctx->cq_mem;
7115         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7116         mem_size = ctx->cq_entry_size * ctx_pg->entries;
7117         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7118         if (rc)
7119                 return rc;
7120
7121         ctx_pg = &ctx->vnic_mem;
7122         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7123                           ctx->vnic_max_ring_table_entries;
7124         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7125         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7126         if (rc)
7127                 return rc;
7128
7129         ctx_pg = &ctx->stat_mem;
7130         ctx_pg->entries = ctx->stat_max_entries;
7131         mem_size = ctx->stat_entry_size * ctx_pg->entries;
7132         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7133         if (rc)
7134                 return rc;
7135
7136         ena = 0;
7137         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7138                 goto skip_rdma;
7139
7140         ctx_pg = &ctx->mrav_mem;
7141         /* 128K extra is needed to accommodate static AH context
7142          * allocation by f/w.
7143          */
7144         num_mr = 1024 * 256;
7145         num_ah = 1024 * 128;
7146         ctx_pg->entries = num_mr + num_ah;
7147         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7148         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7149         if (rc)
7150                 return rc;
7151         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7152         if (ctx->mrav_num_entries_units)
7153                 ctx_pg->entries =
7154                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7155                          (num_ah / ctx->mrav_num_entries_units);
7156
7157         ctx_pg = &ctx->tim_mem;
7158         ctx_pg->entries = ctx->qp_mem.entries;
7159         mem_size = ctx->tim_entry_size * ctx_pg->entries;
7160         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7161         if (rc)
7162                 return rc;
7163         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7164
7165 skip_rdma:
7166         min = ctx->tqm_min_entries_per_ring;
7167         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7168                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7169         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7170         entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7171         entries = roundup(entries, ctx->tqm_entries_multiple);
7172         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7173         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7174                 ctx_pg = ctx->tqm_mem[i];
7175                 ctx_pg->entries = i ? entries : entries_sp;
7176                 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7177                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7178                 if (rc)
7179                         return rc;
7180                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7181         }
7182         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7183         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7184         if (rc) {
7185                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7186                            rc);
7187                 return rc;
7188         }
7189         ctx->flags |= BNXT_CTX_FLAG_INITED;
7190         return 0;
7191 }
7192
7193 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7194 {
7195         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7196         struct hwrm_func_resource_qcaps_input req = {0};
7197         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7198         int rc;
7199
7200         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7201         req.fid = cpu_to_le16(0xffff);
7202
7203         mutex_lock(&bp->hwrm_cmd_lock);
7204         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7205                                        HWRM_CMD_TIMEOUT);
7206         if (rc)
7207                 goto hwrm_func_resc_qcaps_exit;
7208
7209         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7210         if (!all)
7211                 goto hwrm_func_resc_qcaps_exit;
7212
7213         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7214         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7215         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7216         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7217         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7218         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7219         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7220         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7221         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7222         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7223         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7224         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7225         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7226         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7227         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7228         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7229
7230         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7231                 u16 max_msix = le16_to_cpu(resp->max_msix);
7232
7233                 hw_resc->max_nqs = max_msix;
7234                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7235         }
7236
7237         if (BNXT_PF(bp)) {
7238                 struct bnxt_pf_info *pf = &bp->pf;
7239
7240                 pf->vf_resv_strategy =
7241                         le16_to_cpu(resp->vf_reservation_strategy);
7242                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7243                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7244         }
7245 hwrm_func_resc_qcaps_exit:
7246         mutex_unlock(&bp->hwrm_cmd_lock);
7247         return rc;
7248 }
7249
7250 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7251 {
7252         int rc = 0;
7253         struct hwrm_func_qcaps_input req = {0};
7254         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7255         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7256         u32 flags, flags_ext;
7257
7258         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7259         req.fid = cpu_to_le16(0xffff);
7260
7261         mutex_lock(&bp->hwrm_cmd_lock);
7262         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7263         if (rc)
7264                 goto hwrm_func_qcaps_exit;
7265
7266         flags = le32_to_cpu(resp->flags);
7267         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7268                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7269         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7270                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7271         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7272                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7273         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7274                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7275         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7276                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7277         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7278                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7279         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7280                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7281         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7282                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7283
7284         flags_ext = le32_to_cpu(resp->flags_ext);
7285         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7286                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7287
7288         bp->tx_push_thresh = 0;
7289         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7290             BNXT_FW_MAJ(bp) > 217)
7291                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7292
7293         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7294         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7295         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7296         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7297         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7298         if (!hw_resc->max_hw_ring_grps)
7299                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7300         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7301         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7302         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7303
7304         if (BNXT_PF(bp)) {
7305                 struct bnxt_pf_info *pf = &bp->pf;
7306
7307                 pf->fw_fid = le16_to_cpu(resp->fid);
7308                 pf->port_id = le16_to_cpu(resp->port_id);
7309                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7310                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7311                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7312                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7313                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7314                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7315                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7316                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7317                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7318                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7319                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7320                         bp->flags |= BNXT_FLAG_WOL_CAP;
7321         } else {
7322 #ifdef CONFIG_BNXT_SRIOV
7323                 struct bnxt_vf_info *vf = &bp->vf;
7324
7325                 vf->fw_fid = le16_to_cpu(resp->fid);
7326                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7327 #endif
7328         }
7329
7330 hwrm_func_qcaps_exit:
7331         mutex_unlock(&bp->hwrm_cmd_lock);
7332         return rc;
7333 }
7334
7335 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7336
7337 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7338 {
7339         int rc;
7340
7341         rc = __bnxt_hwrm_func_qcaps(bp);
7342         if (rc)
7343                 return rc;
7344         rc = bnxt_hwrm_queue_qportcfg(bp);
7345         if (rc) {
7346                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7347                 return rc;
7348         }
7349         if (bp->hwrm_spec_code >= 0x10803) {
7350                 rc = bnxt_alloc_ctx_mem(bp);
7351                 if (rc)
7352                         return rc;
7353                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7354                 if (!rc)
7355                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7356         }
7357         return 0;
7358 }
7359
7360 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7361 {
7362         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7363         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7364         int rc = 0;
7365         u32 flags;
7366
7367         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7368                 return 0;
7369
7370         resp = bp->hwrm_cmd_resp_addr;
7371         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7372
7373         mutex_lock(&bp->hwrm_cmd_lock);
7374         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7375         if (rc)
7376                 goto hwrm_cfa_adv_qcaps_exit;
7377
7378         flags = le32_to_cpu(resp->flags);
7379         if (flags &
7380             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7381                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7382
7383 hwrm_cfa_adv_qcaps_exit:
7384         mutex_unlock(&bp->hwrm_cmd_lock);
7385         return rc;
7386 }
7387
7388 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7389 {
7390         if (bp->fw_health)
7391                 return 0;
7392
7393         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7394         if (!bp->fw_health)
7395                 return -ENOMEM;
7396
7397         return 0;
7398 }
7399
7400 static int bnxt_alloc_fw_health(struct bnxt *bp)
7401 {
7402         int rc;
7403
7404         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7405             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7406                 return 0;
7407
7408         rc = __bnxt_alloc_fw_health(bp);
7409         if (rc) {
7410                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7411                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7412                 return rc;
7413         }
7414
7415         return 0;
7416 }
7417
7418 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7419 {
7420         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7421                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7422                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7423 }
7424
7425 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7426 {
7427         void __iomem *hs;
7428         u32 status_loc;
7429         u32 reg_type;
7430         u32 sig;
7431
7432         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7433         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7434
7435         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7436         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7437                 if (bp->fw_health)
7438                         bp->fw_health->status_reliable = false;
7439                 return;
7440         }
7441
7442         if (__bnxt_alloc_fw_health(bp)) {
7443                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7444                 return;
7445         }
7446
7447         status_loc = readl(hs + offsetof(struct hcomm_status, fw_status_loc));
7448         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7449         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7450         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7451                 __bnxt_map_fw_health_reg(bp, status_loc);
7452                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7453                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7454         }
7455
7456         bp->fw_health->status_reliable = true;
7457 }
7458
7459 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7460 {
7461         struct bnxt_fw_health *fw_health = bp->fw_health;
7462         u32 reg_base = 0xffffffff;
7463         int i;
7464
7465         /* Only pre-map the monitoring GRC registers using window 3 */
7466         for (i = 0; i < 4; i++) {
7467                 u32 reg = fw_health->regs[i];
7468
7469                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7470                         continue;
7471                 if (reg_base == 0xffffffff)
7472                         reg_base = reg & BNXT_GRC_BASE_MASK;
7473                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7474                         return -ERANGE;
7475                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7476         }
7477         if (reg_base == 0xffffffff)
7478                 return 0;
7479
7480         __bnxt_map_fw_health_reg(bp, reg_base);
7481         return 0;
7482 }
7483
7484 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7485 {
7486         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7487         struct bnxt_fw_health *fw_health = bp->fw_health;
7488         struct hwrm_error_recovery_qcfg_input req = {0};
7489         int rc, i;
7490
7491         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7492                 return 0;
7493
7494         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7495         mutex_lock(&bp->hwrm_cmd_lock);
7496         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7497         if (rc)
7498                 goto err_recovery_out;
7499         fw_health->flags = le32_to_cpu(resp->flags);
7500         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7501             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7502                 rc = -EINVAL;
7503                 goto err_recovery_out;
7504         }
7505         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7506         fw_health->master_func_wait_dsecs =
7507                 le32_to_cpu(resp->master_func_wait_period);
7508         fw_health->normal_func_wait_dsecs =
7509                 le32_to_cpu(resp->normal_func_wait_period);
7510         fw_health->post_reset_wait_dsecs =
7511                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7512         fw_health->post_reset_max_wait_dsecs =
7513                 le32_to_cpu(resp->max_bailout_time_after_reset);
7514         fw_health->regs[BNXT_FW_HEALTH_REG] =
7515                 le32_to_cpu(resp->fw_health_status_reg);
7516         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7517                 le32_to_cpu(resp->fw_heartbeat_reg);
7518         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7519                 le32_to_cpu(resp->fw_reset_cnt_reg);
7520         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7521                 le32_to_cpu(resp->reset_inprogress_reg);
7522         fw_health->fw_reset_inprog_reg_mask =
7523                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7524         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7525         if (fw_health->fw_reset_seq_cnt >= 16) {
7526                 rc = -EINVAL;
7527                 goto err_recovery_out;
7528         }
7529         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7530                 fw_health->fw_reset_seq_regs[i] =
7531                         le32_to_cpu(resp->reset_reg[i]);
7532                 fw_health->fw_reset_seq_vals[i] =
7533                         le32_to_cpu(resp->reset_reg_val[i]);
7534                 fw_health->fw_reset_seq_delay_msec[i] =
7535                         resp->delay_after_reset[i];
7536         }
7537 err_recovery_out:
7538         mutex_unlock(&bp->hwrm_cmd_lock);
7539         if (!rc)
7540                 rc = bnxt_map_fw_health_regs(bp);
7541         if (rc)
7542                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7543         return rc;
7544 }
7545
7546 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7547 {
7548         struct hwrm_func_reset_input req = {0};
7549
7550         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7551         req.enables = 0;
7552
7553         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7554 }
7555
7556 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
7557 {
7558         struct hwrm_nvm_get_dev_info_output nvm_info;
7559
7560         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
7561                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
7562                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
7563                          nvm_info.nvm_cfg_ver_upd);
7564 }
7565
7566 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7567 {
7568         int rc = 0;
7569         struct hwrm_queue_qportcfg_input req = {0};
7570         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7571         u8 i, j, *qptr;
7572         bool no_rdma;
7573
7574         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7575
7576         mutex_lock(&bp->hwrm_cmd_lock);
7577         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7578         if (rc)
7579                 goto qportcfg_exit;
7580
7581         if (!resp->max_configurable_queues) {
7582                 rc = -EINVAL;
7583                 goto qportcfg_exit;
7584         }
7585         bp->max_tc = resp->max_configurable_queues;
7586         bp->max_lltc = resp->max_configurable_lossless_queues;
7587         if (bp->max_tc > BNXT_MAX_QUEUE)
7588                 bp->max_tc = BNXT_MAX_QUEUE;
7589
7590         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7591         qptr = &resp->queue_id0;
7592         for (i = 0, j = 0; i < bp->max_tc; i++) {
7593                 bp->q_info[j].queue_id = *qptr;
7594                 bp->q_ids[i] = *qptr++;
7595                 bp->q_info[j].queue_profile = *qptr++;
7596                 bp->tc_to_qidx[j] = j;
7597                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7598                     (no_rdma && BNXT_PF(bp)))
7599                         j++;
7600         }
7601         bp->max_q = bp->max_tc;
7602         bp->max_tc = max_t(u8, j, 1);
7603
7604         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7605                 bp->max_tc = 1;
7606
7607         if (bp->max_lltc > bp->max_tc)
7608                 bp->max_lltc = bp->max_tc;
7609
7610 qportcfg_exit:
7611         mutex_unlock(&bp->hwrm_cmd_lock);
7612         return rc;
7613 }
7614
7615 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7616 {
7617         struct hwrm_ver_get_input req = {0};
7618         int rc;
7619
7620         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7621         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7622         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7623         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7624
7625         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7626                                    silent);
7627         return rc;
7628 }
7629
7630 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7631 {
7632         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7633         u16 fw_maj, fw_min, fw_bld, fw_rsv;
7634         u32 dev_caps_cfg, hwrm_ver;
7635         int rc, len;
7636
7637         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7638         mutex_lock(&bp->hwrm_cmd_lock);
7639         rc = __bnxt_hwrm_ver_get(bp, false);
7640         if (rc)
7641                 goto hwrm_ver_get_exit;
7642
7643         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7644
7645         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7646                              resp->hwrm_intf_min_8b << 8 |
7647                              resp->hwrm_intf_upd_8b;
7648         if (resp->hwrm_intf_maj_8b < 1) {
7649                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7650                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7651                             resp->hwrm_intf_upd_8b);
7652                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7653         }
7654
7655         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7656                         HWRM_VERSION_UPDATE;
7657
7658         if (bp->hwrm_spec_code > hwrm_ver)
7659                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7660                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7661                          HWRM_VERSION_UPDATE);
7662         else
7663                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7664                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7665                          resp->hwrm_intf_upd_8b);
7666
7667         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7668         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7669                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7670                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7671                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7672                 len = FW_VER_STR_LEN;
7673         } else {
7674                 fw_maj = resp->hwrm_fw_maj_8b;
7675                 fw_min = resp->hwrm_fw_min_8b;
7676                 fw_bld = resp->hwrm_fw_bld_8b;
7677                 fw_rsv = resp->hwrm_fw_rsvd_8b;
7678                 len = BC_HWRM_STR_LEN;
7679         }
7680         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7681         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7682                  fw_rsv);
7683
7684         if (strlen(resp->active_pkg_name)) {
7685                 int fw_ver_len = strlen(bp->fw_ver_str);
7686
7687                 snprintf(bp->fw_ver_str + fw_ver_len,
7688                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7689                          resp->active_pkg_name);
7690                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7691         }
7692
7693         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7694         if (!bp->hwrm_cmd_timeout)
7695                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7696
7697         if (resp->hwrm_intf_maj_8b >= 1) {
7698                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7699                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7700         }
7701         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7702                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7703
7704         bp->chip_num = le16_to_cpu(resp->chip_num);
7705         bp->chip_rev = resp->chip_rev;
7706         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7707             !resp->chip_metal)
7708                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7709
7710         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7711         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7712             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7713                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7714
7715         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7716                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7717
7718         if (dev_caps_cfg &
7719             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7720                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7721
7722         if (dev_caps_cfg &
7723             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7724                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7725
7726         if (dev_caps_cfg &
7727             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7728                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7729
7730 hwrm_ver_get_exit:
7731         mutex_unlock(&bp->hwrm_cmd_lock);
7732         return rc;
7733 }
7734
7735 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7736 {
7737         struct hwrm_fw_set_time_input req = {0};
7738         struct tm tm;
7739         time64_t now = ktime_get_real_seconds();
7740
7741         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7742             bp->hwrm_spec_code < 0x10400)
7743                 return -EOPNOTSUPP;
7744
7745         time64_to_tm(now, 0, &tm);
7746         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7747         req.year = cpu_to_le16(1900 + tm.tm_year);
7748         req.month = 1 + tm.tm_mon;
7749         req.day = tm.tm_mday;
7750         req.hour = tm.tm_hour;
7751         req.minute = tm.tm_min;
7752         req.second = tm.tm_sec;
7753         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7754 }
7755
7756 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7757 {
7758         u64 sw_tmp;
7759
7760         sw_tmp = (*sw & ~mask) | hw;
7761         if (hw < (*sw & mask))
7762                 sw_tmp += mask + 1;
7763         WRITE_ONCE(*sw, sw_tmp);
7764 }
7765
7766 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7767                                     int count, bool ignore_zero)
7768 {
7769         int i;
7770
7771         for (i = 0; i < count; i++) {
7772                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7773
7774                 if (ignore_zero && !hw)
7775                         continue;
7776
7777                 if (masks[i] == -1ULL)
7778                         sw_stats[i] = hw;
7779                 else
7780                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7781         }
7782 }
7783
7784 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7785 {
7786         if (!stats->hw_stats)
7787                 return;
7788
7789         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7790                                 stats->hw_masks, stats->len / 8, false);
7791 }
7792
7793 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7794 {
7795         struct bnxt_stats_mem *ring0_stats;
7796         bool ignore_zero = false;
7797         int i;
7798
7799         /* Chip bug.  Counter intermittently becomes 0. */
7800         if (bp->flags & BNXT_FLAG_CHIP_P5)
7801                 ignore_zero = true;
7802
7803         for (i = 0; i < bp->cp_nr_rings; i++) {
7804                 struct bnxt_napi *bnapi = bp->bnapi[i];
7805                 struct bnxt_cp_ring_info *cpr;
7806                 struct bnxt_stats_mem *stats;
7807
7808                 cpr = &bnapi->cp_ring;
7809                 stats = &cpr->stats;
7810                 if (!i)
7811                         ring0_stats = stats;
7812                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7813                                         ring0_stats->hw_masks,
7814                                         ring0_stats->len / 8, ignore_zero);
7815         }
7816         if (bp->flags & BNXT_FLAG_PORT_STATS) {
7817                 struct bnxt_stats_mem *stats = &bp->port_stats;
7818                 __le64 *hw_stats = stats->hw_stats;
7819                 u64 *sw_stats = stats->sw_stats;
7820                 u64 *masks = stats->hw_masks;
7821                 int cnt;
7822
7823                 cnt = sizeof(struct rx_port_stats) / 8;
7824                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7825
7826                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7827                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7828                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7829                 cnt = sizeof(struct tx_port_stats) / 8;
7830                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7831         }
7832         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7833                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7834                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7835         }
7836 }
7837
7838 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7839 {
7840         struct bnxt_pf_info *pf = &bp->pf;
7841         struct hwrm_port_qstats_input req = {0};
7842
7843         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7844                 return 0;
7845
7846         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7847                 return -EOPNOTSUPP;
7848
7849         req.flags = flags;
7850         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7851         req.port_id = cpu_to_le16(pf->port_id);
7852         req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7853                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
7854         req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7855         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7856 }
7857
7858 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7859 {
7860         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7861         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7862         struct hwrm_port_qstats_ext_input req = {0};
7863         struct bnxt_pf_info *pf = &bp->pf;
7864         u32 tx_stat_size;
7865         int rc;
7866
7867         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7868                 return 0;
7869
7870         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7871                 return -EOPNOTSUPP;
7872
7873         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7874         req.flags = flags;
7875         req.port_id = cpu_to_le16(pf->port_id);
7876         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7877         req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7878         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7879                        sizeof(struct tx_port_stats_ext) : 0;
7880         req.tx_stat_size = cpu_to_le16(tx_stat_size);
7881         req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7882         mutex_lock(&bp->hwrm_cmd_lock);
7883         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7884         if (!rc) {
7885                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7886                 bp->fw_tx_stats_ext_size = tx_stat_size ?
7887                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7888         } else {
7889                 bp->fw_rx_stats_ext_size = 0;
7890                 bp->fw_tx_stats_ext_size = 0;
7891         }
7892         if (flags)
7893                 goto qstats_done;
7894
7895         if (bp->fw_tx_stats_ext_size <=
7896             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7897                 mutex_unlock(&bp->hwrm_cmd_lock);
7898                 bp->pri2cos_valid = 0;
7899                 return rc;
7900         }
7901
7902         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7903         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7904
7905         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7906         if (!rc) {
7907                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7908                 u8 *pri2cos;
7909                 int i, j;
7910
7911                 resp2 = bp->hwrm_cmd_resp_addr;
7912                 pri2cos = &resp2->pri0_cos_queue_id;
7913                 for (i = 0; i < 8; i++) {
7914                         u8 queue_id = pri2cos[i];
7915                         u8 queue_idx;
7916
7917                         /* Per port queue IDs start from 0, 10, 20, etc */
7918                         queue_idx = queue_id % 10;
7919                         if (queue_idx > BNXT_MAX_QUEUE) {
7920                                 bp->pri2cos_valid = false;
7921                                 goto qstats_done;
7922                         }
7923                         for (j = 0; j < bp->max_q; j++) {
7924                                 if (bp->q_ids[j] == queue_id)
7925                                         bp->pri2cos_idx[i] = queue_idx;
7926                         }
7927                 }
7928                 bp->pri2cos_valid = 1;
7929         }
7930 qstats_done:
7931         mutex_unlock(&bp->hwrm_cmd_lock);
7932         return rc;
7933 }
7934
7935 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7936 {
7937         if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7938                 bnxt_hwrm_tunnel_dst_port_free(
7939                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7940         if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7941                 bnxt_hwrm_tunnel_dst_port_free(
7942                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7943 }
7944
7945 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7946 {
7947         int rc, i;
7948         u32 tpa_flags = 0;
7949
7950         if (set_tpa)
7951                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7952         else if (BNXT_NO_FW_ACCESS(bp))
7953                 return 0;
7954         for (i = 0; i < bp->nr_vnics; i++) {
7955                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7956                 if (rc) {
7957                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7958                                    i, rc);
7959                         return rc;
7960                 }
7961         }
7962         return 0;
7963 }
7964
7965 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7966 {
7967         int i;
7968
7969         for (i = 0; i < bp->nr_vnics; i++)
7970                 bnxt_hwrm_vnic_set_rss(bp, i, false);
7971 }
7972
7973 static void bnxt_clear_vnic(struct bnxt *bp)
7974 {
7975         if (!bp->vnic_info)
7976                 return;
7977
7978         bnxt_hwrm_clear_vnic_filter(bp);
7979         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7980                 /* clear all RSS setting before free vnic ctx */
7981                 bnxt_hwrm_clear_vnic_rss(bp);
7982                 bnxt_hwrm_vnic_ctx_free(bp);
7983         }
7984         /* before free the vnic, undo the vnic tpa settings */
7985         if (bp->flags & BNXT_FLAG_TPA)
7986                 bnxt_set_tpa(bp, false);
7987         bnxt_hwrm_vnic_free(bp);
7988         if (bp->flags & BNXT_FLAG_CHIP_P5)
7989                 bnxt_hwrm_vnic_ctx_free(bp);
7990 }
7991
7992 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7993                                     bool irq_re_init)
7994 {
7995         bnxt_clear_vnic(bp);
7996         bnxt_hwrm_ring_free(bp, close_path);
7997         bnxt_hwrm_ring_grp_free(bp);
7998         if (irq_re_init) {
7999                 bnxt_hwrm_stat_ctx_free(bp);
8000                 bnxt_hwrm_free_tunnel_ports(bp);
8001         }
8002 }
8003
8004 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8005 {
8006         struct hwrm_func_cfg_input req = {0};
8007
8008         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8009         req.fid = cpu_to_le16(0xffff);
8010         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8011         if (br_mode == BRIDGE_MODE_VEB)
8012                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8013         else if (br_mode == BRIDGE_MODE_VEPA)
8014                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8015         else
8016                 return -EINVAL;
8017         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8018 }
8019
8020 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8021 {
8022         struct hwrm_func_cfg_input req = {0};
8023
8024         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8025                 return 0;
8026
8027         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8028         req.fid = cpu_to_le16(0xffff);
8029         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8030         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8031         if (size == 128)
8032                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8033
8034         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8035 }
8036
8037 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8038 {
8039         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8040         int rc;
8041
8042         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8043                 goto skip_rss_ctx;
8044
8045         /* allocate context for vnic */
8046         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8047         if (rc) {
8048                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8049                            vnic_id, rc);
8050                 goto vnic_setup_err;
8051         }
8052         bp->rsscos_nr_ctxs++;
8053
8054         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8055                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8056                 if (rc) {
8057                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8058                                    vnic_id, rc);
8059                         goto vnic_setup_err;
8060                 }
8061                 bp->rsscos_nr_ctxs++;
8062         }
8063
8064 skip_rss_ctx:
8065         /* configure default vnic, ring grp */
8066         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8067         if (rc) {
8068                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8069                            vnic_id, rc);
8070                 goto vnic_setup_err;
8071         }
8072
8073         /* Enable RSS hashing on vnic */
8074         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8075         if (rc) {
8076                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8077                            vnic_id, rc);
8078                 goto vnic_setup_err;
8079         }
8080
8081         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8082                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8083                 if (rc) {
8084                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8085                                    vnic_id, rc);
8086                 }
8087         }
8088
8089 vnic_setup_err:
8090         return rc;
8091 }
8092
8093 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8094 {
8095         int rc, i, nr_ctxs;
8096
8097         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8098         for (i = 0; i < nr_ctxs; i++) {
8099                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8100                 if (rc) {
8101                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8102                                    vnic_id, i, rc);
8103                         break;
8104                 }
8105                 bp->rsscos_nr_ctxs++;
8106         }
8107         if (i < nr_ctxs)
8108                 return -ENOMEM;
8109
8110         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8111         if (rc) {
8112                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8113                            vnic_id, rc);
8114                 return rc;
8115         }
8116         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8117         if (rc) {
8118                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8119                            vnic_id, rc);
8120                 return rc;
8121         }
8122         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8123                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8124                 if (rc) {
8125                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8126                                    vnic_id, rc);
8127                 }
8128         }
8129         return rc;
8130 }
8131
8132 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8133 {
8134         if (bp->flags & BNXT_FLAG_CHIP_P5)
8135                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8136         else
8137                 return __bnxt_setup_vnic(bp, vnic_id);
8138 }
8139
8140 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8141 {
8142 #ifdef CONFIG_RFS_ACCEL
8143         int i, rc = 0;
8144
8145         if (bp->flags & BNXT_FLAG_CHIP_P5)
8146                 return 0;
8147
8148         for (i = 0; i < bp->rx_nr_rings; i++) {
8149                 struct bnxt_vnic_info *vnic;
8150                 u16 vnic_id = i + 1;
8151                 u16 ring_id = i;
8152
8153                 if (vnic_id >= bp->nr_vnics)
8154                         break;
8155
8156                 vnic = &bp->vnic_info[vnic_id];
8157                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8158                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8159                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8160                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8161                 if (rc) {
8162                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8163                                    vnic_id, rc);
8164                         break;
8165                 }
8166                 rc = bnxt_setup_vnic(bp, vnic_id);
8167                 if (rc)
8168                         break;
8169         }
8170         return rc;
8171 #else
8172         return 0;
8173 #endif
8174 }
8175
8176 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8177 static bool bnxt_promisc_ok(struct bnxt *bp)
8178 {
8179 #ifdef CONFIG_BNXT_SRIOV
8180         if (BNXT_VF(bp) && !bp->vf.vlan)
8181                 return false;
8182 #endif
8183         return true;
8184 }
8185
8186 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8187 {
8188         unsigned int rc = 0;
8189
8190         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8191         if (rc) {
8192                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8193                            rc);
8194                 return rc;
8195         }
8196
8197         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8198         if (rc) {
8199                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8200                            rc);
8201                 return rc;
8202         }
8203         return rc;
8204 }
8205
8206 static int bnxt_cfg_rx_mode(struct bnxt *);
8207 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8208
8209 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8210 {
8211         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8212         int rc = 0;
8213         unsigned int rx_nr_rings = bp->rx_nr_rings;
8214
8215         if (irq_re_init) {
8216                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8217                 if (rc) {
8218                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8219                                    rc);
8220                         goto err_out;
8221                 }
8222         }
8223
8224         rc = bnxt_hwrm_ring_alloc(bp);
8225         if (rc) {
8226                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8227                 goto err_out;
8228         }
8229
8230         rc = bnxt_hwrm_ring_grp_alloc(bp);
8231         if (rc) {
8232                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8233                 goto err_out;
8234         }
8235
8236         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8237                 rx_nr_rings--;
8238
8239         /* default vnic 0 */
8240         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8241         if (rc) {
8242                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8243                 goto err_out;
8244         }
8245
8246         rc = bnxt_setup_vnic(bp, 0);
8247         if (rc)
8248                 goto err_out;
8249
8250         if (bp->flags & BNXT_FLAG_RFS) {
8251                 rc = bnxt_alloc_rfs_vnics(bp);
8252                 if (rc)
8253                         goto err_out;
8254         }
8255
8256         if (bp->flags & BNXT_FLAG_TPA) {
8257                 rc = bnxt_set_tpa(bp, true);
8258                 if (rc)
8259                         goto err_out;
8260         }
8261
8262         if (BNXT_VF(bp))
8263                 bnxt_update_vf_mac(bp);
8264
8265         /* Filter for default vnic 0 */
8266         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8267         if (rc) {
8268                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8269                 goto err_out;
8270         }
8271         vnic->uc_filter_count = 1;
8272
8273         vnic->rx_mask = 0;
8274         if (bp->dev->flags & IFF_BROADCAST)
8275                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8276
8277         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8278                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8279
8280         if (bp->dev->flags & IFF_ALLMULTI) {
8281                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8282                 vnic->mc_list_count = 0;
8283         } else {
8284                 u32 mask = 0;
8285
8286                 bnxt_mc_list_updated(bp, &mask);
8287                 vnic->rx_mask |= mask;
8288         }
8289
8290         rc = bnxt_cfg_rx_mode(bp);
8291         if (rc)
8292                 goto err_out;
8293
8294         rc = bnxt_hwrm_set_coal(bp);
8295         if (rc)
8296                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8297                                 rc);
8298
8299         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8300                 rc = bnxt_setup_nitroa0_vnic(bp);
8301                 if (rc)
8302                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8303                                    rc);
8304         }
8305
8306         if (BNXT_VF(bp)) {
8307                 bnxt_hwrm_func_qcfg(bp);
8308                 netdev_update_features(bp->dev);
8309         }
8310
8311         return 0;
8312
8313 err_out:
8314         bnxt_hwrm_resource_free(bp, 0, true);
8315
8316         return rc;
8317 }
8318
8319 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8320 {
8321         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8322         return 0;
8323 }
8324
8325 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8326 {
8327         bnxt_init_cp_rings(bp);
8328         bnxt_init_rx_rings(bp);
8329         bnxt_init_tx_rings(bp);
8330         bnxt_init_ring_grps(bp, irq_re_init);
8331         bnxt_init_vnics(bp);
8332
8333         return bnxt_init_chip(bp, irq_re_init);
8334 }
8335
8336 static int bnxt_set_real_num_queues(struct bnxt *bp)
8337 {
8338         int rc;
8339         struct net_device *dev = bp->dev;
8340
8341         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8342                                           bp->tx_nr_rings_xdp);
8343         if (rc)
8344                 return rc;
8345
8346         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8347         if (rc)
8348                 return rc;
8349
8350 #ifdef CONFIG_RFS_ACCEL
8351         if (bp->flags & BNXT_FLAG_RFS)
8352                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8353 #endif
8354
8355         return rc;
8356 }
8357
8358 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8359                            bool shared)
8360 {
8361         int _rx = *rx, _tx = *tx;
8362
8363         if (shared) {
8364                 *rx = min_t(int, _rx, max);
8365                 *tx = min_t(int, _tx, max);
8366         } else {
8367                 if (max < 2)
8368                         return -ENOMEM;
8369
8370                 while (_rx + _tx > max) {
8371                         if (_rx > _tx && _rx > 1)
8372                                 _rx--;
8373                         else if (_tx > 1)
8374                                 _tx--;
8375                 }
8376                 *rx = _rx;
8377                 *tx = _tx;
8378         }
8379         return 0;
8380 }
8381
8382 static void bnxt_setup_msix(struct bnxt *bp)
8383 {
8384         const int len = sizeof(bp->irq_tbl[0].name);
8385         struct net_device *dev = bp->dev;
8386         int tcs, i;
8387
8388         tcs = netdev_get_num_tc(dev);
8389         if (tcs) {
8390                 int i, off, count;
8391
8392                 for (i = 0; i < tcs; i++) {
8393                         count = bp->tx_nr_rings_per_tc;
8394                         off = i * count;
8395                         netdev_set_tc_queue(dev, i, count, off);
8396                 }
8397         }
8398
8399         for (i = 0; i < bp->cp_nr_rings; i++) {
8400                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8401                 char *attr;
8402
8403                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8404                         attr = "TxRx";
8405                 else if (i < bp->rx_nr_rings)
8406                         attr = "rx";
8407                 else
8408                         attr = "tx";
8409
8410                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8411                          attr, i);
8412                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8413         }
8414 }
8415
8416 static void bnxt_setup_inta(struct bnxt *bp)
8417 {
8418         const int len = sizeof(bp->irq_tbl[0].name);
8419
8420         if (netdev_get_num_tc(bp->dev))
8421                 netdev_reset_tc(bp->dev);
8422
8423         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8424                  0);
8425         bp->irq_tbl[0].handler = bnxt_inta;
8426 }
8427
8428 static int bnxt_setup_int_mode(struct bnxt *bp)
8429 {
8430         int rc;
8431
8432         if (bp->flags & BNXT_FLAG_USING_MSIX)
8433                 bnxt_setup_msix(bp);
8434         else
8435                 bnxt_setup_inta(bp);
8436
8437         rc = bnxt_set_real_num_queues(bp);
8438         return rc;
8439 }
8440
8441 #ifdef CONFIG_RFS_ACCEL
8442 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8443 {
8444         return bp->hw_resc.max_rsscos_ctxs;
8445 }
8446
8447 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8448 {
8449         return bp->hw_resc.max_vnics;
8450 }
8451 #endif
8452
8453 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8454 {
8455         return bp->hw_resc.max_stat_ctxs;
8456 }
8457
8458 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8459 {
8460         return bp->hw_resc.max_cp_rings;
8461 }
8462
8463 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8464 {
8465         unsigned int cp = bp->hw_resc.max_cp_rings;
8466
8467         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8468                 cp -= bnxt_get_ulp_msix_num(bp);
8469
8470         return cp;
8471 }
8472
8473 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8474 {
8475         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8476
8477         if (bp->flags & BNXT_FLAG_CHIP_P5)
8478                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8479
8480         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8481 }
8482
8483 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8484 {
8485         bp->hw_resc.max_irqs = max_irqs;
8486 }
8487
8488 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8489 {
8490         unsigned int cp;
8491
8492         cp = bnxt_get_max_func_cp_rings_for_en(bp);
8493         if (bp->flags & BNXT_FLAG_CHIP_P5)
8494                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8495         else
8496                 return cp - bp->cp_nr_rings;
8497 }
8498
8499 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8500 {
8501         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8502 }
8503
8504 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8505 {
8506         int max_cp = bnxt_get_max_func_cp_rings(bp);
8507         int max_irq = bnxt_get_max_func_irqs(bp);
8508         int total_req = bp->cp_nr_rings + num;
8509         int max_idx, avail_msix;
8510
8511         max_idx = bp->total_irqs;
8512         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8513                 max_idx = min_t(int, bp->total_irqs, max_cp);
8514         avail_msix = max_idx - bp->cp_nr_rings;
8515         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8516                 return avail_msix;
8517
8518         if (max_irq < total_req) {
8519                 num = max_irq - bp->cp_nr_rings;
8520                 if (num <= 0)
8521                         return 0;
8522         }
8523         return num;
8524 }
8525
8526 static int bnxt_get_num_msix(struct bnxt *bp)
8527 {
8528         if (!BNXT_NEW_RM(bp))
8529                 return bnxt_get_max_func_irqs(bp);
8530
8531         return bnxt_nq_rings_in_use(bp);
8532 }
8533
8534 static int bnxt_init_msix(struct bnxt *bp)
8535 {
8536         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8537         struct msix_entry *msix_ent;
8538
8539         total_vecs = bnxt_get_num_msix(bp);
8540         max = bnxt_get_max_func_irqs(bp);
8541         if (total_vecs > max)
8542                 total_vecs = max;
8543
8544         if (!total_vecs)
8545                 return 0;
8546
8547         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8548         if (!msix_ent)
8549                 return -ENOMEM;
8550
8551         for (i = 0; i < total_vecs; i++) {
8552                 msix_ent[i].entry = i;
8553                 msix_ent[i].vector = 0;
8554         }
8555
8556         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8557                 min = 2;
8558
8559         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8560         ulp_msix = bnxt_get_ulp_msix_num(bp);
8561         if (total_vecs < 0 || total_vecs < ulp_msix) {
8562                 rc = -ENODEV;
8563                 goto msix_setup_exit;
8564         }
8565
8566         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8567         if (bp->irq_tbl) {
8568                 for (i = 0; i < total_vecs; i++)
8569                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8570
8571                 bp->total_irqs = total_vecs;
8572                 /* Trim rings based upon num of vectors allocated */
8573                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8574                                      total_vecs - ulp_msix, min == 1);
8575                 if (rc)
8576                         goto msix_setup_exit;
8577
8578                 bp->cp_nr_rings = (min == 1) ?
8579                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8580                                   bp->tx_nr_rings + bp->rx_nr_rings;
8581
8582         } else {
8583                 rc = -ENOMEM;
8584                 goto msix_setup_exit;
8585         }
8586         bp->flags |= BNXT_FLAG_USING_MSIX;
8587         kfree(msix_ent);
8588         return 0;
8589
8590 msix_setup_exit:
8591         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8592         kfree(bp->irq_tbl);
8593         bp->irq_tbl = NULL;
8594         pci_disable_msix(bp->pdev);
8595         kfree(msix_ent);
8596         return rc;
8597 }
8598
8599 static int bnxt_init_inta(struct bnxt *bp)
8600 {
8601         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8602         if (!bp->irq_tbl)
8603                 return -ENOMEM;
8604
8605         bp->total_irqs = 1;
8606         bp->rx_nr_rings = 1;
8607         bp->tx_nr_rings = 1;
8608         bp->cp_nr_rings = 1;
8609         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8610         bp->irq_tbl[0].vector = bp->pdev->irq;
8611         return 0;
8612 }
8613
8614 static int bnxt_init_int_mode(struct bnxt *bp)
8615 {
8616         int rc = 0;
8617
8618         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8619                 rc = bnxt_init_msix(bp);
8620
8621         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8622                 /* fallback to INTA */
8623                 rc = bnxt_init_inta(bp);
8624         }
8625         return rc;
8626 }
8627
8628 static void bnxt_clear_int_mode(struct bnxt *bp)
8629 {
8630         if (bp->flags & BNXT_FLAG_USING_MSIX)
8631                 pci_disable_msix(bp->pdev);
8632
8633         kfree(bp->irq_tbl);
8634         bp->irq_tbl = NULL;
8635         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8636 }
8637
8638 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8639 {
8640         int tcs = netdev_get_num_tc(bp->dev);
8641         bool irq_cleared = false;
8642         int rc;
8643
8644         if (!bnxt_need_reserve_rings(bp))
8645                 return 0;
8646
8647         if (irq_re_init && BNXT_NEW_RM(bp) &&
8648             bnxt_get_num_msix(bp) != bp->total_irqs) {
8649                 bnxt_ulp_irq_stop(bp);
8650                 bnxt_clear_int_mode(bp);
8651                 irq_cleared = true;
8652         }
8653         rc = __bnxt_reserve_rings(bp);
8654         if (irq_cleared) {
8655                 if (!rc)
8656                         rc = bnxt_init_int_mode(bp);
8657                 bnxt_ulp_irq_restart(bp, rc);
8658         }
8659         if (rc) {
8660                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8661                 return rc;
8662         }
8663         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8664                 netdev_err(bp->dev, "tx ring reservation failure\n");
8665                 netdev_reset_tc(bp->dev);
8666                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8667                 return -ENOMEM;
8668         }
8669         return 0;
8670 }
8671
8672 static void bnxt_free_irq(struct bnxt *bp)
8673 {
8674         struct bnxt_irq *irq;
8675         int i;
8676
8677 #ifdef CONFIG_RFS_ACCEL
8678         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8679         bp->dev->rx_cpu_rmap = NULL;
8680 #endif
8681         if (!bp->irq_tbl || !bp->bnapi)
8682                 return;
8683
8684         for (i = 0; i < bp->cp_nr_rings; i++) {
8685                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8686
8687                 irq = &bp->irq_tbl[map_idx];
8688                 if (irq->requested) {
8689                         if (irq->have_cpumask) {
8690                                 irq_set_affinity_hint(irq->vector, NULL);
8691                                 free_cpumask_var(irq->cpu_mask);
8692                                 irq->have_cpumask = 0;
8693                         }
8694                         free_irq(irq->vector, bp->bnapi[i]);
8695                 }
8696
8697                 irq->requested = 0;
8698         }
8699 }
8700
8701 static int bnxt_request_irq(struct bnxt *bp)
8702 {
8703         int i, j, rc = 0;
8704         unsigned long flags = 0;
8705 #ifdef CONFIG_RFS_ACCEL
8706         struct cpu_rmap *rmap;
8707 #endif
8708
8709         rc = bnxt_setup_int_mode(bp);
8710         if (rc) {
8711                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8712                            rc);
8713                 return rc;
8714         }
8715 #ifdef CONFIG_RFS_ACCEL
8716         rmap = bp->dev->rx_cpu_rmap;
8717 #endif
8718         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8719                 flags = IRQF_SHARED;
8720
8721         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8722                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8723                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8724
8725 #ifdef CONFIG_RFS_ACCEL
8726                 if (rmap && bp->bnapi[i]->rx_ring) {
8727                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8728                         if (rc)
8729                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8730                                             j);
8731                         j++;
8732                 }
8733 #endif
8734                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8735                                  bp->bnapi[i]);
8736                 if (rc)
8737                         break;
8738
8739                 irq->requested = 1;
8740
8741                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8742                         int numa_node = dev_to_node(&bp->pdev->dev);
8743
8744                         irq->have_cpumask = 1;
8745                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8746                                         irq->cpu_mask);
8747                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8748                         if (rc) {
8749                                 netdev_warn(bp->dev,
8750                                             "Set affinity failed, IRQ = %d\n",
8751                                             irq->vector);
8752                                 break;
8753                         }
8754                 }
8755         }
8756         return rc;
8757 }
8758
8759 static void bnxt_del_napi(struct bnxt *bp)
8760 {
8761         int i;
8762
8763         if (!bp->bnapi)
8764                 return;
8765
8766         for (i = 0; i < bp->cp_nr_rings; i++) {
8767                 struct bnxt_napi *bnapi = bp->bnapi[i];
8768
8769                 __netif_napi_del(&bnapi->napi);
8770         }
8771         /* We called __netif_napi_del(), we need
8772          * to respect an RCU grace period before freeing napi structures.
8773          */
8774         synchronize_net();
8775 }
8776
8777 static void bnxt_init_napi(struct bnxt *bp)
8778 {
8779         int i;
8780         unsigned int cp_nr_rings = bp->cp_nr_rings;
8781         struct bnxt_napi *bnapi;
8782
8783         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8784                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8785
8786                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8787                         poll_fn = bnxt_poll_p5;
8788                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8789                         cp_nr_rings--;
8790                 for (i = 0; i < cp_nr_rings; i++) {
8791                         bnapi = bp->bnapi[i];
8792                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8793                 }
8794                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8795                         bnapi = bp->bnapi[cp_nr_rings];
8796                         netif_napi_add(bp->dev, &bnapi->napi,
8797                                        bnxt_poll_nitroa0, 64);
8798                 }
8799         } else {
8800                 bnapi = bp->bnapi[0];
8801                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8802         }
8803 }
8804
8805 static void bnxt_disable_napi(struct bnxt *bp)
8806 {
8807         int i;
8808
8809         if (!bp->bnapi)
8810                 return;
8811
8812         for (i = 0; i < bp->cp_nr_rings; i++) {
8813                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8814
8815                 if (bp->bnapi[i]->rx_ring)
8816                         cancel_work_sync(&cpr->dim.work);
8817
8818                 napi_disable(&bp->bnapi[i]->napi);
8819         }
8820 }
8821
8822 static void bnxt_enable_napi(struct bnxt *bp)
8823 {
8824         int i;
8825
8826         for (i = 0; i < bp->cp_nr_rings; i++) {
8827                 struct bnxt_napi *bnapi = bp->bnapi[i];
8828                 struct bnxt_cp_ring_info *cpr;
8829
8830                 cpr = &bnapi->cp_ring;
8831                 if (bnapi->in_reset)
8832                         cpr->sw_stats.rx.rx_resets++;
8833                 bnapi->in_reset = false;
8834
8835                 if (bnapi->rx_ring) {
8836                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8837                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8838                 }
8839                 napi_enable(&bnapi->napi);
8840         }
8841 }
8842
8843 void bnxt_tx_disable(struct bnxt *bp)
8844 {
8845         int i;
8846         struct bnxt_tx_ring_info *txr;
8847
8848         if (bp->tx_ring) {
8849                 for (i = 0; i < bp->tx_nr_rings; i++) {
8850                         txr = &bp->tx_ring[i];
8851                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8852                 }
8853         }
8854         /* Stop all TX queues */
8855         netif_tx_disable(bp->dev);
8856         netif_carrier_off(bp->dev);
8857 }
8858
8859 void bnxt_tx_enable(struct bnxt *bp)
8860 {
8861         int i;
8862         struct bnxt_tx_ring_info *txr;
8863
8864         for (i = 0; i < bp->tx_nr_rings; i++) {
8865                 txr = &bp->tx_ring[i];
8866                 txr->dev_state = 0;
8867         }
8868         netif_tx_wake_all_queues(bp->dev);
8869         if (bp->link_info.link_up)
8870                 netif_carrier_on(bp->dev);
8871 }
8872
8873 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
8874 {
8875         u8 active_fec = link_info->active_fec_sig_mode &
8876                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
8877
8878         switch (active_fec) {
8879         default:
8880         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
8881                 return "None";
8882         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
8883                 return "Clause 74 BaseR";
8884         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
8885                 return "Clause 91 RS(528,514)";
8886         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
8887                 return "Clause 91 RS544_1XN";
8888         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
8889                 return "Clause 91 RS(544,514)";
8890         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
8891                 return "Clause 91 RS272_1XN";
8892         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
8893                 return "Clause 91 RS(272,257)";
8894         }
8895 }
8896
8897 static void bnxt_report_link(struct bnxt *bp)
8898 {
8899         if (bp->link_info.link_up) {
8900                 const char *duplex;
8901                 const char *flow_ctrl;
8902                 u32 speed;
8903                 u16 fec;
8904
8905                 netif_carrier_on(bp->dev);
8906                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8907                 if (speed == SPEED_UNKNOWN) {
8908                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
8909                         return;
8910                 }
8911                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8912                         duplex = "full";
8913                 else
8914                         duplex = "half";
8915                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8916                         flow_ctrl = "ON - receive & transmit";
8917                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8918                         flow_ctrl = "ON - transmit";
8919                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8920                         flow_ctrl = "ON - receive";
8921                 else
8922                         flow_ctrl = "none";
8923                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8924                             speed, duplex, flow_ctrl);
8925                 if (bp->flags & BNXT_FLAG_EEE_CAP)
8926                         netdev_info(bp->dev, "EEE is %s\n",
8927                                     bp->eee.eee_active ? "active" :
8928                                                          "not active");
8929                 fec = bp->link_info.fec_cfg;
8930                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8931                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
8932                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8933                                     bnxt_report_fec(&bp->link_info));
8934         } else {
8935                 netif_carrier_off(bp->dev);
8936                 netdev_err(bp->dev, "NIC Link is Down\n");
8937         }
8938 }
8939
8940 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
8941 {
8942         if (!resp->supported_speeds_auto_mode &&
8943             !resp->supported_speeds_force_mode &&
8944             !resp->supported_pam4_speeds_auto_mode &&
8945             !resp->supported_pam4_speeds_force_mode)
8946                 return true;
8947         return false;
8948 }
8949
8950 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8951 {
8952         int rc = 0;
8953         struct hwrm_port_phy_qcaps_input req = {0};
8954         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8955         struct bnxt_link_info *link_info = &bp->link_info;
8956
8957         bp->flags &= ~BNXT_FLAG_EEE_CAP;
8958         if (bp->test_info)
8959                 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8960                                           BNXT_TEST_FL_AN_PHY_LPBK);
8961         if (bp->hwrm_spec_code < 0x10201)
8962                 return 0;
8963
8964         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8965
8966         mutex_lock(&bp->hwrm_cmd_lock);
8967         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8968         if (rc)
8969                 goto hwrm_phy_qcaps_exit;
8970
8971         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8972                 struct ethtool_eee *eee = &bp->eee;
8973                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8974
8975                 bp->flags |= BNXT_FLAG_EEE_CAP;
8976                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8977                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8978                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8979                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8980                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8981         }
8982         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8983                 if (bp->test_info)
8984                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8985         }
8986         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8987                 if (bp->test_info)
8988                         bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8989         }
8990         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8991                 if (BNXT_PF(bp))
8992                         bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8993         }
8994         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8995                 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8996
8997         if (bp->hwrm_spec_code >= 0x10a01) {
8998                 if (bnxt_phy_qcaps_no_speed(resp)) {
8999                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9000                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9001                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9002                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9003                         netdev_info(bp->dev, "Ethernet link enabled\n");
9004                         /* Phy re-enabled, reprobe the speeds */
9005                         link_info->support_auto_speeds = 0;
9006                         link_info->support_pam4_auto_speeds = 0;
9007                 }
9008         }
9009         if (resp->supported_speeds_auto_mode)
9010                 link_info->support_auto_speeds =
9011                         le16_to_cpu(resp->supported_speeds_auto_mode);
9012         if (resp->supported_pam4_speeds_auto_mode)
9013                 link_info->support_pam4_auto_speeds =
9014                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9015
9016         bp->port_count = resp->port_cnt;
9017
9018 hwrm_phy_qcaps_exit:
9019         mutex_unlock(&bp->hwrm_cmd_lock);
9020         return rc;
9021 }
9022
9023 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9024 {
9025         u16 diff = advertising ^ supported;
9026
9027         return ((supported | diff) != supported);
9028 }
9029
9030 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9031 {
9032         int rc = 0;
9033         struct bnxt_link_info *link_info = &bp->link_info;
9034         struct hwrm_port_phy_qcfg_input req = {0};
9035         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9036         u8 link_up = link_info->link_up;
9037         bool support_changed = false;
9038
9039         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
9040
9041         mutex_lock(&bp->hwrm_cmd_lock);
9042         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9043         if (rc) {
9044                 mutex_unlock(&bp->hwrm_cmd_lock);
9045                 return rc;
9046         }
9047
9048         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9049         link_info->phy_link_status = resp->link;
9050         link_info->duplex = resp->duplex_cfg;
9051         if (bp->hwrm_spec_code >= 0x10800)
9052                 link_info->duplex = resp->duplex_state;
9053         link_info->pause = resp->pause;
9054         link_info->auto_mode = resp->auto_mode;
9055         link_info->auto_pause_setting = resp->auto_pause;
9056         link_info->lp_pause = resp->link_partner_adv_pause;
9057         link_info->force_pause_setting = resp->force_pause;
9058         link_info->duplex_setting = resp->duplex_cfg;
9059         if (link_info->phy_link_status == BNXT_LINK_LINK)
9060                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9061         else
9062                 link_info->link_speed = 0;
9063         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9064         link_info->force_pam4_link_speed =
9065                 le16_to_cpu(resp->force_pam4_link_speed);
9066         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9067         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9068         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9069         link_info->auto_pam4_link_speeds =
9070                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9071         link_info->lp_auto_link_speeds =
9072                 le16_to_cpu(resp->link_partner_adv_speeds);
9073         link_info->lp_auto_pam4_link_speeds =
9074                 resp->link_partner_pam4_adv_speeds;
9075         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9076         link_info->phy_ver[0] = resp->phy_maj;
9077         link_info->phy_ver[1] = resp->phy_min;
9078         link_info->phy_ver[2] = resp->phy_bld;
9079         link_info->media_type = resp->media_type;
9080         link_info->phy_type = resp->phy_type;
9081         link_info->transceiver = resp->xcvr_pkg_type;
9082         link_info->phy_addr = resp->eee_config_phy_addr &
9083                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9084         link_info->module_status = resp->module_status;
9085
9086         if (bp->flags & BNXT_FLAG_EEE_CAP) {
9087                 struct ethtool_eee *eee = &bp->eee;
9088                 u16 fw_speeds;
9089
9090                 eee->eee_active = 0;
9091                 if (resp->eee_config_phy_addr &
9092                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9093                         eee->eee_active = 1;
9094                         fw_speeds = le16_to_cpu(
9095                                 resp->link_partner_adv_eee_link_speed_mask);
9096                         eee->lp_advertised =
9097                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9098                 }
9099
9100                 /* Pull initial EEE config */
9101                 if (!chng_link_state) {
9102                         if (resp->eee_config_phy_addr &
9103                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9104                                 eee->eee_enabled = 1;
9105
9106                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9107                         eee->advertised =
9108                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9109
9110                         if (resp->eee_config_phy_addr &
9111                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9112                                 __le32 tmr;
9113
9114                                 eee->tx_lpi_enabled = 1;
9115                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9116                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9117                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9118                         }
9119                 }
9120         }
9121
9122         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9123         if (bp->hwrm_spec_code >= 0x10504) {
9124                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9125                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9126         }
9127         /* TODO: need to add more logic to report VF link */
9128         if (chng_link_state) {
9129                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9130                         link_info->link_up = 1;
9131                 else
9132                         link_info->link_up = 0;
9133                 if (link_up != link_info->link_up)
9134                         bnxt_report_link(bp);
9135         } else {
9136                 /* alwasy link down if not require to update link state */
9137                 link_info->link_up = 0;
9138         }
9139         mutex_unlock(&bp->hwrm_cmd_lock);
9140
9141         if (!BNXT_PHY_CFG_ABLE(bp))
9142                 return 0;
9143
9144         /* Check if any advertised speeds are no longer supported. The caller
9145          * holds the link_lock mutex, so we can modify link_info settings.
9146          */
9147         if (bnxt_support_dropped(link_info->advertising,
9148                                  link_info->support_auto_speeds)) {
9149                 link_info->advertising = link_info->support_auto_speeds;
9150                 support_changed = true;
9151         }
9152         if (bnxt_support_dropped(link_info->advertising_pam4,
9153                                  link_info->support_pam4_auto_speeds)) {
9154                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9155                 support_changed = true;
9156         }
9157         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9158                 bnxt_hwrm_set_link_setting(bp, true, false);
9159         return 0;
9160 }
9161
9162 static void bnxt_get_port_module_status(struct bnxt *bp)
9163 {
9164         struct bnxt_link_info *link_info = &bp->link_info;
9165         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9166         u8 module_status;
9167
9168         if (bnxt_update_link(bp, true))
9169                 return;
9170
9171         module_status = link_info->module_status;
9172         switch (module_status) {
9173         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9174         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9175         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9176                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9177                             bp->pf.port_id);
9178                 if (bp->hwrm_spec_code >= 0x10201) {
9179                         netdev_warn(bp->dev, "Module part number %s\n",
9180                                     resp->phy_vendor_partnumber);
9181                 }
9182                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9183                         netdev_warn(bp->dev, "TX is disabled\n");
9184                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9185                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9186         }
9187 }
9188
9189 static void
9190 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9191 {
9192         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9193                 if (bp->hwrm_spec_code >= 0x10201)
9194                         req->auto_pause =
9195                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9196                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9197                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9198                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9199                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9200                 req->enables |=
9201                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9202         } else {
9203                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9204                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9205                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9206                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9207                 req->enables |=
9208                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9209                 if (bp->hwrm_spec_code >= 0x10201) {
9210                         req->auto_pause = req->force_pause;
9211                         req->enables |= cpu_to_le32(
9212                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9213                 }
9214         }
9215 }
9216
9217 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9218 {
9219         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9220                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9221                 if (bp->link_info.advertising) {
9222                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9223                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9224                 }
9225                 if (bp->link_info.advertising_pam4) {
9226                         req->enables |=
9227                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9228                         req->auto_link_pam4_speed_mask =
9229                                 cpu_to_le16(bp->link_info.advertising_pam4);
9230                 }
9231                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9232                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9233         } else {
9234                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9235                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9236                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9237                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9238                 } else {
9239                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9240                 }
9241         }
9242
9243         /* tell chimp that the setting takes effect immediately */
9244         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9245 }
9246
9247 int bnxt_hwrm_set_pause(struct bnxt *bp)
9248 {
9249         struct hwrm_port_phy_cfg_input req = {0};
9250         int rc;
9251
9252         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9253         bnxt_hwrm_set_pause_common(bp, &req);
9254
9255         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9256             bp->link_info.force_link_chng)
9257                 bnxt_hwrm_set_link_common(bp, &req);
9258
9259         mutex_lock(&bp->hwrm_cmd_lock);
9260         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9261         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9262                 /* since changing of pause setting doesn't trigger any link
9263                  * change event, the driver needs to update the current pause
9264                  * result upon successfully return of the phy_cfg command
9265                  */
9266                 bp->link_info.pause =
9267                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9268                 bp->link_info.auto_pause_setting = 0;
9269                 if (!bp->link_info.force_link_chng)
9270                         bnxt_report_link(bp);
9271         }
9272         bp->link_info.force_link_chng = false;
9273         mutex_unlock(&bp->hwrm_cmd_lock);
9274         return rc;
9275 }
9276
9277 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9278                               struct hwrm_port_phy_cfg_input *req)
9279 {
9280         struct ethtool_eee *eee = &bp->eee;
9281
9282         if (eee->eee_enabled) {
9283                 u16 eee_speeds;
9284                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9285
9286                 if (eee->tx_lpi_enabled)
9287                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9288                 else
9289                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9290
9291                 req->flags |= cpu_to_le32(flags);
9292                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9293                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9294                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9295         } else {
9296                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9297         }
9298 }
9299
9300 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9301 {
9302         struct hwrm_port_phy_cfg_input req = {0};
9303
9304         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9305         if (set_pause)
9306                 bnxt_hwrm_set_pause_common(bp, &req);
9307
9308         bnxt_hwrm_set_link_common(bp, &req);
9309
9310         if (set_eee)
9311                 bnxt_hwrm_set_eee(bp, &req);
9312         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9313 }
9314
9315 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9316 {
9317         struct hwrm_port_phy_cfg_input req = {0};
9318
9319         if (!BNXT_SINGLE_PF(bp))
9320                 return 0;
9321
9322         if (pci_num_vf(bp->pdev))
9323                 return 0;
9324
9325         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9326         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9327         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9328 }
9329
9330 static int bnxt_fw_init_one(struct bnxt *bp);
9331
9332 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9333 {
9334         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9335         struct hwrm_func_drv_if_change_input req = {0};
9336         bool resc_reinit = false, fw_reset = false;
9337         u32 flags = 0;
9338         int rc;
9339
9340         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9341                 return 0;
9342
9343         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9344         if (up)
9345                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9346         mutex_lock(&bp->hwrm_cmd_lock);
9347         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9348         if (!rc)
9349                 flags = le32_to_cpu(resp->flags);
9350         mutex_unlock(&bp->hwrm_cmd_lock);
9351         if (rc)
9352                 return rc;
9353
9354         if (!up)
9355                 return 0;
9356
9357         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9358                 resc_reinit = true;
9359         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9360                 fw_reset = true;
9361
9362         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9363                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9364                 return -ENODEV;
9365         }
9366         if (resc_reinit || fw_reset) {
9367                 if (fw_reset) {
9368                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9369                                 bnxt_ulp_stop(bp);
9370                         bnxt_free_ctx_mem(bp);
9371                         kfree(bp->ctx);
9372                         bp->ctx = NULL;
9373                         bnxt_dcb_free(bp);
9374                         rc = bnxt_fw_init_one(bp);
9375                         if (rc) {
9376                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9377                                 return rc;
9378                         }
9379                         bnxt_clear_int_mode(bp);
9380                         rc = bnxt_init_int_mode(bp);
9381                         if (rc) {
9382                                 netdev_err(bp->dev, "init int mode failed\n");
9383                                 return rc;
9384                         }
9385                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9386                 }
9387                 if (BNXT_NEW_RM(bp)) {
9388                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9389
9390                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9391                         hw_resc->resv_cp_rings = 0;
9392                         hw_resc->resv_stat_ctxs = 0;
9393                         hw_resc->resv_irqs = 0;
9394                         hw_resc->resv_tx_rings = 0;
9395                         hw_resc->resv_rx_rings = 0;
9396                         hw_resc->resv_hw_ring_grps = 0;
9397                         hw_resc->resv_vnics = 0;
9398                         if (!fw_reset) {
9399                                 bp->tx_nr_rings = 0;
9400                                 bp->rx_nr_rings = 0;
9401                         }
9402                 }
9403         }
9404         return 0;
9405 }
9406
9407 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9408 {
9409         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9410         struct hwrm_port_led_qcaps_input req = {0};
9411         struct bnxt_pf_info *pf = &bp->pf;
9412         int rc;
9413
9414         bp->num_leds = 0;
9415         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9416                 return 0;
9417
9418         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9419         req.port_id = cpu_to_le16(pf->port_id);
9420         mutex_lock(&bp->hwrm_cmd_lock);
9421         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9422         if (rc) {
9423                 mutex_unlock(&bp->hwrm_cmd_lock);
9424                 return rc;
9425         }
9426         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9427                 int i;
9428
9429                 bp->num_leds = resp->num_leds;
9430                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9431                                                  bp->num_leds);
9432                 for (i = 0; i < bp->num_leds; i++) {
9433                         struct bnxt_led_info *led = &bp->leds[i];
9434                         __le16 caps = led->led_state_caps;
9435
9436                         if (!led->led_group_id ||
9437                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
9438                                 bp->num_leds = 0;
9439                                 break;
9440                         }
9441                 }
9442         }
9443         mutex_unlock(&bp->hwrm_cmd_lock);
9444         return 0;
9445 }
9446
9447 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9448 {
9449         struct hwrm_wol_filter_alloc_input req = {0};
9450         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9451         int rc;
9452
9453         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9454         req.port_id = cpu_to_le16(bp->pf.port_id);
9455         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9456         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9457         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9458         mutex_lock(&bp->hwrm_cmd_lock);
9459         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9460         if (!rc)
9461                 bp->wol_filter_id = resp->wol_filter_id;
9462         mutex_unlock(&bp->hwrm_cmd_lock);
9463         return rc;
9464 }
9465
9466 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9467 {
9468         struct hwrm_wol_filter_free_input req = {0};
9469
9470         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9471         req.port_id = cpu_to_le16(bp->pf.port_id);
9472         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9473         req.wol_filter_id = bp->wol_filter_id;
9474         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9475 }
9476
9477 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9478 {
9479         struct hwrm_wol_filter_qcfg_input req = {0};
9480         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9481         u16 next_handle = 0;
9482         int rc;
9483
9484         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9485         req.port_id = cpu_to_le16(bp->pf.port_id);
9486         req.handle = cpu_to_le16(handle);
9487         mutex_lock(&bp->hwrm_cmd_lock);
9488         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9489         if (!rc) {
9490                 next_handle = le16_to_cpu(resp->next_handle);
9491                 if (next_handle != 0) {
9492                         if (resp->wol_type ==
9493                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9494                                 bp->wol = 1;
9495                                 bp->wol_filter_id = resp->wol_filter_id;
9496                         }
9497                 }
9498         }
9499         mutex_unlock(&bp->hwrm_cmd_lock);
9500         return next_handle;
9501 }
9502
9503 static void bnxt_get_wol_settings(struct bnxt *bp)
9504 {
9505         u16 handle = 0;
9506
9507         bp->wol = 0;
9508         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9509                 return;
9510
9511         do {
9512                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9513         } while (handle && handle != 0xffff);
9514 }
9515
9516 #ifdef CONFIG_BNXT_HWMON
9517 static ssize_t bnxt_show_temp(struct device *dev,
9518                               struct device_attribute *devattr, char *buf)
9519 {
9520         struct hwrm_temp_monitor_query_input req = {0};
9521         struct hwrm_temp_monitor_query_output *resp;
9522         struct bnxt *bp = dev_get_drvdata(dev);
9523         u32 len = 0;
9524         int rc;
9525
9526         resp = bp->hwrm_cmd_resp_addr;
9527         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9528         mutex_lock(&bp->hwrm_cmd_lock);
9529         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9530         if (!rc)
9531                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9532         mutex_unlock(&bp->hwrm_cmd_lock);
9533         return rc ?: len;
9534 }
9535 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9536
9537 static struct attribute *bnxt_attrs[] = {
9538         &sensor_dev_attr_temp1_input.dev_attr.attr,
9539         NULL
9540 };
9541 ATTRIBUTE_GROUPS(bnxt);
9542
9543 static void bnxt_hwmon_close(struct bnxt *bp)
9544 {
9545         if (bp->hwmon_dev) {
9546                 hwmon_device_unregister(bp->hwmon_dev);
9547                 bp->hwmon_dev = NULL;
9548         }
9549 }
9550
9551 static void bnxt_hwmon_open(struct bnxt *bp)
9552 {
9553         struct hwrm_temp_monitor_query_input req = {0};
9554         struct pci_dev *pdev = bp->pdev;
9555         int rc;
9556
9557         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9558         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9559         if (rc == -EACCES || rc == -EOPNOTSUPP) {
9560                 bnxt_hwmon_close(bp);
9561                 return;
9562         }
9563
9564         if (bp->hwmon_dev)
9565                 return;
9566
9567         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9568                                                           DRV_MODULE_NAME, bp,
9569                                                           bnxt_groups);
9570         if (IS_ERR(bp->hwmon_dev)) {
9571                 bp->hwmon_dev = NULL;
9572                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9573         }
9574 }
9575 #else
9576 static void bnxt_hwmon_close(struct bnxt *bp)
9577 {
9578 }
9579
9580 static void bnxt_hwmon_open(struct bnxt *bp)
9581 {
9582 }
9583 #endif
9584
9585 static bool bnxt_eee_config_ok(struct bnxt *bp)
9586 {
9587         struct ethtool_eee *eee = &bp->eee;
9588         struct bnxt_link_info *link_info = &bp->link_info;
9589
9590         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9591                 return true;
9592
9593         if (eee->eee_enabled) {
9594                 u32 advertising =
9595                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9596
9597                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9598                         eee->eee_enabled = 0;
9599                         return false;
9600                 }
9601                 if (eee->advertised & ~advertising) {
9602                         eee->advertised = advertising & eee->supported;
9603                         return false;
9604                 }
9605         }
9606         return true;
9607 }
9608
9609 static int bnxt_update_phy_setting(struct bnxt *bp)
9610 {
9611         int rc;
9612         bool update_link = false;
9613         bool update_pause = false;
9614         bool update_eee = false;
9615         struct bnxt_link_info *link_info = &bp->link_info;
9616
9617         rc = bnxt_update_link(bp, true);
9618         if (rc) {
9619                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9620                            rc);
9621                 return rc;
9622         }
9623         if (!BNXT_SINGLE_PF(bp))
9624                 return 0;
9625
9626         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9627             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9628             link_info->req_flow_ctrl)
9629                 update_pause = true;
9630         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9631             link_info->force_pause_setting != link_info->req_flow_ctrl)
9632                 update_pause = true;
9633         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9634                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9635                         update_link = true;
9636                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9637                     link_info->req_link_speed != link_info->force_link_speed)
9638                         update_link = true;
9639                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9640                          link_info->req_link_speed != link_info->force_pam4_link_speed)
9641                         update_link = true;
9642                 if (link_info->req_duplex != link_info->duplex_setting)
9643                         update_link = true;
9644         } else {
9645                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9646                         update_link = true;
9647                 if (link_info->advertising != link_info->auto_link_speeds ||
9648                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9649                         update_link = true;
9650         }
9651
9652         /* The last close may have shutdown the link, so need to call
9653          * PHY_CFG to bring it back up.
9654          */
9655         if (!bp->link_info.link_up)
9656                 update_link = true;
9657
9658         if (!bnxt_eee_config_ok(bp))
9659                 update_eee = true;
9660
9661         if (update_link)
9662                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9663         else if (update_pause)
9664                 rc = bnxt_hwrm_set_pause(bp);
9665         if (rc) {
9666                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9667                            rc);
9668                 return rc;
9669         }
9670
9671         return rc;
9672 }
9673
9674 /* Common routine to pre-map certain register block to different GRC window.
9675  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9676  * in PF and 3 windows in VF that can be customized to map in different
9677  * register blocks.
9678  */
9679 static void bnxt_preset_reg_win(struct bnxt *bp)
9680 {
9681         if (BNXT_PF(bp)) {
9682                 /* CAG registers map to GRC window #4 */
9683                 writel(BNXT_CAG_REG_BASE,
9684                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9685         }
9686 }
9687
9688 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9689
9690 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9691 {
9692         int rc = 0;
9693
9694         bnxt_preset_reg_win(bp);
9695         netif_carrier_off(bp->dev);
9696         if (irq_re_init) {
9697                 /* Reserve rings now if none were reserved at driver probe. */
9698                 rc = bnxt_init_dflt_ring_mode(bp);
9699                 if (rc) {
9700                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9701                         return rc;
9702                 }
9703         }
9704         rc = bnxt_reserve_rings(bp, irq_re_init);
9705         if (rc)
9706                 return rc;
9707         if ((bp->flags & BNXT_FLAG_RFS) &&
9708             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9709                 /* disable RFS if falling back to INTA */
9710                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9711                 bp->flags &= ~BNXT_FLAG_RFS;
9712         }
9713
9714         rc = bnxt_alloc_mem(bp, irq_re_init);
9715         if (rc) {
9716                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9717                 goto open_err_free_mem;
9718         }
9719
9720         if (irq_re_init) {
9721                 bnxt_init_napi(bp);
9722                 rc = bnxt_request_irq(bp);
9723                 if (rc) {
9724                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9725                         goto open_err_irq;
9726                 }
9727         }
9728
9729         rc = bnxt_init_nic(bp, irq_re_init);
9730         if (rc) {
9731                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9732                 goto open_err_irq;
9733         }
9734
9735         bnxt_enable_napi(bp);
9736         bnxt_debug_dev_init(bp);
9737
9738         if (link_re_init) {
9739                 mutex_lock(&bp->link_lock);
9740                 rc = bnxt_update_phy_setting(bp);
9741                 mutex_unlock(&bp->link_lock);
9742                 if (rc) {
9743                         netdev_warn(bp->dev, "failed to update phy settings\n");
9744                         if (BNXT_SINGLE_PF(bp)) {
9745                                 bp->link_info.phy_retry = true;
9746                                 bp->link_info.phy_retry_expires =
9747                                         jiffies + 5 * HZ;
9748                         }
9749                 }
9750         }
9751
9752         if (irq_re_init)
9753                 udp_tunnel_nic_reset_ntf(bp->dev);
9754
9755         set_bit(BNXT_STATE_OPEN, &bp->state);
9756         bnxt_enable_int(bp);
9757         /* Enable TX queues */
9758         bnxt_tx_enable(bp);
9759         mod_timer(&bp->timer, jiffies + bp->current_interval);
9760         /* Poll link status and check for SFP+ module status */
9761         bnxt_get_port_module_status(bp);
9762
9763         /* VF-reps may need to be re-opened after the PF is re-opened */
9764         if (BNXT_PF(bp))
9765                 bnxt_vf_reps_open(bp);
9766         return 0;
9767
9768 open_err_irq:
9769         bnxt_del_napi(bp);
9770
9771 open_err_free_mem:
9772         bnxt_free_skbs(bp);
9773         bnxt_free_irq(bp);
9774         bnxt_free_mem(bp, true);
9775         return rc;
9776 }
9777
9778 /* rtnl_lock held */
9779 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9780 {
9781         int rc = 0;
9782
9783         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
9784                 rc = -EIO;
9785         if (!rc)
9786                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9787         if (rc) {
9788                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9789                 dev_close(bp->dev);
9790         }
9791         return rc;
9792 }
9793
9794 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9795  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9796  * self tests.
9797  */
9798 int bnxt_half_open_nic(struct bnxt *bp)
9799 {
9800         int rc = 0;
9801
9802         rc = bnxt_alloc_mem(bp, false);
9803         if (rc) {
9804                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9805                 goto half_open_err;
9806         }
9807         rc = bnxt_init_nic(bp, false);
9808         if (rc) {
9809                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9810                 goto half_open_err;
9811         }
9812         return 0;
9813
9814 half_open_err:
9815         bnxt_free_skbs(bp);
9816         bnxt_free_mem(bp, false);
9817         dev_close(bp->dev);
9818         return rc;
9819 }
9820
9821 /* rtnl_lock held, this call can only be made after a previous successful
9822  * call to bnxt_half_open_nic().
9823  */
9824 void bnxt_half_close_nic(struct bnxt *bp)
9825 {
9826         bnxt_hwrm_resource_free(bp, false, false);
9827         bnxt_free_skbs(bp);
9828         bnxt_free_mem(bp, false);
9829 }
9830
9831 static void bnxt_reenable_sriov(struct bnxt *bp)
9832 {
9833         if (BNXT_PF(bp)) {
9834                 struct bnxt_pf_info *pf = &bp->pf;
9835                 int n = pf->active_vfs;
9836
9837                 if (n)
9838                         bnxt_cfg_hw_sriov(bp, &n, true);
9839         }
9840 }
9841
9842 static int bnxt_open(struct net_device *dev)
9843 {
9844         struct bnxt *bp = netdev_priv(dev);
9845         int rc;
9846
9847         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9848                 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9849                 return -ENODEV;
9850         }
9851
9852         rc = bnxt_hwrm_if_change(bp, true);
9853         if (rc)
9854                 return rc;
9855         rc = __bnxt_open_nic(bp, true, true);
9856         if (rc) {
9857                 bnxt_hwrm_if_change(bp, false);
9858         } else {
9859                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9860                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9861                                 bnxt_ulp_start(bp, 0);
9862                                 bnxt_reenable_sriov(bp);
9863                         }
9864                 }
9865                 bnxt_hwmon_open(bp);
9866         }
9867
9868         return rc;
9869 }
9870
9871 static bool bnxt_drv_busy(struct bnxt *bp)
9872 {
9873         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9874                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9875 }
9876
9877 static void bnxt_get_ring_stats(struct bnxt *bp,
9878                                 struct rtnl_link_stats64 *stats);
9879
9880 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9881                              bool link_re_init)
9882 {
9883         /* Close the VF-reps before closing PF */
9884         if (BNXT_PF(bp))
9885                 bnxt_vf_reps_close(bp);
9886
9887         /* Change device state to avoid TX queue wake up's */
9888         bnxt_tx_disable(bp);
9889
9890         clear_bit(BNXT_STATE_OPEN, &bp->state);
9891         smp_mb__after_atomic();
9892         while (bnxt_drv_busy(bp))
9893                 msleep(20);
9894
9895         /* Flush rings and and disable interrupts */
9896         bnxt_shutdown_nic(bp, irq_re_init);
9897
9898         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9899
9900         bnxt_debug_dev_exit(bp);
9901         bnxt_disable_napi(bp);
9902         del_timer_sync(&bp->timer);
9903         bnxt_free_skbs(bp);
9904
9905         /* Save ring stats before shutdown */
9906         if (bp->bnapi && irq_re_init)
9907                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9908         if (irq_re_init) {
9909                 bnxt_free_irq(bp);
9910                 bnxt_del_napi(bp);
9911         }
9912         bnxt_free_mem(bp, irq_re_init);
9913 }
9914
9915 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9916 {
9917         int rc = 0;
9918
9919         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9920                 /* If we get here, it means firmware reset is in progress
9921                  * while we are trying to close.  We can safely proceed with
9922                  * the close because we are holding rtnl_lock().  Some firmware
9923                  * messages may fail as we proceed to close.  We set the
9924                  * ABORT_ERR flag here so that the FW reset thread will later
9925                  * abort when it gets the rtnl_lock() and sees the flag.
9926                  */
9927                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9928                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9929         }
9930
9931 #ifdef CONFIG_BNXT_SRIOV
9932         if (bp->sriov_cfg) {
9933                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9934                                                       !bp->sriov_cfg,
9935                                                       BNXT_SRIOV_CFG_WAIT_TMO);
9936                 if (rc)
9937                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9938         }
9939 #endif
9940         __bnxt_close_nic(bp, irq_re_init, link_re_init);
9941         return rc;
9942 }
9943
9944 static int bnxt_close(struct net_device *dev)
9945 {
9946         struct bnxt *bp = netdev_priv(dev);
9947
9948         bnxt_hwmon_close(bp);
9949         bnxt_close_nic(bp, true, true);
9950         bnxt_hwrm_shutdown_link(bp);
9951         bnxt_hwrm_if_change(bp, false);
9952         return 0;
9953 }
9954
9955 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9956                                    u16 *val)
9957 {
9958         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9959         struct hwrm_port_phy_mdio_read_input req = {0};
9960         int rc;
9961
9962         if (bp->hwrm_spec_code < 0x10a00)
9963                 return -EOPNOTSUPP;
9964
9965         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9966         req.port_id = cpu_to_le16(bp->pf.port_id);
9967         req.phy_addr = phy_addr;
9968         req.reg_addr = cpu_to_le16(reg & 0x1f);
9969         if (mdio_phy_id_is_c45(phy_addr)) {
9970                 req.cl45_mdio = 1;
9971                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9972                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9973                 req.reg_addr = cpu_to_le16(reg);
9974         }
9975
9976         mutex_lock(&bp->hwrm_cmd_lock);
9977         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9978         if (!rc)
9979                 *val = le16_to_cpu(resp->reg_data);
9980         mutex_unlock(&bp->hwrm_cmd_lock);
9981         return rc;
9982 }
9983
9984 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9985                                     u16 val)
9986 {
9987         struct hwrm_port_phy_mdio_write_input req = {0};
9988
9989         if (bp->hwrm_spec_code < 0x10a00)
9990                 return -EOPNOTSUPP;
9991
9992         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9993         req.port_id = cpu_to_le16(bp->pf.port_id);
9994         req.phy_addr = phy_addr;
9995         req.reg_addr = cpu_to_le16(reg & 0x1f);
9996         if (mdio_phy_id_is_c45(phy_addr)) {
9997                 req.cl45_mdio = 1;
9998                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9999                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10000                 req.reg_addr = cpu_to_le16(reg);
10001         }
10002         req.reg_data = cpu_to_le16(val);
10003
10004         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10005 }
10006
10007 /* rtnl_lock held */
10008 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10009 {
10010         struct mii_ioctl_data *mdio = if_mii(ifr);
10011         struct bnxt *bp = netdev_priv(dev);
10012         int rc;
10013
10014         switch (cmd) {
10015         case SIOCGMIIPHY:
10016                 mdio->phy_id = bp->link_info.phy_addr;
10017
10018                 fallthrough;
10019         case SIOCGMIIREG: {
10020                 u16 mii_regval = 0;
10021
10022                 if (!netif_running(dev))
10023                         return -EAGAIN;
10024
10025                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10026                                              &mii_regval);
10027                 mdio->val_out = mii_regval;
10028                 return rc;
10029         }
10030
10031         case SIOCSMIIREG:
10032                 if (!netif_running(dev))
10033                         return -EAGAIN;
10034
10035                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10036                                                 mdio->val_in);
10037
10038         default:
10039                 /* do nothing */
10040                 break;
10041         }
10042         return -EOPNOTSUPP;
10043 }
10044
10045 static void bnxt_get_ring_stats(struct bnxt *bp,
10046                                 struct rtnl_link_stats64 *stats)
10047 {
10048         int i;
10049
10050         for (i = 0; i < bp->cp_nr_rings; i++) {
10051                 struct bnxt_napi *bnapi = bp->bnapi[i];
10052                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10053                 u64 *sw = cpr->stats.sw_stats;
10054
10055                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10056                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10057                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10058
10059                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10060                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10061                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10062
10063                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10064                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10065                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10066
10067                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10068                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10069                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10070
10071                 stats->rx_missed_errors +=
10072                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10073
10074                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10075
10076                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10077         }
10078 }
10079
10080 static void bnxt_add_prev_stats(struct bnxt *bp,
10081                                 struct rtnl_link_stats64 *stats)
10082 {
10083         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10084
10085         stats->rx_packets += prev_stats->rx_packets;
10086         stats->tx_packets += prev_stats->tx_packets;
10087         stats->rx_bytes += prev_stats->rx_bytes;
10088         stats->tx_bytes += prev_stats->tx_bytes;
10089         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10090         stats->multicast += prev_stats->multicast;
10091         stats->tx_dropped += prev_stats->tx_dropped;
10092 }
10093
10094 static void
10095 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10096 {
10097         struct bnxt *bp = netdev_priv(dev);
10098
10099         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10100         /* Make sure bnxt_close_nic() sees that we are reading stats before
10101          * we check the BNXT_STATE_OPEN flag.
10102          */
10103         smp_mb__after_atomic();
10104         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10105                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10106                 *stats = bp->net_stats_prev;
10107                 return;
10108         }
10109
10110         bnxt_get_ring_stats(bp, stats);
10111         bnxt_add_prev_stats(bp, stats);
10112
10113         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10114                 u64 *rx = bp->port_stats.sw_stats;
10115                 u64 *tx = bp->port_stats.sw_stats +
10116                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10117
10118                 stats->rx_crc_errors =
10119                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10120                 stats->rx_frame_errors =
10121                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10122                 stats->rx_length_errors =
10123                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10124                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10125                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10126                 stats->rx_errors =
10127                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10128                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10129                 stats->collisions =
10130                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10131                 stats->tx_fifo_errors =
10132                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10133                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10134         }
10135         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10136 }
10137
10138 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10139 {
10140         struct net_device *dev = bp->dev;
10141         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10142         struct netdev_hw_addr *ha;
10143         u8 *haddr;
10144         int mc_count = 0;
10145         bool update = false;
10146         int off = 0;
10147
10148         netdev_for_each_mc_addr(ha, dev) {
10149                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10150                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10151                         vnic->mc_list_count = 0;
10152                         return false;
10153                 }
10154                 haddr = ha->addr;
10155                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10156                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10157                         update = true;
10158                 }
10159                 off += ETH_ALEN;
10160                 mc_count++;
10161         }
10162         if (mc_count)
10163                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10164
10165         if (mc_count != vnic->mc_list_count) {
10166                 vnic->mc_list_count = mc_count;
10167                 update = true;
10168         }
10169         return update;
10170 }
10171
10172 static bool bnxt_uc_list_updated(struct bnxt *bp)
10173 {
10174         struct net_device *dev = bp->dev;
10175         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10176         struct netdev_hw_addr *ha;
10177         int off = 0;
10178
10179         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10180                 return true;
10181
10182         netdev_for_each_uc_addr(ha, dev) {
10183                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10184                         return true;
10185
10186                 off += ETH_ALEN;
10187         }
10188         return false;
10189 }
10190
10191 static void bnxt_set_rx_mode(struct net_device *dev)
10192 {
10193         struct bnxt *bp = netdev_priv(dev);
10194         struct bnxt_vnic_info *vnic;
10195         bool mc_update = false;
10196         bool uc_update;
10197         u32 mask;
10198
10199         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10200                 return;
10201
10202         vnic = &bp->vnic_info[0];
10203         mask = vnic->rx_mask;
10204         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10205                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10206                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10207                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10208
10209         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
10210                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10211
10212         uc_update = bnxt_uc_list_updated(bp);
10213
10214         if (dev->flags & IFF_BROADCAST)
10215                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10216         if (dev->flags & IFF_ALLMULTI) {
10217                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10218                 vnic->mc_list_count = 0;
10219         } else {
10220                 mc_update = bnxt_mc_list_updated(bp, &mask);
10221         }
10222
10223         if (mask != vnic->rx_mask || uc_update || mc_update) {
10224                 vnic->rx_mask = mask;
10225
10226                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10227                 bnxt_queue_sp_work(bp);
10228         }
10229 }
10230
10231 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10232 {
10233         struct net_device *dev = bp->dev;
10234         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10235         struct netdev_hw_addr *ha;
10236         int i, off = 0, rc;
10237         bool uc_update;
10238
10239         netif_addr_lock_bh(dev);
10240         uc_update = bnxt_uc_list_updated(bp);
10241         netif_addr_unlock_bh(dev);
10242
10243         if (!uc_update)
10244                 goto skip_uc;
10245
10246         mutex_lock(&bp->hwrm_cmd_lock);
10247         for (i = 1; i < vnic->uc_filter_count; i++) {
10248                 struct hwrm_cfa_l2_filter_free_input req = {0};
10249
10250                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10251                                        -1);
10252
10253                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10254
10255                 rc = _hwrm_send_message(bp, &req, sizeof(req),
10256                                         HWRM_CMD_TIMEOUT);
10257         }
10258         mutex_unlock(&bp->hwrm_cmd_lock);
10259
10260         vnic->uc_filter_count = 1;
10261
10262         netif_addr_lock_bh(dev);
10263         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10264                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10265         } else {
10266                 netdev_for_each_uc_addr(ha, dev) {
10267                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10268                         off += ETH_ALEN;
10269                         vnic->uc_filter_count++;
10270                 }
10271         }
10272         netif_addr_unlock_bh(dev);
10273
10274         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10275                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10276                 if (rc) {
10277                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10278                                    rc);
10279                         vnic->uc_filter_count = i;
10280                         return rc;
10281                 }
10282         }
10283
10284 skip_uc:
10285         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10286         if (rc && vnic->mc_list_count) {
10287                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10288                             rc);
10289                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10290                 vnic->mc_list_count = 0;
10291                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10292         }
10293         if (rc)
10294                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10295                            rc);
10296
10297         return rc;
10298 }
10299
10300 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10301 {
10302 #ifdef CONFIG_BNXT_SRIOV
10303         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10304                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10305
10306                 /* No minimum rings were provisioned by the PF.  Don't
10307                  * reserve rings by default when device is down.
10308                  */
10309                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10310                         return true;
10311
10312                 if (!netif_running(bp->dev))
10313                         return false;
10314         }
10315 #endif
10316         return true;
10317 }
10318
10319 /* If the chip and firmware supports RFS */
10320 static bool bnxt_rfs_supported(struct bnxt *bp)
10321 {
10322         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10323                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10324                         return true;
10325                 return false;
10326         }
10327         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10328                 return true;
10329         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10330                 return true;
10331         return false;
10332 }
10333
10334 /* If runtime conditions support RFS */
10335 static bool bnxt_rfs_capable(struct bnxt *bp)
10336 {
10337 #ifdef CONFIG_RFS_ACCEL
10338         int vnics, max_vnics, max_rss_ctxs;
10339
10340         if (bp->flags & BNXT_FLAG_CHIP_P5)
10341                 return bnxt_rfs_supported(bp);
10342         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10343                 return false;
10344
10345         vnics = 1 + bp->rx_nr_rings;
10346         max_vnics = bnxt_get_max_func_vnics(bp);
10347         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10348
10349         /* RSS contexts not a limiting factor */
10350         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10351                 max_rss_ctxs = max_vnics;
10352         if (vnics > max_vnics || vnics > max_rss_ctxs) {
10353                 if (bp->rx_nr_rings > 1)
10354                         netdev_warn(bp->dev,
10355                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10356                                     min(max_rss_ctxs - 1, max_vnics - 1));
10357                 return false;
10358         }
10359
10360         if (!BNXT_NEW_RM(bp))
10361                 return true;
10362
10363         if (vnics == bp->hw_resc.resv_vnics)
10364                 return true;
10365
10366         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10367         if (vnics <= bp->hw_resc.resv_vnics)
10368                 return true;
10369
10370         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10371         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10372         return false;
10373 #else
10374         return false;
10375 #endif
10376 }
10377
10378 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10379                                            netdev_features_t features)
10380 {
10381         struct bnxt *bp = netdev_priv(dev);
10382         netdev_features_t vlan_features;
10383
10384         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10385                 features &= ~NETIF_F_NTUPLE;
10386
10387         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10388                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10389
10390         if (!(features & NETIF_F_GRO))
10391                 features &= ~NETIF_F_GRO_HW;
10392
10393         if (features & NETIF_F_GRO_HW)
10394                 features &= ~NETIF_F_LRO;
10395
10396         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10397          * turned on or off together.
10398          */
10399         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10400         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10401                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10402                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10403                 else if (vlan_features)
10404                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10405         }
10406 #ifdef CONFIG_BNXT_SRIOV
10407         if (BNXT_VF(bp) && bp->vf.vlan)
10408                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10409 #endif
10410         return features;
10411 }
10412
10413 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10414 {
10415         struct bnxt *bp = netdev_priv(dev);
10416         u32 flags = bp->flags;
10417         u32 changes;
10418         int rc = 0;
10419         bool re_init = false;
10420         bool update_tpa = false;
10421
10422         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10423         if (features & NETIF_F_GRO_HW)
10424                 flags |= BNXT_FLAG_GRO;
10425         else if (features & NETIF_F_LRO)
10426                 flags |= BNXT_FLAG_LRO;
10427
10428         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10429                 flags &= ~BNXT_FLAG_TPA;
10430
10431         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10432                 flags |= BNXT_FLAG_STRIP_VLAN;
10433
10434         if (features & NETIF_F_NTUPLE)
10435                 flags |= BNXT_FLAG_RFS;
10436
10437         changes = flags ^ bp->flags;
10438         if (changes & BNXT_FLAG_TPA) {
10439                 update_tpa = true;
10440                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10441                     (flags & BNXT_FLAG_TPA) == 0 ||
10442                     (bp->flags & BNXT_FLAG_CHIP_P5))
10443                         re_init = true;
10444         }
10445
10446         if (changes & ~BNXT_FLAG_TPA)
10447                 re_init = true;
10448
10449         if (flags != bp->flags) {
10450                 u32 old_flags = bp->flags;
10451
10452                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10453                         bp->flags = flags;
10454                         if (update_tpa)
10455                                 bnxt_set_ring_params(bp);
10456                         return rc;
10457                 }
10458
10459                 if (re_init) {
10460                         bnxt_close_nic(bp, false, false);
10461                         bp->flags = flags;
10462                         if (update_tpa)
10463                                 bnxt_set_ring_params(bp);
10464
10465                         return bnxt_open_nic(bp, false, false);
10466                 }
10467                 if (update_tpa) {
10468                         bp->flags = flags;
10469                         rc = bnxt_set_tpa(bp,
10470                                           (flags & BNXT_FLAG_TPA) ?
10471                                           true : false);
10472                         if (rc)
10473                                 bp->flags = old_flags;
10474                 }
10475         }
10476         return rc;
10477 }
10478
10479 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10480                          u32 *reg_buf)
10481 {
10482         struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10483         struct hwrm_dbg_read_direct_input req = {0};
10484         __le32 *dbg_reg_buf;
10485         dma_addr_t mapping;
10486         int rc, i;
10487
10488         dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10489                                          &mapping, GFP_KERNEL);
10490         if (!dbg_reg_buf)
10491                 return -ENOMEM;
10492         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10493         req.host_dest_addr = cpu_to_le64(mapping);
10494         req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10495         req.read_len32 = cpu_to_le32(num_words);
10496         mutex_lock(&bp->hwrm_cmd_lock);
10497         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10498         if (rc || resp->error_code) {
10499                 rc = -EIO;
10500                 goto dbg_rd_reg_exit;
10501         }
10502         for (i = 0; i < num_words; i++)
10503                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10504
10505 dbg_rd_reg_exit:
10506         mutex_unlock(&bp->hwrm_cmd_lock);
10507         dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10508         return rc;
10509 }
10510
10511 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10512                                        u32 ring_id, u32 *prod, u32 *cons)
10513 {
10514         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10515         struct hwrm_dbg_ring_info_get_input req = {0};
10516         int rc;
10517
10518         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10519         req.ring_type = ring_type;
10520         req.fw_ring_id = cpu_to_le32(ring_id);
10521         mutex_lock(&bp->hwrm_cmd_lock);
10522         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10523         if (!rc) {
10524                 *prod = le32_to_cpu(resp->producer_index);
10525                 *cons = le32_to_cpu(resp->consumer_index);
10526         }
10527         mutex_unlock(&bp->hwrm_cmd_lock);
10528         return rc;
10529 }
10530
10531 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10532 {
10533         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10534         int i = bnapi->index;
10535
10536         if (!txr)
10537                 return;
10538
10539         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10540                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10541                     txr->tx_cons);
10542 }
10543
10544 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10545 {
10546         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10547         int i = bnapi->index;
10548
10549         if (!rxr)
10550                 return;
10551
10552         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10553                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10554                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10555                     rxr->rx_sw_agg_prod);
10556 }
10557
10558 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10559 {
10560         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10561         int i = bnapi->index;
10562
10563         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10564                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10565 }
10566
10567 static void bnxt_dbg_dump_states(struct bnxt *bp)
10568 {
10569         int i;
10570         struct bnxt_napi *bnapi;
10571
10572         for (i = 0; i < bp->cp_nr_rings; i++) {
10573                 bnapi = bp->bnapi[i];
10574                 if (netif_msg_drv(bp)) {
10575                         bnxt_dump_tx_sw_state(bnapi);
10576                         bnxt_dump_rx_sw_state(bnapi);
10577                         bnxt_dump_cp_sw_state(bnapi);
10578                 }
10579         }
10580 }
10581
10582 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
10583 {
10584         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
10585         struct hwrm_ring_reset_input req = {0};
10586         struct bnxt_napi *bnapi = rxr->bnapi;
10587         struct bnxt_cp_ring_info *cpr;
10588         u16 cp_ring_id;
10589
10590         cpr = &bnapi->cp_ring;
10591         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
10592         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1);
10593         req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
10594         req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
10595         return hwrm_send_message_silent(bp, &req, sizeof(req),
10596                                         HWRM_CMD_TIMEOUT);
10597 }
10598
10599 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10600 {
10601         if (!silent)
10602                 bnxt_dbg_dump_states(bp);
10603         if (netif_running(bp->dev)) {
10604                 int rc;
10605
10606                 if (silent) {
10607                         bnxt_close_nic(bp, false, false);
10608                         bnxt_open_nic(bp, false, false);
10609                 } else {
10610                         bnxt_ulp_stop(bp);
10611                         bnxt_close_nic(bp, true, false);
10612                         rc = bnxt_open_nic(bp, true, false);
10613                         bnxt_ulp_start(bp, rc);
10614                 }
10615         }
10616 }
10617
10618 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10619 {
10620         struct bnxt *bp = netdev_priv(dev);
10621
10622         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10623         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10624         bnxt_queue_sp_work(bp);
10625 }
10626
10627 static void bnxt_fw_health_check(struct bnxt *bp)
10628 {
10629         struct bnxt_fw_health *fw_health = bp->fw_health;
10630         u32 val;
10631
10632         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10633                 return;
10634
10635         if (fw_health->tmr_counter) {
10636                 fw_health->tmr_counter--;
10637                 return;
10638         }
10639
10640         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10641         if (val == fw_health->last_fw_heartbeat)
10642                 goto fw_reset;
10643
10644         fw_health->last_fw_heartbeat = val;
10645
10646         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10647         if (val != fw_health->last_fw_reset_cnt)
10648                 goto fw_reset;
10649
10650         fw_health->tmr_counter = fw_health->tmr_multiplier;
10651         return;
10652
10653 fw_reset:
10654         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10655         bnxt_queue_sp_work(bp);
10656 }
10657
10658 static void bnxt_timer(struct timer_list *t)
10659 {
10660         struct bnxt *bp = from_timer(bp, t, timer);
10661         struct net_device *dev = bp->dev;
10662
10663         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10664                 return;
10665
10666         if (atomic_read(&bp->intr_sem) != 0)
10667                 goto bnxt_restart_timer;
10668
10669         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10670                 bnxt_fw_health_check(bp);
10671
10672         if (bp->link_info.link_up && bp->stats_coal_ticks) {
10673                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10674                 bnxt_queue_sp_work(bp);
10675         }
10676
10677         if (bnxt_tc_flower_enabled(bp)) {
10678                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10679                 bnxt_queue_sp_work(bp);
10680         }
10681
10682 #ifdef CONFIG_RFS_ACCEL
10683         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10684                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10685                 bnxt_queue_sp_work(bp);
10686         }
10687 #endif /*CONFIG_RFS_ACCEL*/
10688
10689         if (bp->link_info.phy_retry) {
10690                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10691                         bp->link_info.phy_retry = false;
10692                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10693                 } else {
10694                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10695                         bnxt_queue_sp_work(bp);
10696                 }
10697         }
10698
10699         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10700             netif_carrier_ok(dev)) {
10701                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10702                 bnxt_queue_sp_work(bp);
10703         }
10704 bnxt_restart_timer:
10705         mod_timer(&bp->timer, jiffies + bp->current_interval);
10706 }
10707
10708 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10709 {
10710         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10711          * set.  If the device is being closed, bnxt_close() may be holding
10712          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10713          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10714          */
10715         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10716         rtnl_lock();
10717 }
10718
10719 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10720 {
10721         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10722         rtnl_unlock();
10723 }
10724
10725 /* Only called from bnxt_sp_task() */
10726 static void bnxt_reset(struct bnxt *bp, bool silent)
10727 {
10728         bnxt_rtnl_lock_sp(bp);
10729         if (test_bit(BNXT_STATE_OPEN, &bp->state))
10730                 bnxt_reset_task(bp, silent);
10731         bnxt_rtnl_unlock_sp(bp);
10732 }
10733
10734 /* Only called from bnxt_sp_task() */
10735 static void bnxt_rx_ring_reset(struct bnxt *bp)
10736 {
10737         int i;
10738
10739         bnxt_rtnl_lock_sp(bp);
10740         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10741                 bnxt_rtnl_unlock_sp(bp);
10742                 return;
10743         }
10744         /* Disable and flush TPA before resetting the RX ring */
10745         if (bp->flags & BNXT_FLAG_TPA)
10746                 bnxt_set_tpa(bp, false);
10747         for (i = 0; i < bp->rx_nr_rings; i++) {
10748                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
10749                 struct bnxt_cp_ring_info *cpr;
10750                 int rc;
10751
10752                 if (!rxr->bnapi->in_reset)
10753                         continue;
10754
10755                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
10756                 if (rc) {
10757                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
10758                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
10759                         else
10760                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
10761                                             rc);
10762                         bnxt_reset_task(bp, true);
10763                         break;
10764                 }
10765                 bnxt_free_one_rx_ring_skbs(bp, i);
10766                 rxr->rx_prod = 0;
10767                 rxr->rx_agg_prod = 0;
10768                 rxr->rx_sw_agg_prod = 0;
10769                 rxr->rx_next_cons = 0;
10770                 rxr->bnapi->in_reset = false;
10771                 bnxt_alloc_one_rx_ring(bp, i);
10772                 cpr = &rxr->bnapi->cp_ring;
10773                 cpr->sw_stats.rx.rx_resets++;
10774                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10775                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10776                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10777         }
10778         if (bp->flags & BNXT_FLAG_TPA)
10779                 bnxt_set_tpa(bp, true);
10780         bnxt_rtnl_unlock_sp(bp);
10781 }
10782
10783 static void bnxt_fw_reset_close(struct bnxt *bp)
10784 {
10785         bnxt_ulp_stop(bp);
10786         /* When firmware is fatal state, disable PCI device to prevent
10787          * any potential bad DMAs before freeing kernel memory.
10788          */
10789         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10790                 pci_disable_device(bp->pdev);
10791         __bnxt_close_nic(bp, true, false);
10792         bnxt_clear_int_mode(bp);
10793         bnxt_hwrm_func_drv_unrgtr(bp);
10794         if (pci_is_enabled(bp->pdev))
10795                 pci_disable_device(bp->pdev);
10796         bnxt_free_ctx_mem(bp);
10797         kfree(bp->ctx);
10798         bp->ctx = NULL;
10799 }
10800
10801 static bool is_bnxt_fw_ok(struct bnxt *bp)
10802 {
10803         struct bnxt_fw_health *fw_health = bp->fw_health;
10804         bool no_heartbeat = false, has_reset = false;
10805         u32 val;
10806
10807         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10808         if (val == fw_health->last_fw_heartbeat)
10809                 no_heartbeat = true;
10810
10811         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10812         if (val != fw_health->last_fw_reset_cnt)
10813                 has_reset = true;
10814
10815         if (!no_heartbeat && has_reset)
10816                 return true;
10817
10818         return false;
10819 }
10820
10821 /* rtnl_lock is acquired before calling this function */
10822 static void bnxt_force_fw_reset(struct bnxt *bp)
10823 {
10824         struct bnxt_fw_health *fw_health = bp->fw_health;
10825         u32 wait_dsecs;
10826
10827         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10828             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10829                 return;
10830
10831         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10832         bnxt_fw_reset_close(bp);
10833         wait_dsecs = fw_health->master_func_wait_dsecs;
10834         if (fw_health->master) {
10835                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10836                         wait_dsecs = 0;
10837                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10838         } else {
10839                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10840                 wait_dsecs = fw_health->normal_func_wait_dsecs;
10841                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10842         }
10843
10844         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10845         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10846         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10847 }
10848
10849 void bnxt_fw_exception(struct bnxt *bp)
10850 {
10851         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10852         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10853         bnxt_rtnl_lock_sp(bp);
10854         bnxt_force_fw_reset(bp);
10855         bnxt_rtnl_unlock_sp(bp);
10856 }
10857
10858 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10859  * < 0 on error.
10860  */
10861 static int bnxt_get_registered_vfs(struct bnxt *bp)
10862 {
10863 #ifdef CONFIG_BNXT_SRIOV
10864         int rc;
10865
10866         if (!BNXT_PF(bp))
10867                 return 0;
10868
10869         rc = bnxt_hwrm_func_qcfg(bp);
10870         if (rc) {
10871                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10872                 return rc;
10873         }
10874         if (bp->pf.registered_vfs)
10875                 return bp->pf.registered_vfs;
10876         if (bp->sriov_cfg)
10877                 return 1;
10878 #endif
10879         return 0;
10880 }
10881
10882 void bnxt_fw_reset(struct bnxt *bp)
10883 {
10884         bnxt_rtnl_lock_sp(bp);
10885         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10886             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10887                 int n = 0, tmo;
10888
10889                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10890                 if (bp->pf.active_vfs &&
10891                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10892                         n = bnxt_get_registered_vfs(bp);
10893                 if (n < 0) {
10894                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10895                                    n);
10896                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10897                         dev_close(bp->dev);
10898                         goto fw_reset_exit;
10899                 } else if (n > 0) {
10900                         u16 vf_tmo_dsecs = n * 10;
10901
10902                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10903                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10904                         bp->fw_reset_state =
10905                                 BNXT_FW_RESET_STATE_POLL_VF;
10906                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10907                         goto fw_reset_exit;
10908                 }
10909                 bnxt_fw_reset_close(bp);
10910                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10911                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10912                         tmo = HZ / 10;
10913                 } else {
10914                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10915                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
10916                 }
10917                 bnxt_queue_fw_reset_work(bp, tmo);
10918         }
10919 fw_reset_exit:
10920         bnxt_rtnl_unlock_sp(bp);
10921 }
10922
10923 static void bnxt_chk_missed_irq(struct bnxt *bp)
10924 {
10925         int i;
10926
10927         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10928                 return;
10929
10930         for (i = 0; i < bp->cp_nr_rings; i++) {
10931                 struct bnxt_napi *bnapi = bp->bnapi[i];
10932                 struct bnxt_cp_ring_info *cpr;
10933                 u32 fw_ring_id;
10934                 int j;
10935
10936                 if (!bnapi)
10937                         continue;
10938
10939                 cpr = &bnapi->cp_ring;
10940                 for (j = 0; j < 2; j++) {
10941                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10942                         u32 val[2];
10943
10944                         if (!cpr2 || cpr2->has_more_work ||
10945                             !bnxt_has_work(bp, cpr2))
10946                                 continue;
10947
10948                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10949                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10950                                 continue;
10951                         }
10952                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10953                         bnxt_dbg_hwrm_ring_info_get(bp,
10954                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10955                                 fw_ring_id, &val[0], &val[1]);
10956                         cpr->sw_stats.cmn.missed_irqs++;
10957                 }
10958         }
10959 }
10960
10961 static void bnxt_cfg_ntp_filters(struct bnxt *);
10962
10963 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10964 {
10965         struct bnxt_link_info *link_info = &bp->link_info;
10966
10967         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10968                 link_info->autoneg = BNXT_AUTONEG_SPEED;
10969                 if (bp->hwrm_spec_code >= 0x10201) {
10970                         if (link_info->auto_pause_setting &
10971                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10972                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10973                 } else {
10974                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10975                 }
10976                 link_info->advertising = link_info->auto_link_speeds;
10977                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
10978         } else {
10979                 link_info->req_link_speed = link_info->force_link_speed;
10980                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
10981                 if (link_info->force_pam4_link_speed) {
10982                         link_info->req_link_speed =
10983                                 link_info->force_pam4_link_speed;
10984                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
10985                 }
10986                 link_info->req_duplex = link_info->duplex_setting;
10987         }
10988         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10989                 link_info->req_flow_ctrl =
10990                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10991         else
10992                 link_info->req_flow_ctrl = link_info->force_pause_setting;
10993 }
10994
10995 static void bnxt_sp_task(struct work_struct *work)
10996 {
10997         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10998
10999         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11000         smp_mb__after_atomic();
11001         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11002                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11003                 return;
11004         }
11005
11006         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11007                 bnxt_cfg_rx_mode(bp);
11008
11009         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11010                 bnxt_cfg_ntp_filters(bp);
11011         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11012                 bnxt_hwrm_exec_fwd_req(bp);
11013         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
11014                 bnxt_hwrm_port_qstats(bp, 0);
11015                 bnxt_hwrm_port_qstats_ext(bp, 0);
11016                 bnxt_accumulate_all_stats(bp);
11017         }
11018
11019         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
11020                 int rc;
11021
11022                 mutex_lock(&bp->link_lock);
11023                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
11024                                        &bp->sp_event))
11025                         bnxt_hwrm_phy_qcaps(bp);
11026
11027                 rc = bnxt_update_link(bp, true);
11028                 if (rc)
11029                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
11030                                    rc);
11031
11032                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
11033                                        &bp->sp_event))
11034                         bnxt_init_ethtool_link_settings(bp);
11035                 mutex_unlock(&bp->link_lock);
11036         }
11037         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
11038                 int rc;
11039
11040                 mutex_lock(&bp->link_lock);
11041                 rc = bnxt_update_phy_setting(bp);
11042                 mutex_unlock(&bp->link_lock);
11043                 if (rc) {
11044                         netdev_warn(bp->dev, "update phy settings retry failed\n");
11045                 } else {
11046                         bp->link_info.phy_retry = false;
11047                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
11048                 }
11049         }
11050         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
11051                 mutex_lock(&bp->link_lock);
11052                 bnxt_get_port_module_status(bp);
11053                 mutex_unlock(&bp->link_lock);
11054         }
11055
11056         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
11057                 bnxt_tc_flow_stats_work(bp);
11058
11059         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
11060                 bnxt_chk_missed_irq(bp);
11061
11062         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
11063          * must be the last functions to be called before exiting.
11064          */
11065         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
11066                 bnxt_reset(bp, false);
11067
11068         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
11069                 bnxt_reset(bp, true);
11070
11071         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
11072                 bnxt_rx_ring_reset(bp);
11073
11074         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
11075                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
11076
11077         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
11078                 if (!is_bnxt_fw_ok(bp))
11079                         bnxt_devlink_health_report(bp,
11080                                                    BNXT_FW_EXCEPTION_SP_EVENT);
11081         }
11082
11083         smp_mb__before_atomic();
11084         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11085 }
11086
11087 /* Under rtnl_lock */
11088 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
11089                      int tx_xdp)
11090 {
11091         int max_rx, max_tx, tx_sets = 1;
11092         int tx_rings_needed, stats;
11093         int rx_rings = rx;
11094         int cp, vnics, rc;
11095
11096         if (tcs)
11097                 tx_sets = tcs;
11098
11099         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
11100         if (rc)
11101                 return rc;
11102
11103         if (max_rx < rx)
11104                 return -ENOMEM;
11105
11106         tx_rings_needed = tx * tx_sets + tx_xdp;
11107         if (max_tx < tx_rings_needed)
11108                 return -ENOMEM;
11109
11110         vnics = 1;
11111         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
11112                 vnics += rx_rings;
11113
11114         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11115                 rx_rings <<= 1;
11116         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
11117         stats = cp;
11118         if (BNXT_NEW_RM(bp)) {
11119                 cp += bnxt_get_ulp_msix_num(bp);
11120                 stats += bnxt_get_ulp_stat_ctxs(bp);
11121         }
11122         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
11123                                      stats, vnics);
11124 }
11125
11126 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
11127 {
11128         if (bp->bar2) {
11129                 pci_iounmap(pdev, bp->bar2);
11130                 bp->bar2 = NULL;
11131         }
11132
11133         if (bp->bar1) {
11134                 pci_iounmap(pdev, bp->bar1);
11135                 bp->bar1 = NULL;
11136         }
11137
11138         if (bp->bar0) {
11139                 pci_iounmap(pdev, bp->bar0);
11140                 bp->bar0 = NULL;
11141         }
11142 }
11143
11144 static void bnxt_cleanup_pci(struct bnxt *bp)
11145 {
11146         bnxt_unmap_bars(bp, bp->pdev);
11147         pci_release_regions(bp->pdev);
11148         if (pci_is_enabled(bp->pdev))
11149                 pci_disable_device(bp->pdev);
11150 }
11151
11152 static void bnxt_init_dflt_coal(struct bnxt *bp)
11153 {
11154         struct bnxt_coal *coal;
11155
11156         /* Tick values in micro seconds.
11157          * 1 coal_buf x bufs_per_record = 1 completion record.
11158          */
11159         coal = &bp->rx_coal;
11160         coal->coal_ticks = 10;
11161         coal->coal_bufs = 30;
11162         coal->coal_ticks_irq = 1;
11163         coal->coal_bufs_irq = 2;
11164         coal->idle_thresh = 50;
11165         coal->bufs_per_record = 2;
11166         coal->budget = 64;              /* NAPI budget */
11167
11168         coal = &bp->tx_coal;
11169         coal->coal_ticks = 28;
11170         coal->coal_bufs = 30;
11171         coal->coal_ticks_irq = 2;
11172         coal->coal_bufs_irq = 2;
11173         coal->bufs_per_record = 1;
11174
11175         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
11176 }
11177
11178 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11179 {
11180 #ifdef CONFIG_TEE_BNXT_FW
11181         int rc = tee_bnxt_fw_load();
11182
11183         if (rc)
11184                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11185
11186         return rc;
11187 #else
11188         netdev_err(bp->dev, "OP-TEE not supported\n");
11189         return -ENODEV;
11190 #endif
11191 }
11192
11193 static int bnxt_fw_init_one_p1(struct bnxt *bp)
11194 {
11195         int rc;
11196
11197         bp->fw_cap = 0;
11198         rc = bnxt_hwrm_ver_get(bp);
11199         bnxt_try_map_fw_health_reg(bp);
11200         if (rc) {
11201                 if (bp->fw_health && bp->fw_health->status_reliable) {
11202                         u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11203
11204                         netdev_err(bp->dev,
11205                                    "Firmware not responding, status: 0x%x\n",
11206                                    sts);
11207                         if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11208                                 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11209                                 rc = bnxt_fw_reset_via_optee(bp);
11210                                 if (!rc)
11211                                         rc = bnxt_hwrm_ver_get(bp);
11212                         }
11213                 }
11214                 if (rc)
11215                         return rc;
11216         }
11217
11218         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
11219                 rc = bnxt_alloc_kong_hwrm_resources(bp);
11220                 if (rc)
11221                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
11222         }
11223
11224         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
11225             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
11226                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
11227                 if (rc)
11228                         return rc;
11229         }
11230         bnxt_nvm_cfg_ver_get(bp);
11231
11232         rc = bnxt_hwrm_func_reset(bp);
11233         if (rc)
11234                 return -ENODEV;
11235
11236         bnxt_hwrm_fw_set_time(bp);
11237         return 0;
11238 }
11239
11240 static int bnxt_fw_init_one_p2(struct bnxt *bp)
11241 {
11242         int rc;
11243
11244         /* Get the MAX capabilities for this function */
11245         rc = bnxt_hwrm_func_qcaps(bp);
11246         if (rc) {
11247                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
11248                            rc);
11249                 return -ENODEV;
11250         }
11251
11252         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11253         if (rc)
11254                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11255                             rc);
11256
11257         if (bnxt_alloc_fw_health(bp)) {
11258                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
11259         } else {
11260                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
11261                 if (rc)
11262                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11263                                     rc);
11264         }
11265
11266         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11267         if (rc)
11268                 return -ENODEV;
11269
11270         bnxt_hwrm_func_qcfg(bp);
11271         bnxt_hwrm_vnic_qcaps(bp);
11272         bnxt_hwrm_port_led_qcaps(bp);
11273         bnxt_ethtool_init(bp);
11274         bnxt_dcb_init(bp);
11275         return 0;
11276 }
11277
11278 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11279 {
11280         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11281         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11282                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11283                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11284                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11285         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11286                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11287                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11288                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11289         }
11290 }
11291
11292 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11293 {
11294         struct net_device *dev = bp->dev;
11295
11296         dev->hw_features &= ~NETIF_F_NTUPLE;
11297         dev->features &= ~NETIF_F_NTUPLE;
11298         bp->flags &= ~BNXT_FLAG_RFS;
11299         if (bnxt_rfs_supported(bp)) {
11300                 dev->hw_features |= NETIF_F_NTUPLE;
11301                 if (bnxt_rfs_capable(bp)) {
11302                         bp->flags |= BNXT_FLAG_RFS;
11303                         dev->features |= NETIF_F_NTUPLE;
11304                 }
11305         }
11306 }
11307
11308 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11309 {
11310         struct pci_dev *pdev = bp->pdev;
11311
11312         bnxt_set_dflt_rss_hash_type(bp);
11313         bnxt_set_dflt_rfs(bp);
11314
11315         bnxt_get_wol_settings(bp);
11316         if (bp->flags & BNXT_FLAG_WOL_CAP)
11317                 device_set_wakeup_enable(&pdev->dev, bp->wol);
11318         else
11319                 device_set_wakeup_capable(&pdev->dev, false);
11320
11321         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11322         bnxt_hwrm_coal_params_qcaps(bp);
11323 }
11324
11325 static int bnxt_fw_init_one(struct bnxt *bp)
11326 {
11327         int rc;
11328
11329         rc = bnxt_fw_init_one_p1(bp);
11330         if (rc) {
11331                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11332                 return rc;
11333         }
11334         rc = bnxt_fw_init_one_p2(bp);
11335         if (rc) {
11336                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11337                 return rc;
11338         }
11339         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11340         if (rc)
11341                 return rc;
11342
11343         /* In case fw capabilities have changed, destroy the unneeded
11344          * reporters and create newly capable ones.
11345          */
11346         bnxt_dl_fw_reporters_destroy(bp, false);
11347         bnxt_dl_fw_reporters_create(bp);
11348         bnxt_fw_init_one_p3(bp);
11349         return 0;
11350 }
11351
11352 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11353 {
11354         struct bnxt_fw_health *fw_health = bp->fw_health;
11355         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11356         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11357         u32 reg_type, reg_off, delay_msecs;
11358
11359         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11360         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11361         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11362         switch (reg_type) {
11363         case BNXT_FW_HEALTH_REG_TYPE_CFG:
11364                 pci_write_config_dword(bp->pdev, reg_off, val);
11365                 break;
11366         case BNXT_FW_HEALTH_REG_TYPE_GRC:
11367                 writel(reg_off & BNXT_GRC_BASE_MASK,
11368                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11369                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11370                 fallthrough;
11371         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11372                 writel(val, bp->bar0 + reg_off);
11373                 break;
11374         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11375                 writel(val, bp->bar1 + reg_off);
11376                 break;
11377         }
11378         if (delay_msecs) {
11379                 pci_read_config_dword(bp->pdev, 0, &val);
11380                 msleep(delay_msecs);
11381         }
11382 }
11383
11384 static void bnxt_reset_all(struct bnxt *bp)
11385 {
11386         struct bnxt_fw_health *fw_health = bp->fw_health;
11387         int i, rc;
11388
11389         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11390                 bnxt_fw_reset_via_optee(bp);
11391                 bp->fw_reset_timestamp = jiffies;
11392                 return;
11393         }
11394
11395         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11396                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11397                         bnxt_fw_reset_writel(bp, i);
11398         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11399                 struct hwrm_fw_reset_input req = {0};
11400
11401                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11402                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11403                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11404                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11405                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11406                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11407                 if (rc)
11408                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11409         }
11410         bp->fw_reset_timestamp = jiffies;
11411 }
11412
11413 static void bnxt_fw_reset_task(struct work_struct *work)
11414 {
11415         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11416         int rc;
11417
11418         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11419                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11420                 return;
11421         }
11422
11423         switch (bp->fw_reset_state) {
11424         case BNXT_FW_RESET_STATE_POLL_VF: {
11425                 int n = bnxt_get_registered_vfs(bp);
11426                 int tmo;
11427
11428                 if (n < 0) {
11429                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11430                                    n, jiffies_to_msecs(jiffies -
11431                                    bp->fw_reset_timestamp));
11432                         goto fw_reset_abort;
11433                 } else if (n > 0) {
11434                         if (time_after(jiffies, bp->fw_reset_timestamp +
11435                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11436                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11437                                 bp->fw_reset_state = 0;
11438                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11439                                            n);
11440                                 return;
11441                         }
11442                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11443                         return;
11444                 }
11445                 bp->fw_reset_timestamp = jiffies;
11446                 rtnl_lock();
11447                 bnxt_fw_reset_close(bp);
11448                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11449                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11450                         tmo = HZ / 10;
11451                 } else {
11452                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11453                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11454                 }
11455                 rtnl_unlock();
11456                 bnxt_queue_fw_reset_work(bp, tmo);
11457                 return;
11458         }
11459         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11460                 u32 val;
11461
11462                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11463                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11464                     !time_after(jiffies, bp->fw_reset_timestamp +
11465                     (bp->fw_reset_max_dsecs * HZ / 10))) {
11466                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11467                         return;
11468                 }
11469
11470                 if (!bp->fw_health->master) {
11471                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11472
11473                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11474                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11475                         return;
11476                 }
11477                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11478         }
11479                 fallthrough;
11480         case BNXT_FW_RESET_STATE_RESET_FW:
11481                 bnxt_reset_all(bp);
11482                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11483                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11484                 return;
11485         case BNXT_FW_RESET_STATE_ENABLE_DEV:
11486                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11487                         u32 val;
11488
11489                         val = bnxt_fw_health_readl(bp,
11490                                                    BNXT_FW_RESET_INPROG_REG);
11491                         if (val)
11492                                 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11493                                             val);
11494                 }
11495                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11496                 if (pci_enable_device(bp->pdev)) {
11497                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11498                         goto fw_reset_abort;
11499                 }
11500                 pci_set_master(bp->pdev);
11501                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11502                 fallthrough;
11503         case BNXT_FW_RESET_STATE_POLL_FW:
11504                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11505                 rc = __bnxt_hwrm_ver_get(bp, true);
11506                 if (rc) {
11507                         if (time_after(jiffies, bp->fw_reset_timestamp +
11508                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11509                                 netdev_err(bp->dev, "Firmware reset aborted\n");
11510                                 goto fw_reset_abort_status;
11511                         }
11512                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11513                         return;
11514                 }
11515                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11516                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11517                 fallthrough;
11518         case BNXT_FW_RESET_STATE_OPENING:
11519                 while (!rtnl_trylock()) {
11520                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11521                         return;
11522                 }
11523                 rc = bnxt_open(bp->dev);
11524                 if (rc) {
11525                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11526                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11527                         dev_close(bp->dev);
11528                 }
11529
11530                 bp->fw_reset_state = 0;
11531                 /* Make sure fw_reset_state is 0 before clearing the flag */
11532                 smp_mb__before_atomic();
11533                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11534                 bnxt_ulp_start(bp, rc);
11535                 if (!rc)
11536                         bnxt_reenable_sriov(bp);
11537                 bnxt_dl_health_recovery_done(bp);
11538                 bnxt_dl_health_status_update(bp, true);
11539                 rtnl_unlock();
11540                 break;
11541         }
11542         return;
11543
11544 fw_reset_abort_status:
11545         if (bp->fw_health->status_reliable ||
11546             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
11547                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11548
11549                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
11550         }
11551 fw_reset_abort:
11552         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11553         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11554                 bnxt_dl_health_status_update(bp, false);
11555         bp->fw_reset_state = 0;
11556         rtnl_lock();
11557         dev_close(bp->dev);
11558         rtnl_unlock();
11559 }
11560
11561 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11562 {
11563         int rc;
11564         struct bnxt *bp = netdev_priv(dev);
11565
11566         SET_NETDEV_DEV(dev, &pdev->dev);
11567
11568         /* enable device (incl. PCI PM wakeup), and bus-mastering */
11569         rc = pci_enable_device(pdev);
11570         if (rc) {
11571                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11572                 goto init_err;
11573         }
11574
11575         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11576                 dev_err(&pdev->dev,
11577                         "Cannot find PCI device base address, aborting\n");
11578                 rc = -ENODEV;
11579                 goto init_err_disable;
11580         }
11581
11582         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11583         if (rc) {
11584                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11585                 goto init_err_disable;
11586         }
11587
11588         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11589             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11590                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11591                 goto init_err_disable;
11592         }
11593
11594         pci_set_master(pdev);
11595
11596         bp->dev = dev;
11597         bp->pdev = pdev;
11598
11599         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11600          * determines the BAR size.
11601          */
11602         bp->bar0 = pci_ioremap_bar(pdev, 0);
11603         if (!bp->bar0) {
11604                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11605                 rc = -ENOMEM;
11606                 goto init_err_release;
11607         }
11608
11609         bp->bar2 = pci_ioremap_bar(pdev, 4);
11610         if (!bp->bar2) {
11611                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11612                 rc = -ENOMEM;
11613                 goto init_err_release;
11614         }
11615
11616         pci_enable_pcie_error_reporting(pdev);
11617
11618         INIT_WORK(&bp->sp_task, bnxt_sp_task);
11619         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11620
11621         spin_lock_init(&bp->ntp_fltr_lock);
11622 #if BITS_PER_LONG == 32
11623         spin_lock_init(&bp->db_lock);
11624 #endif
11625
11626         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11627         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11628
11629         bnxt_init_dflt_coal(bp);
11630
11631         timer_setup(&bp->timer, bnxt_timer, 0);
11632         bp->current_interval = BNXT_TIMER_INTERVAL;
11633
11634         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11635         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11636
11637         clear_bit(BNXT_STATE_OPEN, &bp->state);
11638         return 0;
11639
11640 init_err_release:
11641         bnxt_unmap_bars(bp, pdev);
11642         pci_release_regions(pdev);
11643
11644 init_err_disable:
11645         pci_disable_device(pdev);
11646
11647 init_err:
11648         return rc;
11649 }
11650
11651 /* rtnl_lock held */
11652 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11653 {
11654         struct sockaddr *addr = p;
11655         struct bnxt *bp = netdev_priv(dev);
11656         int rc = 0;
11657
11658         if (!is_valid_ether_addr(addr->sa_data))
11659                 return -EADDRNOTAVAIL;
11660
11661         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11662                 return 0;
11663
11664         rc = bnxt_approve_mac(bp, addr->sa_data, true);
11665         if (rc)
11666                 return rc;
11667
11668         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11669         if (netif_running(dev)) {
11670                 bnxt_close_nic(bp, false, false);
11671                 rc = bnxt_open_nic(bp, false, false);
11672         }
11673
11674         return rc;
11675 }
11676
11677 /* rtnl_lock held */
11678 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11679 {
11680         struct bnxt *bp = netdev_priv(dev);
11681
11682         if (netif_running(dev))
11683                 bnxt_close_nic(bp, true, false);
11684
11685         dev->mtu = new_mtu;
11686         bnxt_set_ring_params(bp);
11687
11688         if (netif_running(dev))
11689                 return bnxt_open_nic(bp, true, false);
11690
11691         return 0;
11692 }
11693
11694 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11695 {
11696         struct bnxt *bp = netdev_priv(dev);
11697         bool sh = false;
11698         int rc;
11699
11700         if (tc > bp->max_tc) {
11701                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11702                            tc, bp->max_tc);
11703                 return -EINVAL;
11704         }
11705
11706         if (netdev_get_num_tc(dev) == tc)
11707                 return 0;
11708
11709         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11710                 sh = true;
11711
11712         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11713                               sh, tc, bp->tx_nr_rings_xdp);
11714         if (rc)
11715                 return rc;
11716
11717         /* Needs to close the device and do hw resource re-allocations */
11718         if (netif_running(bp->dev))
11719                 bnxt_close_nic(bp, true, false);
11720
11721         if (tc) {
11722                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11723                 netdev_set_num_tc(dev, tc);
11724         } else {
11725                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11726                 netdev_reset_tc(dev);
11727         }
11728         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11729         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11730                                bp->tx_nr_rings + bp->rx_nr_rings;
11731
11732         if (netif_running(bp->dev))
11733                 return bnxt_open_nic(bp, true, false);
11734
11735         return 0;
11736 }
11737
11738 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11739                                   void *cb_priv)
11740 {
11741         struct bnxt *bp = cb_priv;
11742
11743         if (!bnxt_tc_flower_enabled(bp) ||
11744             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11745                 return -EOPNOTSUPP;
11746
11747         switch (type) {
11748         case TC_SETUP_CLSFLOWER:
11749                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11750         default:
11751                 return -EOPNOTSUPP;
11752         }
11753 }
11754
11755 LIST_HEAD(bnxt_block_cb_list);
11756
11757 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11758                          void *type_data)
11759 {
11760         struct bnxt *bp = netdev_priv(dev);
11761
11762         switch (type) {
11763         case TC_SETUP_BLOCK:
11764                 return flow_block_cb_setup_simple(type_data,
11765                                                   &bnxt_block_cb_list,
11766                                                   bnxt_setup_tc_block_cb,
11767                                                   bp, bp, true);
11768         case TC_SETUP_QDISC_MQPRIO: {
11769                 struct tc_mqprio_qopt *mqprio = type_data;
11770
11771                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11772
11773                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11774         }
11775         default:
11776                 return -EOPNOTSUPP;
11777         }
11778 }
11779
11780 #ifdef CONFIG_RFS_ACCEL
11781 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11782                             struct bnxt_ntuple_filter *f2)
11783 {
11784         struct flow_keys *keys1 = &f1->fkeys;
11785         struct flow_keys *keys2 = &f2->fkeys;
11786
11787         if (keys1->basic.n_proto != keys2->basic.n_proto ||
11788             keys1->basic.ip_proto != keys2->basic.ip_proto)
11789                 return false;
11790
11791         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11792                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11793                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11794                         return false;
11795         } else {
11796                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11797                            sizeof(keys1->addrs.v6addrs.src)) ||
11798                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11799                            sizeof(keys1->addrs.v6addrs.dst)))
11800                         return false;
11801         }
11802
11803         if (keys1->ports.ports == keys2->ports.ports &&
11804             keys1->control.flags == keys2->control.flags &&
11805             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11806             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11807                 return true;
11808
11809         return false;
11810 }
11811
11812 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11813                               u16 rxq_index, u32 flow_id)
11814 {
11815         struct bnxt *bp = netdev_priv(dev);
11816         struct bnxt_ntuple_filter *fltr, *new_fltr;
11817         struct flow_keys *fkeys;
11818         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11819         int rc = 0, idx, bit_id, l2_idx = 0;
11820         struct hlist_head *head;
11821         u32 flags;
11822
11823         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11824                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11825                 int off = 0, j;
11826
11827                 netif_addr_lock_bh(dev);
11828                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11829                         if (ether_addr_equal(eth->h_dest,
11830                                              vnic->uc_list + off)) {
11831                                 l2_idx = j + 1;
11832                                 break;
11833                         }
11834                 }
11835                 netif_addr_unlock_bh(dev);
11836                 if (!l2_idx)
11837                         return -EINVAL;
11838         }
11839         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11840         if (!new_fltr)
11841                 return -ENOMEM;
11842
11843         fkeys = &new_fltr->fkeys;
11844         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11845                 rc = -EPROTONOSUPPORT;
11846                 goto err_free;
11847         }
11848
11849         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11850              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11851             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11852              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11853                 rc = -EPROTONOSUPPORT;
11854                 goto err_free;
11855         }
11856         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11857             bp->hwrm_spec_code < 0x10601) {
11858                 rc = -EPROTONOSUPPORT;
11859                 goto err_free;
11860         }
11861         flags = fkeys->control.flags;
11862         if (((flags & FLOW_DIS_ENCAPSULATION) &&
11863              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11864                 rc = -EPROTONOSUPPORT;
11865                 goto err_free;
11866         }
11867
11868         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11869         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11870
11871         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11872         head = &bp->ntp_fltr_hash_tbl[idx];
11873         rcu_read_lock();
11874         hlist_for_each_entry_rcu(fltr, head, hash) {
11875                 if (bnxt_fltr_match(fltr, new_fltr)) {
11876                         rcu_read_unlock();
11877                         rc = 0;
11878                         goto err_free;
11879                 }
11880         }
11881         rcu_read_unlock();
11882
11883         spin_lock_bh(&bp->ntp_fltr_lock);
11884         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11885                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
11886         if (bit_id < 0) {
11887                 spin_unlock_bh(&bp->ntp_fltr_lock);
11888                 rc = -ENOMEM;
11889                 goto err_free;
11890         }
11891
11892         new_fltr->sw_id = (u16)bit_id;
11893         new_fltr->flow_id = flow_id;
11894         new_fltr->l2_fltr_idx = l2_idx;
11895         new_fltr->rxq = rxq_index;
11896         hlist_add_head_rcu(&new_fltr->hash, head);
11897         bp->ntp_fltr_count++;
11898         spin_unlock_bh(&bp->ntp_fltr_lock);
11899
11900         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11901         bnxt_queue_sp_work(bp);
11902
11903         return new_fltr->sw_id;
11904
11905 err_free:
11906         kfree(new_fltr);
11907         return rc;
11908 }
11909
11910 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11911 {
11912         int i;
11913
11914         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11915                 struct hlist_head *head;
11916                 struct hlist_node *tmp;
11917                 struct bnxt_ntuple_filter *fltr;
11918                 int rc;
11919
11920                 head = &bp->ntp_fltr_hash_tbl[i];
11921                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11922                         bool del = false;
11923
11924                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11925                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11926                                                         fltr->flow_id,
11927                                                         fltr->sw_id)) {
11928                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
11929                                                                          fltr);
11930                                         del = true;
11931                                 }
11932                         } else {
11933                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11934                                                                        fltr);
11935                                 if (rc)
11936                                         del = true;
11937                                 else
11938                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
11939                         }
11940
11941                         if (del) {
11942                                 spin_lock_bh(&bp->ntp_fltr_lock);
11943                                 hlist_del_rcu(&fltr->hash);
11944                                 bp->ntp_fltr_count--;
11945                                 spin_unlock_bh(&bp->ntp_fltr_lock);
11946                                 synchronize_rcu();
11947                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11948                                 kfree(fltr);
11949                         }
11950                 }
11951         }
11952         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11953                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
11954 }
11955
11956 #else
11957
11958 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11959 {
11960 }
11961
11962 #endif /* CONFIG_RFS_ACCEL */
11963
11964 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11965 {
11966         struct bnxt *bp = netdev_priv(netdev);
11967         struct udp_tunnel_info ti;
11968         unsigned int cmd;
11969
11970         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11971         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11972                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11973         else
11974                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11975
11976         if (ti.port)
11977                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11978
11979         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11980 }
11981
11982 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11983         .sync_table     = bnxt_udp_tunnel_sync,
11984         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11985                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11986         .tables         = {
11987                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
11988                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11989         },
11990 };
11991
11992 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11993                                struct net_device *dev, u32 filter_mask,
11994                                int nlflags)
11995 {
11996         struct bnxt *bp = netdev_priv(dev);
11997
11998         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11999                                        nlflags, filter_mask, NULL);
12000 }
12001
12002 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
12003                                u16 flags, struct netlink_ext_ack *extack)
12004 {
12005         struct bnxt *bp = netdev_priv(dev);
12006         struct nlattr *attr, *br_spec;
12007         int rem, rc = 0;
12008
12009         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
12010                 return -EOPNOTSUPP;
12011
12012         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12013         if (!br_spec)
12014                 return -EINVAL;
12015
12016         nla_for_each_nested(attr, br_spec, rem) {
12017                 u16 mode;
12018
12019                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12020                         continue;
12021
12022                 if (nla_len(attr) < sizeof(mode))
12023                         return -EINVAL;
12024
12025                 mode = nla_get_u16(attr);
12026                 if (mode == bp->br_mode)
12027                         break;
12028
12029                 rc = bnxt_hwrm_set_br_mode(bp, mode);
12030                 if (!rc)
12031                         bp->br_mode = mode;
12032                 break;
12033         }
12034         return rc;
12035 }
12036
12037 int bnxt_get_port_parent_id(struct net_device *dev,
12038                             struct netdev_phys_item_id *ppid)
12039 {
12040         struct bnxt *bp = netdev_priv(dev);
12041
12042         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
12043                 return -EOPNOTSUPP;
12044
12045         /* The PF and it's VF-reps only support the switchdev framework */
12046         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
12047                 return -EOPNOTSUPP;
12048
12049         ppid->id_len = sizeof(bp->dsn);
12050         memcpy(ppid->id, bp->dsn, ppid->id_len);
12051
12052         return 0;
12053 }
12054
12055 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
12056 {
12057         struct bnxt *bp = netdev_priv(dev);
12058
12059         return &bp->dl_port;
12060 }
12061
12062 static const struct net_device_ops bnxt_netdev_ops = {
12063         .ndo_open               = bnxt_open,
12064         .ndo_start_xmit         = bnxt_start_xmit,
12065         .ndo_stop               = bnxt_close,
12066         .ndo_get_stats64        = bnxt_get_stats64,
12067         .ndo_set_rx_mode        = bnxt_set_rx_mode,
12068         .ndo_do_ioctl           = bnxt_ioctl,
12069         .ndo_validate_addr      = eth_validate_addr,
12070         .ndo_set_mac_address    = bnxt_change_mac_addr,
12071         .ndo_change_mtu         = bnxt_change_mtu,
12072         .ndo_fix_features       = bnxt_fix_features,
12073         .ndo_set_features       = bnxt_set_features,
12074         .ndo_tx_timeout         = bnxt_tx_timeout,
12075 #ifdef CONFIG_BNXT_SRIOV
12076         .ndo_get_vf_config      = bnxt_get_vf_config,
12077         .ndo_set_vf_mac         = bnxt_set_vf_mac,
12078         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
12079         .ndo_set_vf_rate        = bnxt_set_vf_bw,
12080         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
12081         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
12082         .ndo_set_vf_trust       = bnxt_set_vf_trust,
12083 #endif
12084         .ndo_setup_tc           = bnxt_setup_tc,
12085 #ifdef CONFIG_RFS_ACCEL
12086         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
12087 #endif
12088         .ndo_udp_tunnel_add     = udp_tunnel_nic_add_port,
12089         .ndo_udp_tunnel_del     = udp_tunnel_nic_del_port,
12090         .ndo_bpf                = bnxt_xdp,
12091         .ndo_xdp_xmit           = bnxt_xdp_xmit,
12092         .ndo_bridge_getlink     = bnxt_bridge_getlink,
12093         .ndo_bridge_setlink     = bnxt_bridge_setlink,
12094         .ndo_get_devlink_port   = bnxt_get_devlink_port,
12095 };
12096
12097 static void bnxt_remove_one(struct pci_dev *pdev)
12098 {
12099         struct net_device *dev = pci_get_drvdata(pdev);
12100         struct bnxt *bp = netdev_priv(dev);
12101
12102         if (BNXT_PF(bp))
12103                 bnxt_sriov_disable(bp);
12104
12105         if (BNXT_PF(bp))
12106                 devlink_port_type_clear(&bp->dl_port);
12107         pci_disable_pcie_error_reporting(pdev);
12108         unregister_netdev(dev);
12109         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12110         /* Flush any pending tasks */
12111         cancel_work_sync(&bp->sp_task);
12112         cancel_delayed_work_sync(&bp->fw_reset_task);
12113         bp->sp_event = 0;
12114
12115         bnxt_dl_fw_reporters_destroy(bp, true);
12116         bnxt_dl_unregister(bp);
12117         bnxt_shutdown_tc(bp);
12118
12119         bnxt_clear_int_mode(bp);
12120         bnxt_hwrm_func_drv_unrgtr(bp);
12121         bnxt_free_hwrm_resources(bp);
12122         bnxt_free_hwrm_short_cmd_req(bp);
12123         bnxt_ethtool_free(bp);
12124         bnxt_dcb_free(bp);
12125         kfree(bp->edev);
12126         bp->edev = NULL;
12127         kfree(bp->fw_health);
12128         bp->fw_health = NULL;
12129         bnxt_cleanup_pci(bp);
12130         bnxt_free_ctx_mem(bp);
12131         kfree(bp->ctx);
12132         bp->ctx = NULL;
12133         kfree(bp->rss_indir_tbl);
12134         bp->rss_indir_tbl = NULL;
12135         bnxt_free_port_stats(bp);
12136         free_netdev(dev);
12137 }
12138
12139 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
12140 {
12141         int rc = 0;
12142         struct bnxt_link_info *link_info = &bp->link_info;
12143
12144         rc = bnxt_hwrm_phy_qcaps(bp);
12145         if (rc) {
12146                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
12147                            rc);
12148                 return rc;
12149         }
12150         if (!fw_dflt)
12151                 return 0;
12152
12153         rc = bnxt_update_link(bp, false);
12154         if (rc) {
12155                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
12156                            rc);
12157                 return rc;
12158         }
12159
12160         /* Older firmware does not have supported_auto_speeds, so assume
12161          * that all supported speeds can be autonegotiated.
12162          */
12163         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
12164                 link_info->support_auto_speeds = link_info->support_speeds;
12165
12166         bnxt_init_ethtool_link_settings(bp);
12167         return 0;
12168 }
12169
12170 static int bnxt_get_max_irq(struct pci_dev *pdev)
12171 {
12172         u16 ctrl;
12173
12174         if (!pdev->msix_cap)
12175                 return 1;
12176
12177         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
12178         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
12179 }
12180
12181 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12182                                 int *max_cp)
12183 {
12184         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12185         int max_ring_grps = 0, max_irq;
12186
12187         *max_tx = hw_resc->max_tx_rings;
12188         *max_rx = hw_resc->max_rx_rings;
12189         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
12190         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
12191                         bnxt_get_ulp_msix_num(bp),
12192                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
12193         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
12194                 *max_cp = min_t(int, *max_cp, max_irq);
12195         max_ring_grps = hw_resc->max_hw_ring_grps;
12196         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
12197                 *max_cp -= 1;
12198                 *max_rx -= 2;
12199         }
12200         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12201                 *max_rx >>= 1;
12202         if (bp->flags & BNXT_FLAG_CHIP_P5) {
12203                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
12204                 /* On P5 chips, max_cp output param should be available NQs */
12205                 *max_cp = max_irq;
12206         }
12207         *max_rx = min_t(int, *max_rx, max_ring_grps);
12208 }
12209
12210 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
12211 {
12212         int rx, tx, cp;
12213
12214         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
12215         *max_rx = rx;
12216         *max_tx = tx;
12217         if (!rx || !tx || !cp)
12218                 return -ENOMEM;
12219
12220         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
12221 }
12222
12223 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12224                                bool shared)
12225 {
12226         int rc;
12227
12228         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12229         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
12230                 /* Not enough rings, try disabling agg rings. */
12231                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
12232                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12233                 if (rc) {
12234                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
12235                         bp->flags |= BNXT_FLAG_AGG_RINGS;
12236                         return rc;
12237                 }
12238                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
12239                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12240                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12241                 bnxt_set_ring_params(bp);
12242         }
12243
12244         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
12245                 int max_cp, max_stat, max_irq;
12246
12247                 /* Reserve minimum resources for RoCE */
12248                 max_cp = bnxt_get_max_func_cp_rings(bp);
12249                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
12250                 max_irq = bnxt_get_max_func_irqs(bp);
12251                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
12252                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
12253                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
12254                         return 0;
12255
12256                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12257                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12258                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12259                 max_cp = min_t(int, max_cp, max_irq);
12260                 max_cp = min_t(int, max_cp, max_stat);
12261                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12262                 if (rc)
12263                         rc = 0;
12264         }
12265         return rc;
12266 }
12267
12268 /* In initial default shared ring setting, each shared ring must have a
12269  * RX/TX ring pair.
12270  */
12271 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12272 {
12273         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12274         bp->rx_nr_rings = bp->cp_nr_rings;
12275         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12276         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12277 }
12278
12279 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12280 {
12281         int dflt_rings, max_rx_rings, max_tx_rings, rc;
12282
12283         if (!bnxt_can_reserve_rings(bp))
12284                 return 0;
12285
12286         if (sh)
12287                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
12288         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12289         /* Reduce default rings on multi-port cards so that total default
12290          * rings do not exceed CPU count.
12291          */
12292         if (bp->port_count > 1) {
12293                 int max_rings =
12294                         max_t(int, num_online_cpus() / bp->port_count, 1);
12295
12296                 dflt_rings = min_t(int, dflt_rings, max_rings);
12297         }
12298         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12299         if (rc)
12300                 return rc;
12301         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12302         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12303         if (sh)
12304                 bnxt_trim_dflt_sh_rings(bp);
12305         else
12306                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12307         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12308
12309         rc = __bnxt_reserve_rings(bp);
12310         if (rc)
12311                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12312         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12313         if (sh)
12314                 bnxt_trim_dflt_sh_rings(bp);
12315
12316         /* Rings may have been trimmed, re-reserve the trimmed rings. */
12317         if (bnxt_need_reserve_rings(bp)) {
12318                 rc = __bnxt_reserve_rings(bp);
12319                 if (rc)
12320                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12321                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12322         }
12323         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12324                 bp->rx_nr_rings++;
12325                 bp->cp_nr_rings++;
12326         }
12327         if (rc) {
12328                 bp->tx_nr_rings = 0;
12329                 bp->rx_nr_rings = 0;
12330         }
12331         return rc;
12332 }
12333
12334 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12335 {
12336         int rc;
12337
12338         if (bp->tx_nr_rings)
12339                 return 0;
12340
12341         bnxt_ulp_irq_stop(bp);
12342         bnxt_clear_int_mode(bp);
12343         rc = bnxt_set_dflt_rings(bp, true);
12344         if (rc) {
12345                 netdev_err(bp->dev, "Not enough rings available.\n");
12346                 goto init_dflt_ring_err;
12347         }
12348         rc = bnxt_init_int_mode(bp);
12349         if (rc)
12350                 goto init_dflt_ring_err;
12351
12352         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12353         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12354                 bp->flags |= BNXT_FLAG_RFS;
12355                 bp->dev->features |= NETIF_F_NTUPLE;
12356         }
12357 init_dflt_ring_err:
12358         bnxt_ulp_irq_restart(bp, rc);
12359         return rc;
12360 }
12361
12362 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12363 {
12364         int rc;
12365
12366         ASSERT_RTNL();
12367         bnxt_hwrm_func_qcaps(bp);
12368
12369         if (netif_running(bp->dev))
12370                 __bnxt_close_nic(bp, true, false);
12371
12372         bnxt_ulp_irq_stop(bp);
12373         bnxt_clear_int_mode(bp);
12374         rc = bnxt_init_int_mode(bp);
12375         bnxt_ulp_irq_restart(bp, rc);
12376
12377         if (netif_running(bp->dev)) {
12378                 if (rc)
12379                         dev_close(bp->dev);
12380                 else
12381                         rc = bnxt_open_nic(bp, true, false);
12382         }
12383
12384         return rc;
12385 }
12386
12387 static int bnxt_init_mac_addr(struct bnxt *bp)
12388 {
12389         int rc = 0;
12390
12391         if (BNXT_PF(bp)) {
12392                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12393         } else {
12394 #ifdef CONFIG_BNXT_SRIOV
12395                 struct bnxt_vf_info *vf = &bp->vf;
12396                 bool strict_approval = true;
12397
12398                 if (is_valid_ether_addr(vf->mac_addr)) {
12399                         /* overwrite netdev dev_addr with admin VF MAC */
12400                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12401                         /* Older PF driver or firmware may not approve this
12402                          * correctly.
12403                          */
12404                         strict_approval = false;
12405                 } else {
12406                         eth_hw_addr_random(bp->dev);
12407                 }
12408                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12409 #endif
12410         }
12411         return rc;
12412 }
12413
12414 #define BNXT_VPD_LEN    512
12415 static void bnxt_vpd_read_info(struct bnxt *bp)
12416 {
12417         struct pci_dev *pdev = bp->pdev;
12418         int i, len, pos, ro_size, size;
12419         ssize_t vpd_size;
12420         u8 *vpd_data;
12421
12422         vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12423         if (!vpd_data)
12424                 return;
12425
12426         vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12427         if (vpd_size <= 0) {
12428                 netdev_err(bp->dev, "Unable to read VPD\n");
12429                 goto exit;
12430         }
12431
12432         i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12433         if (i < 0) {
12434                 netdev_err(bp->dev, "VPD READ-Only not found\n");
12435                 goto exit;
12436         }
12437
12438         ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12439         i += PCI_VPD_LRDT_TAG_SIZE;
12440         if (i + ro_size > vpd_size)
12441                 goto exit;
12442
12443         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12444                                         PCI_VPD_RO_KEYWORD_PARTNO);
12445         if (pos < 0)
12446                 goto read_sn;
12447
12448         len = pci_vpd_info_field_size(&vpd_data[pos]);
12449         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12450         if (len + pos > vpd_size)
12451                 goto read_sn;
12452
12453         size = min(len, BNXT_VPD_FLD_LEN - 1);
12454         memcpy(bp->board_partno, &vpd_data[pos], size);
12455
12456 read_sn:
12457         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12458                                         PCI_VPD_RO_KEYWORD_SERIALNO);
12459         if (pos < 0)
12460                 goto exit;
12461
12462         len = pci_vpd_info_field_size(&vpd_data[pos]);
12463         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12464         if (len + pos > vpd_size)
12465                 goto exit;
12466
12467         size = min(len, BNXT_VPD_FLD_LEN - 1);
12468         memcpy(bp->board_serialno, &vpd_data[pos], size);
12469 exit:
12470         kfree(vpd_data);
12471 }
12472
12473 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12474 {
12475         struct pci_dev *pdev = bp->pdev;
12476         u64 qword;
12477
12478         qword = pci_get_dsn(pdev);
12479         if (!qword) {
12480                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12481                 return -EOPNOTSUPP;
12482         }
12483
12484         put_unaligned_le64(qword, dsn);
12485
12486         bp->flags |= BNXT_FLAG_DSN_VALID;
12487         return 0;
12488 }
12489
12490 static int bnxt_map_db_bar(struct bnxt *bp)
12491 {
12492         if (!bp->db_size)
12493                 return -ENODEV;
12494         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12495         if (!bp->bar1)
12496                 return -ENOMEM;
12497         return 0;
12498 }
12499
12500 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12501 {
12502         struct net_device *dev;
12503         struct bnxt *bp;
12504         int rc, max_irqs;
12505
12506         if (pci_is_bridge(pdev))
12507                 return -ENODEV;
12508
12509         /* Clear any pending DMA transactions from crash kernel
12510          * while loading driver in capture kernel.
12511          */
12512         if (is_kdump_kernel()) {
12513                 pci_clear_master(pdev);
12514                 pcie_flr(pdev);
12515         }
12516
12517         max_irqs = bnxt_get_max_irq(pdev);
12518         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12519         if (!dev)
12520                 return -ENOMEM;
12521
12522         bp = netdev_priv(dev);
12523         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
12524         bnxt_set_max_func_irqs(bp, max_irqs);
12525
12526         if (bnxt_vf_pciid(ent->driver_data))
12527                 bp->flags |= BNXT_FLAG_VF;
12528
12529         if (pdev->msix_cap)
12530                 bp->flags |= BNXT_FLAG_MSIX_CAP;
12531
12532         rc = bnxt_init_board(pdev, dev);
12533         if (rc < 0)
12534                 goto init_err_free;
12535
12536         dev->netdev_ops = &bnxt_netdev_ops;
12537         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12538         dev->ethtool_ops = &bnxt_ethtool_ops;
12539         pci_set_drvdata(pdev, dev);
12540
12541         if (BNXT_PF(bp))
12542                 bnxt_vpd_read_info(bp);
12543
12544         rc = bnxt_alloc_hwrm_resources(bp);
12545         if (rc)
12546                 goto init_err_pci_clean;
12547
12548         mutex_init(&bp->hwrm_cmd_lock);
12549         mutex_init(&bp->link_lock);
12550
12551         rc = bnxt_fw_init_one_p1(bp);
12552         if (rc)
12553                 goto init_err_pci_clean;
12554
12555         if (BNXT_CHIP_P5(bp)) {
12556                 bp->flags |= BNXT_FLAG_CHIP_P5;
12557                 if (BNXT_CHIP_SR2(bp))
12558                         bp->flags |= BNXT_FLAG_CHIP_SR2;
12559         }
12560
12561         rc = bnxt_alloc_rss_indir_tbl(bp);
12562         if (rc)
12563                 goto init_err_pci_clean;
12564
12565         rc = bnxt_fw_init_one_p2(bp);
12566         if (rc)
12567                 goto init_err_pci_clean;
12568
12569         rc = bnxt_map_db_bar(bp);
12570         if (rc) {
12571                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12572                         rc);
12573                 goto init_err_pci_clean;
12574         }
12575
12576         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12577                            NETIF_F_TSO | NETIF_F_TSO6 |
12578                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12579                            NETIF_F_GSO_IPXIP4 |
12580                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12581                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12582                            NETIF_F_RXCSUM | NETIF_F_GRO;
12583
12584         if (BNXT_SUPPORTS_TPA(bp))
12585                 dev->hw_features |= NETIF_F_LRO;
12586
12587         dev->hw_enc_features =
12588                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12589                         NETIF_F_TSO | NETIF_F_TSO6 |
12590                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12591                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12592                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12593         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12594
12595         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12596                                     NETIF_F_GSO_GRE_CSUM;
12597         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12598         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12599                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12600         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12601                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12602         if (BNXT_SUPPORTS_TPA(bp))
12603                 dev->hw_features |= NETIF_F_GRO_HW;
12604         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12605         if (dev->features & NETIF_F_GRO_HW)
12606                 dev->features &= ~NETIF_F_LRO;
12607         dev->priv_flags |= IFF_UNICAST_FLT;
12608
12609 #ifdef CONFIG_BNXT_SRIOV
12610         init_waitqueue_head(&bp->sriov_cfg_wait);
12611         mutex_init(&bp->sriov_lock);
12612 #endif
12613         if (BNXT_SUPPORTS_TPA(bp)) {
12614                 bp->gro_func = bnxt_gro_func_5730x;
12615                 if (BNXT_CHIP_P4(bp))
12616                         bp->gro_func = bnxt_gro_func_5731x;
12617                 else if (BNXT_CHIP_P5(bp))
12618                         bp->gro_func = bnxt_gro_func_5750x;
12619         }
12620         if (!BNXT_CHIP_P4_PLUS(bp))
12621                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
12622
12623         bp->ulp_probe = bnxt_ulp_probe;
12624
12625         rc = bnxt_init_mac_addr(bp);
12626         if (rc) {
12627                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12628                 rc = -EADDRNOTAVAIL;
12629                 goto init_err_pci_clean;
12630         }
12631
12632         if (BNXT_PF(bp)) {
12633                 /* Read the adapter's DSN to use as the eswitch switch_id */
12634                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12635         }
12636
12637         /* MTU range: 60 - FW defined max */
12638         dev->min_mtu = ETH_ZLEN;
12639         dev->max_mtu = bp->max_mtu;
12640
12641         rc = bnxt_probe_phy(bp, true);
12642         if (rc)
12643                 goto init_err_pci_clean;
12644
12645         bnxt_set_rx_skb_mode(bp, false);
12646         bnxt_set_tpa_flags(bp);
12647         bnxt_set_ring_params(bp);
12648         rc = bnxt_set_dflt_rings(bp, true);
12649         if (rc) {
12650                 netdev_err(bp->dev, "Not enough rings available.\n");
12651                 rc = -ENOMEM;
12652                 goto init_err_pci_clean;
12653         }
12654
12655         bnxt_fw_init_one_p3(bp);
12656
12657         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12658                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
12659
12660         rc = bnxt_init_int_mode(bp);
12661         if (rc)
12662                 goto init_err_pci_clean;
12663
12664         /* No TC has been set yet and rings may have been trimmed due to
12665          * limited MSIX, so we re-initialize the TX rings per TC.
12666          */
12667         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12668
12669         if (BNXT_PF(bp)) {
12670                 if (!bnxt_pf_wq) {
12671                         bnxt_pf_wq =
12672                                 create_singlethread_workqueue("bnxt_pf_wq");
12673                         if (!bnxt_pf_wq) {
12674                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
12675                                 goto init_err_pci_clean;
12676                         }
12677                 }
12678                 rc = bnxt_init_tc(bp);
12679                 if (rc)
12680                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12681                                    rc);
12682         }
12683
12684         bnxt_dl_register(bp);
12685
12686         rc = register_netdev(dev);
12687         if (rc)
12688                 goto init_err_cleanup;
12689
12690         if (BNXT_PF(bp))
12691                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12692         bnxt_dl_fw_reporters_create(bp);
12693
12694         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12695                     board_info[ent->driver_data].name,
12696                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
12697         pcie_print_link_status(pdev);
12698
12699         pci_save_state(pdev);
12700         return 0;
12701
12702 init_err_cleanup:
12703         bnxt_dl_unregister(bp);
12704         bnxt_shutdown_tc(bp);
12705         bnxt_clear_int_mode(bp);
12706
12707 init_err_pci_clean:
12708         bnxt_hwrm_func_drv_unrgtr(bp);
12709         bnxt_free_hwrm_short_cmd_req(bp);
12710         bnxt_free_hwrm_resources(bp);
12711         kfree(bp->fw_health);
12712         bp->fw_health = NULL;
12713         bnxt_cleanup_pci(bp);
12714         bnxt_free_ctx_mem(bp);
12715         kfree(bp->ctx);
12716         bp->ctx = NULL;
12717         kfree(bp->rss_indir_tbl);
12718         bp->rss_indir_tbl = NULL;
12719
12720 init_err_free:
12721         free_netdev(dev);
12722         return rc;
12723 }
12724
12725 static void bnxt_shutdown(struct pci_dev *pdev)
12726 {
12727         struct net_device *dev = pci_get_drvdata(pdev);
12728         struct bnxt *bp;
12729
12730         if (!dev)
12731                 return;
12732
12733         rtnl_lock();
12734         bp = netdev_priv(dev);
12735         if (!bp)
12736                 goto shutdown_exit;
12737
12738         if (netif_running(dev))
12739                 dev_close(dev);
12740
12741         bnxt_ulp_shutdown(bp);
12742         bnxt_clear_int_mode(bp);
12743         pci_disable_device(pdev);
12744
12745         if (system_state == SYSTEM_POWER_OFF) {
12746                 pci_wake_from_d3(pdev, bp->wol);
12747                 pci_set_power_state(pdev, PCI_D3hot);
12748         }
12749
12750 shutdown_exit:
12751         rtnl_unlock();
12752 }
12753
12754 #ifdef CONFIG_PM_SLEEP
12755 static int bnxt_suspend(struct device *device)
12756 {
12757         struct net_device *dev = dev_get_drvdata(device);
12758         struct bnxt *bp = netdev_priv(dev);
12759         int rc = 0;
12760
12761         rtnl_lock();
12762         bnxt_ulp_stop(bp);
12763         if (netif_running(dev)) {
12764                 netif_device_detach(dev);
12765                 rc = bnxt_close(dev);
12766         }
12767         bnxt_hwrm_func_drv_unrgtr(bp);
12768         pci_disable_device(bp->pdev);
12769         bnxt_free_ctx_mem(bp);
12770         kfree(bp->ctx);
12771         bp->ctx = NULL;
12772         rtnl_unlock();
12773         return rc;
12774 }
12775
12776 static int bnxt_resume(struct device *device)
12777 {
12778         struct net_device *dev = dev_get_drvdata(device);
12779         struct bnxt *bp = netdev_priv(dev);
12780         int rc = 0;
12781
12782         rtnl_lock();
12783         rc = pci_enable_device(bp->pdev);
12784         if (rc) {
12785                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12786                            rc);
12787                 goto resume_exit;
12788         }
12789         pci_set_master(bp->pdev);
12790         if (bnxt_hwrm_ver_get(bp)) {
12791                 rc = -ENODEV;
12792                 goto resume_exit;
12793         }
12794         rc = bnxt_hwrm_func_reset(bp);
12795         if (rc) {
12796                 rc = -EBUSY;
12797                 goto resume_exit;
12798         }
12799
12800         rc = bnxt_hwrm_func_qcaps(bp);
12801         if (rc)
12802                 goto resume_exit;
12803
12804         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12805                 rc = -ENODEV;
12806                 goto resume_exit;
12807         }
12808
12809         bnxt_get_wol_settings(bp);
12810         if (netif_running(dev)) {
12811                 rc = bnxt_open(dev);
12812                 if (!rc)
12813                         netif_device_attach(dev);
12814         }
12815
12816 resume_exit:
12817         bnxt_ulp_start(bp, rc);
12818         if (!rc)
12819                 bnxt_reenable_sriov(bp);
12820         rtnl_unlock();
12821         return rc;
12822 }
12823
12824 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12825 #define BNXT_PM_OPS (&bnxt_pm_ops)
12826
12827 #else
12828
12829 #define BNXT_PM_OPS NULL
12830
12831 #endif /* CONFIG_PM_SLEEP */
12832
12833 /**
12834  * bnxt_io_error_detected - called when PCI error is detected
12835  * @pdev: Pointer to PCI device
12836  * @state: The current pci connection state
12837  *
12838  * This function is called after a PCI bus error affecting
12839  * this device has been detected.
12840  */
12841 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12842                                                pci_channel_state_t state)
12843 {
12844         struct net_device *netdev = pci_get_drvdata(pdev);
12845         struct bnxt *bp = netdev_priv(netdev);
12846
12847         netdev_info(netdev, "PCI I/O error detected\n");
12848
12849         rtnl_lock();
12850         netif_device_detach(netdev);
12851
12852         bnxt_ulp_stop(bp);
12853
12854         if (state == pci_channel_io_perm_failure) {
12855                 rtnl_unlock();
12856                 return PCI_ERS_RESULT_DISCONNECT;
12857         }
12858
12859         if (state == pci_channel_io_frozen)
12860                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
12861
12862         if (netif_running(netdev))
12863                 bnxt_close(netdev);
12864
12865         pci_disable_device(pdev);
12866         bnxt_free_ctx_mem(bp);
12867         kfree(bp->ctx);
12868         bp->ctx = NULL;
12869         rtnl_unlock();
12870
12871         /* Request a slot slot reset. */
12872         return PCI_ERS_RESULT_NEED_RESET;
12873 }
12874
12875 /**
12876  * bnxt_io_slot_reset - called after the pci bus has been reset.
12877  * @pdev: Pointer to PCI device
12878  *
12879  * Restart the card from scratch, as if from a cold-boot.
12880  * At this point, the card has exprienced a hard reset,
12881  * followed by fixups by BIOS, and has its config space
12882  * set up identically to what it was at cold boot.
12883  */
12884 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12885 {
12886         struct net_device *netdev = pci_get_drvdata(pdev);
12887         struct bnxt *bp = netdev_priv(netdev);
12888         int err = 0, off;
12889         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12890
12891         netdev_info(bp->dev, "PCI Slot Reset\n");
12892
12893         rtnl_lock();
12894
12895         if (pci_enable_device(pdev)) {
12896                 dev_err(&pdev->dev,
12897                         "Cannot re-enable PCI device after reset.\n");
12898         } else {
12899                 pci_set_master(pdev);
12900                 /* Upon fatal error, our device internal logic that latches to
12901                  * BAR value is getting reset and will restore only upon
12902                  * rewritting the BARs.
12903                  *
12904                  * As pci_restore_state() does not re-write the BARs if the
12905                  * value is same as saved value earlier, driver needs to
12906                  * write the BARs to 0 to force restore, in case of fatal error.
12907                  */
12908                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
12909                                        &bp->state)) {
12910                         for (off = PCI_BASE_ADDRESS_0;
12911                              off <= PCI_BASE_ADDRESS_5; off += 4)
12912                                 pci_write_config_dword(bp->pdev, off, 0);
12913                 }
12914                 pci_restore_state(pdev);
12915                 pci_save_state(pdev);
12916
12917                 err = bnxt_hwrm_func_reset(bp);
12918                 if (!err) {
12919                         err = bnxt_hwrm_func_qcaps(bp);
12920                         if (!err && netif_running(netdev))
12921                                 err = bnxt_open(netdev);
12922                 }
12923                 bnxt_ulp_start(bp, err);
12924                 if (!err) {
12925                         bnxt_reenable_sriov(bp);
12926                         result = PCI_ERS_RESULT_RECOVERED;
12927                 }
12928         }
12929
12930         if (result != PCI_ERS_RESULT_RECOVERED) {
12931                 if (netif_running(netdev))
12932                         dev_close(netdev);
12933                 pci_disable_device(pdev);
12934         }
12935
12936         rtnl_unlock();
12937
12938         return result;
12939 }
12940
12941 /**
12942  * bnxt_io_resume - called when traffic can start flowing again.
12943  * @pdev: Pointer to PCI device
12944  *
12945  * This callback is called when the error recovery driver tells
12946  * us that its OK to resume normal operation.
12947  */
12948 static void bnxt_io_resume(struct pci_dev *pdev)
12949 {
12950         struct net_device *netdev = pci_get_drvdata(pdev);
12951
12952         rtnl_lock();
12953
12954         netif_device_attach(netdev);
12955
12956         rtnl_unlock();
12957 }
12958
12959 static const struct pci_error_handlers bnxt_err_handler = {
12960         .error_detected = bnxt_io_error_detected,
12961         .slot_reset     = bnxt_io_slot_reset,
12962         .resume         = bnxt_io_resume
12963 };
12964
12965 static struct pci_driver bnxt_pci_driver = {
12966         .name           = DRV_MODULE_NAME,
12967         .id_table       = bnxt_pci_tbl,
12968         .probe          = bnxt_init_one,
12969         .remove         = bnxt_remove_one,
12970         .shutdown       = bnxt_shutdown,
12971         .driver.pm      = BNXT_PM_OPS,
12972         .err_handler    = &bnxt_err_handler,
12973 #if defined(CONFIG_BNXT_SRIOV)
12974         .sriov_configure = bnxt_sriov_configure,
12975 #endif
12976 };
12977
12978 static int __init bnxt_init(void)
12979 {
12980         bnxt_debug_init();
12981         return pci_register_driver(&bnxt_pci_driver);
12982 }
12983
12984 static void __exit bnxt_exit(void)
12985 {
12986         pci_unregister_driver(&bnxt_pci_driver);
12987         if (bnxt_pf_wq)
12988                 destroy_workqueue(bnxt_pf_wq);
12989         bnxt_debug_exit();
12990 }
12991
12992 module_init(bnxt_init);
12993 module_exit(bnxt_exit);