drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/gro.h>
  41 #include <net/ip.h>
  42 #include <net/tcp.h>
  43 #include <net/udp.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46 #include <net/udp_tunnel.h>
  47 #include <linux/workqueue.h>
  48 #include <linux/prefetch.h>
  49 #include <linux/cache.h>
  50 #include <linux/log2.h>
  51 #include <linux/aer.h>
  52 #include <linux/bitmap.h>
  53 #include <linux/cpu_rmap.h>
  54 #include <linux/cpumask.h>
  55 #include <net/pkt_cls.h>
  56 #include <linux/hwmon.h>
  57 #include <linux/hwmon-sysfs.h>
  58 #include <net/page_pool.h>
  59 #include <linux/align.h>
  60
  61 #include "bnxt_hsi.h"
  62 #include "bnxt.h"
  63 #include "bnxt_hwrm.h"
  64 #include "bnxt_ulp.h"
  65 #include "bnxt_sriov.h"
  66 #include "bnxt_ethtool.h"
  67 #include "bnxt_dcb.h"
  68 #include "bnxt_xdp.h"
  69 #include "bnxt_ptp.h"
  70 #include "bnxt_vfr.h"
  71 #include "bnxt_tc.h"
  72 #include "bnxt_devlink.h"
  73 #include "bnxt_debugfs.h"
  74
  75 #define BNXT_TX_TIMEOUT         (5 * HZ)
  76 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW | \
  77                                  NETIF_MSG_TX_ERR)
  78
  79 MODULE_LICENSE("GPL");
  80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  81
  82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  84 #define BNXT_RX_COPY_THRESH 256
  85
  86 #define BNXT_TX_PUSH_THRESH 164
  87
  88 /* indexed by enum board_idx */
  89 static const struct {
  90         char *name;
  91 } board_info[] = {
  92         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
  93         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
  94         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
  95         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
  96         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
  97         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
  98         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
  99         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 100         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 101         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 102         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 103         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 104         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 105         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 106         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 107         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 108         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 109         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 110         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 111         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 112         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 113         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 114         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 115         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 116         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 117         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 118         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 119         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 120         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 121         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 122         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 123         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 124         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 125         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 126         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 127         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 128         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 129         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 130         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 131         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 132         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 133         [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
 134         [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
 135         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 136         [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
 137 };
 138
 139 static const struct pci_device_id bnxt_pci_tbl[] = {
 140         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 141         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 142         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 143         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 144         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 145         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 146         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 147         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 148         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 149         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 150         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 151         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 152         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 153         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 154         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 155         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 156         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 157         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 158         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 159         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 160         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 161         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 162         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 163         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 164         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 165         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 166         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 167         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 168         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 169         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 170         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 171         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 172         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 173         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 174         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 175         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 176         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 177         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 178         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 180         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 183         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 185         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 186 #ifdef CONFIG_BNXT_SRIOV
 187         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 188         { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
 189         { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
 190         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 191         { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
 192         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 193         { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
 194         { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
 195         { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
 196         { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
 197         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 198         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 199         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 200         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 201         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 202         { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
 203         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 204         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 205         { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
 206         { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
 207         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 208 #endif
 209         { 0 }
 210 };
 211
 212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 213
 214 static const u16 bnxt_vf_req_snif[] = {
 215         HWRM_FUNC_CFG,
 216         HWRM_FUNC_VF_CFG,
 217         HWRM_PORT_PHY_QCFG,
 218         HWRM_CFA_L2_FILTER_ALLOC,
 219 };
 220
 221 static const u16 bnxt_async_events_arr[] = {
 222         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 223         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 224         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 225         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 226         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 227         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 228         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 229         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 230         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 231         ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
 232         ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
 233         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 234         ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
 235         ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
 236         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
 237         ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
 238 };
 239
 240 static struct workqueue_struct *bnxt_pf_wq;
 241
 242 static bool bnxt_vf_pciid(enum board_idx idx)
 243 {
 244         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 245                 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
 246                 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
 247                 idx == NETXTREME_E_P5_VF_HV);
 248 }
 249
 250 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 251 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 252 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 253
 254 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 255                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 256
 257 #define BNXT_DB_CQ(db, idx)                                             \
 258         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 259
 260 #define BNXT_DB_NQ_P5(db, idx)                                          \
 261         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx),   \
 262                     (db)->doorbell)
 263
 264 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 265         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 266
 267 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 268         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\
 269                     (db)->doorbell)
 270
 271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 272 {
 273         if (bp->flags & BNXT_FLAG_CHIP_P5)
 274                 BNXT_DB_NQ_P5(db, idx);
 275         else
 276                 BNXT_DB_CQ(db, idx);
 277 }
 278
 279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 280 {
 281         if (bp->flags & BNXT_FLAG_CHIP_P5)
 282                 BNXT_DB_NQ_ARM_P5(db, idx);
 283         else
 284                 BNXT_DB_CQ_ARM(db, idx);
 285 }
 286
 287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 288 {
 289         if (bp->flags & BNXT_FLAG_CHIP_P5)
 290                 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
 291                             RING_CMP(idx), db->doorbell);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 const u16 bnxt_lhint_arr[] = {
 297         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 298         TX_BD_FLAGS_LHINT_512_TO_1023,
 299         TX_BD_FLAGS_LHINT_1024_TO_2047,
 300         TX_BD_FLAGS_LHINT_1024_TO_2047,
 301         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 302         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 303         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 304         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 305         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 306         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 307         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 308         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 309         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 310         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 311         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 312         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 313         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 314         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 315         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 316 };
 317
 318 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 319 {
 320         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 321
 322         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 323                 return 0;
 324
 325         return md_dst->u.port_info.port_id;
 326 }
 327
 328 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 329                              u16 prod)
 330 {
 331         bnxt_db_write(bp, &txr->tx_db, prod);
 332         txr->kick_pending = 0;
 333 }
 334
 335 static bool bnxt_txr_netif_try_stop_queue(struct bnxt *bp,
 336                                           struct bnxt_tx_ring_info *txr,
 337                                           struct netdev_queue *txq)
 338 {
 339         netif_tx_stop_queue(txq);
 340
 341         /* netif_tx_stop_queue() must be done before checking
 342          * tx index in bnxt_tx_avail() below, because in
 343          * bnxt_tx_int(), we update tx index before checking for
 344          * netif_tx_queue_stopped().
 345          */
 346         smp_mb();
 347         if (bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh) {
 348                 netif_tx_wake_queue(txq);
 349                 return false;
 350         }
 351
 352         return true;
 353 }
 354
 355 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 356 {
 357         struct bnxt *bp = netdev_priv(dev);
 358         struct tx_bd *txbd;
 359         struct tx_bd_ext *txbd1;
 360         struct netdev_queue *txq;
 361         int i;
 362         dma_addr_t mapping;
 363         unsigned int length, pad = 0;
 364         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 365         u16 prod, last_frag;
 366         struct pci_dev *pdev = bp->pdev;
 367         struct bnxt_tx_ring_info *txr;
 368         struct bnxt_sw_tx_bd *tx_buf;
 369         __le32 lflags = 0;
 370
 371         i = skb_get_queue_mapping(skb);
 372         if (unlikely(i >= bp->tx_nr_rings)) {
 373                 dev_kfree_skb_any(skb);
 374                 dev_core_stats_tx_dropped_inc(dev);
 375                 return NETDEV_TX_OK;
 376         }
 377
 378         txq = netdev_get_tx_queue(dev, i);
 379         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 380         prod = txr->tx_prod;
 381
 382         free_size = bnxt_tx_avail(bp, txr);
 383         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 384                 /* We must have raced with NAPI cleanup */
 385                 if (net_ratelimit() && txr->kick_pending)
 386                         netif_warn(bp, tx_err, dev,
 387                                    "bnxt: ring busy w/ flush pending!\n");
 388                 if (bnxt_txr_netif_try_stop_queue(bp, txr, txq))
 389                         return NETDEV_TX_BUSY;
 390         }
 391
 392         length = skb->len;
 393         len = skb_headlen(skb);
 394         last_frag = skb_shinfo(skb)->nr_frags;
 395
 396         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 397
 398         txbd->tx_bd_opaque = prod;
 399
 400         tx_buf = &txr->tx_buf_ring[prod];
 401         tx_buf->skb = skb;
 402         tx_buf->nr_frags = last_frag;
 403
 404         vlan_tag_flags = 0;
 405         cfa_action = bnxt_xmit_get_cfa_action(skb);
 406         if (skb_vlan_tag_present(skb)) {
 407                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 408                                  skb_vlan_tag_get(skb);
 409                 /* Currently supports 8021Q, 8021AD vlan offloads
 410                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 411                  */
 412                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 413                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 414         }
 415
 416         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
 417                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
 418
 419                 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
 420                     atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
 421                         if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
 422                                             &ptp->tx_hdr_off)) {
 423                                 if (vlan_tag_flags)
 424                                         ptp->tx_hdr_off += VLAN_HLEN;
 425                                 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
 426                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 427                         } else {
 428                                 atomic_inc(&bp->ptp_cfg->tx_avail);
 429                         }
 430                 }
 431         }
 432
 433         if (unlikely(skb->no_fcs))
 434                 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
 435
 436         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
 437             !lflags) {
 438                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 439                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 440                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 441                 void __iomem *db = txr->tx_db.doorbell;
 442                 void *pdata = tx_push_buf->data;
 443                 u64 *end;
 444                 int j, push_len;
 445
 446                 /* Set COAL_NOW to be ready quickly for the next push */
 447                 tx_push->tx_bd_len_flags_type =
 448                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 449                                         TX_BD_TYPE_LONG_TX_BD |
 450                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 451                                         TX_BD_FLAGS_COAL_NOW |
 452                                         TX_BD_FLAGS_PACKET_END |
 453                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 454
 455                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 456                         tx_push1->tx_bd_hsize_lflags =
 457                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 458                 else
 459                         tx_push1->tx_bd_hsize_lflags = 0;
 460
 461                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 462                 tx_push1->tx_bd_cfa_action =
 463                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 464
 465                 end = pdata + length;
 466                 end = PTR_ALIGN(end, 8) - 1;
 467                 *end = 0;
 468
 469                 skb_copy_from_linear_data(skb, pdata, len);
 470                 pdata += len;
 471                 for (j = 0; j < last_frag; j++) {
 472                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 473                         void *fptr;
 474
 475                         fptr = skb_frag_address_safe(frag);
 476                         if (!fptr)
 477                                 goto normal_tx;
 478
 479                         memcpy(pdata, fptr, skb_frag_size(frag));
 480                         pdata += skb_frag_size(frag);
 481                 }
 482
 483                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 484                 txbd->tx_bd_haddr = txr->data_mapping;
 485                 prod = NEXT_TX(prod);
 486                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 487                 memcpy(txbd, tx_push1, sizeof(*txbd));
 488                 prod = NEXT_TX(prod);
 489                 tx_push->doorbell =
 490                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 491                 txr->tx_prod = prod;
 492
 493                 tx_buf->is_push = 1;
 494                 netdev_tx_sent_queue(txq, skb->len);
 495                 wmb();  /* Sync is_push and byte queue before pushing data */
 496
 497                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 498                 if (push_len > 16) {
 499                         __iowrite64_copy(db, tx_push_buf, 16);
 500                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 501                                          (push_len - 16) << 1);
 502                 } else {
 503                         __iowrite64_copy(db, tx_push_buf, push_len);
 504                 }
 505
 506                 goto tx_done;
 507         }
 508
 509 normal_tx:
 510         if (length < BNXT_MIN_PKT_SIZE) {
 511                 pad = BNXT_MIN_PKT_SIZE - length;
 512                 if (skb_pad(skb, pad))
 513                         /* SKB already freed. */
 514                         goto tx_kick_pending;
 515                 length = BNXT_MIN_PKT_SIZE;
 516         }
 517
 518         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 519
 520         if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 521                 goto tx_free;
 522
 523         dma_unmap_addr_set(tx_buf, mapping, mapping);
 524         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 525                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 526
 527         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 528
 529         prod = NEXT_TX(prod);
 530         txbd1 = (struct tx_bd_ext *)
 531                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 532
 533         txbd1->tx_bd_hsize_lflags = lflags;
 534         if (skb_is_gso(skb)) {
 535                 u32 hdr_len;
 536
 537                 if (skb->encapsulation)
 538                         hdr_len = skb_inner_tcp_all_headers(skb);
 539                 else
 540                         hdr_len = skb_tcp_all_headers(skb);
 541
 542                 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
 543                                         TX_BD_FLAGS_T_IPID |
 544                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 545                 length = skb_shinfo(skb)->gso_size;
 546                 txbd1->tx_bd_mss = cpu_to_le32(length);
 547                 length += hdr_len;
 548         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 549                 txbd1->tx_bd_hsize_lflags |=
 550                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 551                 txbd1->tx_bd_mss = 0;
 552         }
 553
 554         length >>= 9;
 555         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 556                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 557                                      skb->len);
 558                 i = 0;
 559                 goto tx_dma_error;
 560         }
 561         flags |= bnxt_lhint_arr[length];
 562         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 563
 564         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 565         txbd1->tx_bd_cfa_action =
 566                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 567         for (i = 0; i < last_frag; i++) {
 568                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 569
 570                 prod = NEXT_TX(prod);
 571                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 572
 573                 len = skb_frag_size(frag);
 574                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 575                                            DMA_TO_DEVICE);
 576
 577                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 578                         goto tx_dma_error;
 579
 580                 tx_buf = &txr->tx_buf_ring[prod];
 581                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 582
 583                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 584
 585                 flags = len << TX_BD_LEN_SHIFT;
 586                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 587         }
 588
 589         flags &= ~TX_BD_LEN;
 590         txbd->tx_bd_len_flags_type =
 591                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 592                             TX_BD_FLAGS_PACKET_END);
 593
 594         netdev_tx_sent_queue(txq, skb->len);
 595
 596         skb_tx_timestamp(skb);
 597
 598         /* Sync BD data before updating doorbell */
 599         wmb();
 600
 601         prod = NEXT_TX(prod);
 602         txr->tx_prod = prod;
 603
 604         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 605                 bnxt_txr_db_kick(bp, txr, prod);
 606         else
 607                 txr->kick_pending = 1;
 608
 609 tx_done:
 610
 611         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 612                 if (netdev_xmit_more() && !tx_buf->is_push)
 613                         bnxt_txr_db_kick(bp, txr, prod);
 614
 615                 bnxt_txr_netif_try_stop_queue(bp, txr, txq);
 616         }
 617         return NETDEV_TX_OK;
 618
 619 tx_dma_error:
 620         if (BNXT_TX_PTP_IS_SET(lflags))
 621                 atomic_inc(&bp->ptp_cfg->tx_avail);
 622
 623         last_frag = i;
 624
 625         /* start back at beginning and unmap skb */
 626         prod = txr->tx_prod;
 627         tx_buf = &txr->tx_buf_ring[prod];
 628         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 629                          skb_headlen(skb), DMA_TO_DEVICE);
 630         prod = NEXT_TX(prod);
 631
 632         /* unmap remaining mapped pages */
 633         for (i = 0; i < last_frag; i++) {
 634                 prod = NEXT_TX(prod);
 635                 tx_buf = &txr->tx_buf_ring[prod];
 636                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 637                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 638                                DMA_TO_DEVICE);
 639         }
 640
 641 tx_free:
 642         dev_kfree_skb_any(skb);
 643 tx_kick_pending:
 644         if (txr->kick_pending)
 645                 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
 646         txr->tx_buf_ring[txr->tx_prod].skb = NULL;
 647         dev_core_stats_tx_dropped_inc(dev);
 648         return NETDEV_TX_OK;
 649 }
 650
 651 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 652 {
 653         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 654         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 655         u16 cons = txr->tx_cons;
 656         struct pci_dev *pdev = bp->pdev;
 657         int i;
 658         unsigned int tx_bytes = 0;
 659
 660         for (i = 0; i < nr_pkts; i++) {
 661                 struct bnxt_sw_tx_bd *tx_buf;
 662                 bool compl_deferred = false;
 663                 struct sk_buff *skb;
 664                 int j, last;
 665
 666                 tx_buf = &txr->tx_buf_ring[cons];
 667                 cons = NEXT_TX(cons);
 668                 skb = tx_buf->skb;
 669                 tx_buf->skb = NULL;
 670
 671                 if (tx_buf->is_push) {
 672                         tx_buf->is_push = 0;
 673                         goto next_tx_int;
 674                 }
 675
 676                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 677                                  skb_headlen(skb), DMA_TO_DEVICE);
 678                 last = tx_buf->nr_frags;
 679
 680                 for (j = 0; j < last; j++) {
 681                         cons = NEXT_TX(cons);
 682                         tx_buf = &txr->tx_buf_ring[cons];
 683                         dma_unmap_page(
 684                                 &pdev->dev,
 685                                 dma_unmap_addr(tx_buf, mapping),
 686                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 687                                 DMA_TO_DEVICE);
 688                 }
 689                 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
 690                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
 691                                 if (!bnxt_get_tx_ts_p5(bp, skb))
 692                                         compl_deferred = true;
 693                                 else
 694                                         atomic_inc(&bp->ptp_cfg->tx_avail);
 695                         }
 696                 }
 697
 698 next_tx_int:
 699                 cons = NEXT_TX(cons);
 700
 701                 tx_bytes += skb->len;
 702                 if (!compl_deferred)
 703                         dev_kfree_skb_any(skb);
 704         }
 705
 706         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 707         txr->tx_cons = cons;
 708
 709         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 710          * before checking for netif_tx_queue_stopped().  Without the
 711          * memory barrier, there is a small possibility that bnxt_start_xmit()
 712          * will miss it and cause the queue to be stopped forever.
 713          */
 714         smp_mb();
 715
 716         if (unlikely(netif_tx_queue_stopped(txq)) &&
 717             bnxt_tx_avail(bp, txr) >= bp->tx_wake_thresh &&
 718             READ_ONCE(txr->dev_state) != BNXT_DEV_STATE_CLOSING)
 719                 netif_tx_wake_queue(txq);
 720 }
 721
 722 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 723                                          struct bnxt_rx_ring_info *rxr,
 724                                          gfp_t gfp)
 725 {
 726         struct device *dev = &bp->pdev->dev;
 727         struct page *page;
 728
 729         page = page_pool_dev_alloc_pages(rxr->page_pool);
 730         if (!page)
 731                 return NULL;
 732
 733         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 734                                       DMA_ATTR_WEAK_ORDERING);
 735         if (dma_mapping_error(dev, *mapping)) {
 736                 page_pool_recycle_direct(rxr->page_pool, page);
 737                 return NULL;
 738         }
 739         return page;
 740 }
 741
 742 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
 743                                        gfp_t gfp)
 744 {
 745         u8 *data;
 746         struct pci_dev *pdev = bp->pdev;
 747
 748         if (gfp == GFP_ATOMIC)
 749                 data = napi_alloc_frag(bp->rx_buf_size);
 750         else
 751                 data = netdev_alloc_frag(bp->rx_buf_size);
 752         if (!data)
 753                 return NULL;
 754
 755         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 756                                         bp->rx_buf_use_size, bp->rx_dir,
 757                                         DMA_ATTR_WEAK_ORDERING);
 758
 759         if (dma_mapping_error(&pdev->dev, *mapping)) {
 760                 skb_free_frag(data);
 761                 data = NULL;
 762         }
 763         return data;
 764 }
 765
 766 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 767                        u16 prod, gfp_t gfp)
 768 {
 769         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 770         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 771         dma_addr_t mapping;
 772
 773         if (BNXT_RX_PAGE_MODE(bp)) {
 774                 struct page *page =
 775                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 776
 777                 if (!page)
 778                         return -ENOMEM;
 779
 780                 mapping += bp->rx_dma_offset;
 781                 rx_buf->data = page;
 782                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 783         } else {
 784                 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
 785
 786                 if (!data)
 787                         return -ENOMEM;
 788
 789                 rx_buf->data = data;
 790                 rx_buf->data_ptr = data + bp->rx_offset;
 791         }
 792         rx_buf->mapping = mapping;
 793
 794         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 795         return 0;
 796 }
 797
 798 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 799 {
 800         u16 prod = rxr->rx_prod;
 801         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 802         struct rx_bd *cons_bd, *prod_bd;
 803
 804         prod_rx_buf = &rxr->rx_buf_ring[prod];
 805         cons_rx_buf = &rxr->rx_buf_ring[cons];
 806
 807         prod_rx_buf->data = data;
 808         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 809
 810         prod_rx_buf->mapping = cons_rx_buf->mapping;
 811
 812         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 813         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 814
 815         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 816 }
 817
 818 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 819 {
 820         u16 next, max = rxr->rx_agg_bmap_size;
 821
 822         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 823         if (next >= max)
 824                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 825         return next;
 826 }
 827
 828 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 829                                      struct bnxt_rx_ring_info *rxr,
 830                                      u16 prod, gfp_t gfp)
 831 {
 832         struct rx_bd *rxbd =
 833                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 834         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 835         struct pci_dev *pdev = bp->pdev;
 836         struct page *page;
 837         dma_addr_t mapping;
 838         u16 sw_prod = rxr->rx_sw_agg_prod;
 839         unsigned int offset = 0;
 840
 841         if (BNXT_RX_PAGE_MODE(bp)) {
 842                 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 843
 844                 if (!page)
 845                         return -ENOMEM;
 846
 847         } else {
 848                 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 849                         page = rxr->rx_page;
 850                         if (!page) {
 851                                 page = alloc_page(gfp);
 852                                 if (!page)
 853                                         return -ENOMEM;
 854                                 rxr->rx_page = page;
 855                                 rxr->rx_page_offset = 0;
 856                         }
 857                         offset = rxr->rx_page_offset;
 858                         rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 859                         if (rxr->rx_page_offset == PAGE_SIZE)
 860                                 rxr->rx_page = NULL;
 861                         else
 862                                 get_page(page);
 863                 } else {
 864                         page = alloc_page(gfp);
 865                         if (!page)
 866                                 return -ENOMEM;
 867                 }
 868
 869                 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 870                                              BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE,
 871                                              DMA_ATTR_WEAK_ORDERING);
 872                 if (dma_mapping_error(&pdev->dev, mapping)) {
 873                         __free_page(page);
 874                         return -EIO;
 875                 }
 876         }
 877
 878         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 879                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 880
 881         __set_bit(sw_prod, rxr->rx_agg_bmap);
 882         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 883         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 884
 885         rx_agg_buf->page = page;
 886         rx_agg_buf->offset = offset;
 887         rx_agg_buf->mapping = mapping;
 888         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 889         rxbd->rx_bd_opaque = sw_prod;
 890         return 0;
 891 }
 892
 893 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 894                                        struct bnxt_cp_ring_info *cpr,
 895                                        u16 cp_cons, u16 curr)
 896 {
 897         struct rx_agg_cmp *agg;
 898
 899         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 900         agg = (struct rx_agg_cmp *)
 901                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 902         return agg;
 903 }
 904
 905 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 906                                               struct bnxt_rx_ring_info *rxr,
 907                                               u16 agg_id, u16 curr)
 908 {
 909         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 910
 911         return &tpa_info->agg_arr[curr];
 912 }
 913
 914 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 915                                    u16 start, u32 agg_bufs, bool tpa)
 916 {
 917         struct bnxt_napi *bnapi = cpr->bnapi;
 918         struct bnxt *bp = bnapi->bp;
 919         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 920         u16 prod = rxr->rx_agg_prod;
 921         u16 sw_prod = rxr->rx_sw_agg_prod;
 922         bool p5_tpa = false;
 923         u32 i;
 924
 925         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 926                 p5_tpa = true;
 927
 928         for (i = 0; i < agg_bufs; i++) {
 929                 u16 cons;
 930                 struct rx_agg_cmp *agg;
 931                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 932                 struct rx_bd *prod_bd;
 933                 struct page *page;
 934
 935                 if (p5_tpa)
 936                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 937                 else
 938                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 939                 cons = agg->rx_agg_cmp_opaque;
 940                 __clear_bit(cons, rxr->rx_agg_bmap);
 941
 942                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 943                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 944
 945                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 946                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 947                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 948
 949                 /* It is possible for sw_prod to be equal to cons, so
 950                  * set cons_rx_buf->page to NULL first.
 951                  */
 952                 page = cons_rx_buf->page;
 953                 cons_rx_buf->page = NULL;
 954                 prod_rx_buf->page = page;
 955                 prod_rx_buf->offset = cons_rx_buf->offset;
 956
 957                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 958
 959                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 960
 961                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 962                 prod_bd->rx_bd_opaque = sw_prod;
 963
 964                 prod = NEXT_RX_AGG(prod);
 965                 sw_prod = NEXT_RX_AGG(sw_prod);
 966         }
 967         rxr->rx_agg_prod = prod;
 968         rxr->rx_sw_agg_prod = sw_prod;
 969 }
 970
 971 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
 972                                               struct bnxt_rx_ring_info *rxr,
 973                                               u16 cons, void *data, u8 *data_ptr,
 974                                               dma_addr_t dma_addr,
 975                                               unsigned int offset_and_len)
 976 {
 977         unsigned int len = offset_and_len & 0xffff;
 978         struct page *page = data;
 979         u16 prod = rxr->rx_prod;
 980         struct sk_buff *skb;
 981         int err;
 982
 983         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 984         if (unlikely(err)) {
 985                 bnxt_reuse_rx_data(rxr, cons, data);
 986                 return NULL;
 987         }
 988         dma_addr -= bp->rx_dma_offset;
 989         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 990                              DMA_ATTR_WEAK_ORDERING);
 991         skb = build_skb(page_address(page), BNXT_PAGE_MODE_BUF_SIZE +
 992                                             bp->rx_dma_offset);
 993         if (!skb) {
 994                 __free_page(page);
 995                 return NULL;
 996         }
 997         skb_mark_for_recycle(skb);
 998         skb_reserve(skb, bp->rx_dma_offset);
 999         __skb_put(skb, len);
1000
1001         return skb;
1002 }
1003
1004 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1005                                         struct bnxt_rx_ring_info *rxr,
1006                                         u16 cons, void *data, u8 *data_ptr,
1007                                         dma_addr_t dma_addr,
1008                                         unsigned int offset_and_len)
1009 {
1010         unsigned int payload = offset_and_len >> 16;
1011         unsigned int len = offset_and_len & 0xffff;
1012         skb_frag_t *frag;
1013         struct page *page = data;
1014         u16 prod = rxr->rx_prod;
1015         struct sk_buff *skb;
1016         int off, err;
1017
1018         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1019         if (unlikely(err)) {
1020                 bnxt_reuse_rx_data(rxr, cons, data);
1021                 return NULL;
1022         }
1023         dma_addr -= bp->rx_dma_offset;
1024         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
1025                              DMA_ATTR_WEAK_ORDERING);
1026
1027         if (unlikely(!payload))
1028                 payload = eth_get_headlen(bp->dev, data_ptr, len);
1029
1030         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1031         if (!skb) {
1032                 __free_page(page);
1033                 return NULL;
1034         }
1035
1036         skb_mark_for_recycle(skb);
1037         off = (void *)data_ptr - page_address(page);
1038         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
1039         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1040                payload + NET_IP_ALIGN);
1041
1042         frag = &skb_shinfo(skb)->frags[0];
1043         skb_frag_size_sub(frag, payload);
1044         skb_frag_off_add(frag, payload);
1045         skb->data_len -= payload;
1046         skb->tail += payload;
1047
1048         return skb;
1049 }
1050
1051 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1052                                    struct bnxt_rx_ring_info *rxr, u16 cons,
1053                                    void *data, u8 *data_ptr,
1054                                    dma_addr_t dma_addr,
1055                                    unsigned int offset_and_len)
1056 {
1057         u16 prod = rxr->rx_prod;
1058         struct sk_buff *skb;
1059         int err;
1060
1061         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1062         if (unlikely(err)) {
1063                 bnxt_reuse_rx_data(rxr, cons, data);
1064                 return NULL;
1065         }
1066
1067         skb = build_skb(data, bp->rx_buf_size);
1068         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1069                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1070         if (!skb) {
1071                 skb_free_frag(data);
1072                 return NULL;
1073         }
1074
1075         skb_reserve(skb, bp->rx_offset);
1076         skb_put(skb, offset_and_len & 0xffff);
1077         return skb;
1078 }
1079
1080 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1081                                struct bnxt_cp_ring_info *cpr,
1082                                struct skb_shared_info *shinfo,
1083                                u16 idx, u32 agg_bufs, bool tpa,
1084                                struct xdp_buff *xdp)
1085 {
1086         struct bnxt_napi *bnapi = cpr->bnapi;
1087         struct pci_dev *pdev = bp->pdev;
1088         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1089         u16 prod = rxr->rx_agg_prod;
1090         u32 i, total_frag_len = 0;
1091         bool p5_tpa = false;
1092
1093         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1094                 p5_tpa = true;
1095
1096         for (i = 0; i < agg_bufs; i++) {
1097                 skb_frag_t *frag = &shinfo->frags[i];
1098                 u16 cons, frag_len;
1099                 struct rx_agg_cmp *agg;
1100                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1101                 struct page *page;
1102                 dma_addr_t mapping;
1103
1104                 if (p5_tpa)
1105                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1106                 else
1107                         agg = bnxt_get_agg(bp, cpr, idx, i);
1108                 cons = agg->rx_agg_cmp_opaque;
1109                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1110                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1111
1112                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1113                 skb_frag_off_set(frag, cons_rx_buf->offset);
1114                 skb_frag_size_set(frag, frag_len);
1115                 __skb_frag_set_page(frag, cons_rx_buf->page);
1116                 shinfo->nr_frags = i + 1;
1117                 __clear_bit(cons, rxr->rx_agg_bmap);
1118
1119                 /* It is possible for bnxt_alloc_rx_page() to allocate
1120                  * a sw_prod index that equals the cons index, so we
1121                  * need to clear the cons entry now.
1122                  */
1123                 mapping = cons_rx_buf->mapping;
1124                 page = cons_rx_buf->page;
1125                 cons_rx_buf->page = NULL;
1126
1127                 if (xdp && page_is_pfmemalloc(page))
1128                         xdp_buff_set_frag_pfmemalloc(xdp);
1129
1130                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1131                         unsigned int nr_frags;
1132
1133                         nr_frags = --shinfo->nr_frags;
1134                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1135                         cons_rx_buf->page = page;
1136
1137                         /* Update prod since possibly some pages have been
1138                          * allocated already.
1139                          */
1140                         rxr->rx_agg_prod = prod;
1141                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1142                         return 0;
1143                 }
1144
1145                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1146                                      bp->rx_dir,
1147                                      DMA_ATTR_WEAK_ORDERING);
1148
1149                 total_frag_len += frag_len;
1150                 prod = NEXT_RX_AGG(prod);
1151         }
1152         rxr->rx_agg_prod = prod;
1153         return total_frag_len;
1154 }
1155
1156 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1157                                              struct bnxt_cp_ring_info *cpr,
1158                                              struct sk_buff *skb, u16 idx,
1159                                              u32 agg_bufs, bool tpa)
1160 {
1161         struct skb_shared_info *shinfo = skb_shinfo(skb);
1162         u32 total_frag_len = 0;
1163
1164         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1165                                              agg_bufs, tpa, NULL);
1166         if (!total_frag_len) {
1167                 dev_kfree_skb(skb);
1168                 return NULL;
1169         }
1170
1171         skb->data_len += total_frag_len;
1172         skb->len += total_frag_len;
1173         skb->truesize += PAGE_SIZE * agg_bufs;
1174         return skb;
1175 }
1176
1177 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1178                                  struct bnxt_cp_ring_info *cpr,
1179                                  struct xdp_buff *xdp, u16 idx,
1180                                  u32 agg_bufs, bool tpa)
1181 {
1182         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1183         u32 total_frag_len = 0;
1184
1185         if (!xdp_buff_has_frags(xdp))
1186                 shinfo->nr_frags = 0;
1187
1188         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1189                                              idx, agg_bufs, tpa, xdp);
1190         if (total_frag_len) {
1191                 xdp_buff_set_frags_flag(xdp);
1192                 shinfo->nr_frags = agg_bufs;
1193                 shinfo->xdp_frags_size = total_frag_len;
1194         }
1195         return total_frag_len;
1196 }
1197
1198 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1199                                u8 agg_bufs, u32 *raw_cons)
1200 {
1201         u16 last;
1202         struct rx_agg_cmp *agg;
1203
1204         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1205         last = RING_CMP(*raw_cons);
1206         agg = (struct rx_agg_cmp *)
1207                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1208         return RX_AGG_CMP_VALID(agg, *raw_cons);
1209 }
1210
1211 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1212                                             unsigned int len,
1213                                             dma_addr_t mapping)
1214 {
1215         struct bnxt *bp = bnapi->bp;
1216         struct pci_dev *pdev = bp->pdev;
1217         struct sk_buff *skb;
1218
1219         skb = napi_alloc_skb(&bnapi->napi, len);
1220         if (!skb)
1221                 return NULL;
1222
1223         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1224                                 bp->rx_dir);
1225
1226         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1227                len + NET_IP_ALIGN);
1228
1229         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1230                                    bp->rx_dir);
1231
1232         skb_put(skb, len);
1233         return skb;
1234 }
1235
1236 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1237                            u32 *raw_cons, void *cmp)
1238 {
1239         struct rx_cmp *rxcmp = cmp;
1240         u32 tmp_raw_cons = *raw_cons;
1241         u8 cmp_type, agg_bufs = 0;
1242
1243         cmp_type = RX_CMP_TYPE(rxcmp);
1244
1245         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1246                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1247                             RX_CMP_AGG_BUFS) >>
1248                            RX_CMP_AGG_BUFS_SHIFT;
1249         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1250                 struct rx_tpa_end_cmp *tpa_end = cmp;
1251
1252                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1253                         return 0;
1254
1255                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1256         }
1257
1258         if (agg_bufs) {
1259                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1260                         return -EBUSY;
1261         }
1262         *raw_cons = tmp_raw_cons;
1263         return 0;
1264 }
1265
1266 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1267 {
1268         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1269                 return;
1270
1271         if (BNXT_PF(bp))
1272                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1273         else
1274                 schedule_delayed_work(&bp->fw_reset_task, delay);
1275 }
1276
1277 static void bnxt_queue_sp_work(struct bnxt *bp)
1278 {
1279         if (BNXT_PF(bp))
1280                 queue_work(bnxt_pf_wq, &bp->sp_task);
1281         else
1282                 schedule_work(&bp->sp_task);
1283 }
1284
1285 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1286 {
1287         if (!rxr->bnapi->in_reset) {
1288                 rxr->bnapi->in_reset = true;
1289                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1290                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1291                 else
1292                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1293                 bnxt_queue_sp_work(bp);
1294         }
1295         rxr->rx_next_cons = 0xffff;
1296 }
1297
1298 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1299 {
1300         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1301         u16 idx = agg_id & MAX_TPA_P5_MASK;
1302
1303         if (test_bit(idx, map->agg_idx_bmap))
1304                 idx = find_first_zero_bit(map->agg_idx_bmap,
1305                                           BNXT_AGG_IDX_BMAP_SIZE);
1306         __set_bit(idx, map->agg_idx_bmap);
1307         map->agg_id_tbl[agg_id] = idx;
1308         return idx;
1309 }
1310
1311 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1312 {
1313         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1314
1315         __clear_bit(idx, map->agg_idx_bmap);
1316 }
1317
1318 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1319 {
1320         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1321
1322         return map->agg_id_tbl[agg_id];
1323 }
1324
1325 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1326                            struct rx_tpa_start_cmp *tpa_start,
1327                            struct rx_tpa_start_cmp_ext *tpa_start1)
1328 {
1329         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1330         struct bnxt_tpa_info *tpa_info;
1331         u16 cons, prod, agg_id;
1332         struct rx_bd *prod_bd;
1333         dma_addr_t mapping;
1334
1335         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1336                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1337                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1338         } else {
1339                 agg_id = TPA_START_AGG_ID(tpa_start);
1340         }
1341         cons = tpa_start->rx_tpa_start_cmp_opaque;
1342         prod = rxr->rx_prod;
1343         cons_rx_buf = &rxr->rx_buf_ring[cons];
1344         prod_rx_buf = &rxr->rx_buf_ring[prod];
1345         tpa_info = &rxr->rx_tpa[agg_id];
1346
1347         if (unlikely(cons != rxr->rx_next_cons ||
1348                      TPA_START_ERROR(tpa_start))) {
1349                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1350                             cons, rxr->rx_next_cons,
1351                             TPA_START_ERROR_CODE(tpa_start1));
1352                 bnxt_sched_reset(bp, rxr);
1353                 return;
1354         }
1355         /* Store cfa_code in tpa_info to use in tpa_end
1356          * completion processing.
1357          */
1358         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1359         prod_rx_buf->data = tpa_info->data;
1360         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1361
1362         mapping = tpa_info->mapping;
1363         prod_rx_buf->mapping = mapping;
1364
1365         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1366
1367         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1368
1369         tpa_info->data = cons_rx_buf->data;
1370         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1371         cons_rx_buf->data = NULL;
1372         tpa_info->mapping = cons_rx_buf->mapping;
1373
1374         tpa_info->len =
1375                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1376                                 RX_TPA_START_CMP_LEN_SHIFT;
1377         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1378                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1379
1380                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1381                 tpa_info->gso_type = SKB_GSO_TCPV4;
1382                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1383                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1384                         tpa_info->gso_type = SKB_GSO_TCPV6;
1385                 tpa_info->rss_hash =
1386                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1387         } else {
1388                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1389                 tpa_info->gso_type = 0;
1390                 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1391         }
1392         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1393         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1394         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1395         tpa_info->agg_count = 0;
1396
1397         rxr->rx_prod = NEXT_RX(prod);
1398         cons = NEXT_RX(cons);
1399         rxr->rx_next_cons = NEXT_RX(cons);
1400         cons_rx_buf = &rxr->rx_buf_ring[cons];
1401
1402         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1403         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1404         cons_rx_buf->data = NULL;
1405 }
1406
1407 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1408 {
1409         if (agg_bufs)
1410                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1411 }
1412
1413 #ifdef CONFIG_INET
1414 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1415 {
1416         struct udphdr *uh = NULL;
1417
1418         if (ip_proto == htons(ETH_P_IP)) {
1419                 struct iphdr *iph = (struct iphdr *)skb->data;
1420
1421                 if (iph->protocol == IPPROTO_UDP)
1422                         uh = (struct udphdr *)(iph + 1);
1423         } else {
1424                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1425
1426                 if (iph->nexthdr == IPPROTO_UDP)
1427                         uh = (struct udphdr *)(iph + 1);
1428         }
1429         if (uh) {
1430                 if (uh->check)
1431                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1432                 else
1433                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1434         }
1435 }
1436 #endif
1437
1438 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1439                                            int payload_off, int tcp_ts,
1440                                            struct sk_buff *skb)
1441 {
1442 #ifdef CONFIG_INET
1443         struct tcphdr *th;
1444         int len, nw_off;
1445         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1446         u32 hdr_info = tpa_info->hdr_info;
1447         bool loopback = false;
1448
1449         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1450         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1451         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1452
1453         /* If the packet is an internal loopback packet, the offsets will
1454          * have an extra 4 bytes.
1455          */
1456         if (inner_mac_off == 4) {
1457                 loopback = true;
1458         } else if (inner_mac_off > 4) {
1459                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1460                                             ETH_HLEN - 2));
1461
1462                 /* We only support inner iPv4/ipv6.  If we don't see the
1463                  * correct protocol ID, it must be a loopback packet where
1464                  * the offsets are off by 4.
1465                  */
1466                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1467                         loopback = true;
1468         }
1469         if (loopback) {
1470                 /* internal loopback packet, subtract all offsets by 4 */
1471                 inner_ip_off -= 4;
1472                 inner_mac_off -= 4;
1473                 outer_ip_off -= 4;
1474         }
1475
1476         nw_off = inner_ip_off - ETH_HLEN;
1477         skb_set_network_header(skb, nw_off);
1478         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1479                 struct ipv6hdr *iph = ipv6_hdr(skb);
1480
1481                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1482                 len = skb->len - skb_transport_offset(skb);
1483                 th = tcp_hdr(skb);
1484                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1485         } else {
1486                 struct iphdr *iph = ip_hdr(skb);
1487
1488                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1489                 len = skb->len - skb_transport_offset(skb);
1490                 th = tcp_hdr(skb);
1491                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1492         }
1493
1494         if (inner_mac_off) { /* tunnel */
1495                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1496                                             ETH_HLEN - 2));
1497
1498                 bnxt_gro_tunnel(skb, proto);
1499         }
1500 #endif
1501         return skb;
1502 }
1503
1504 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1505                                            int payload_off, int tcp_ts,
1506                                            struct sk_buff *skb)
1507 {
1508 #ifdef CONFIG_INET
1509         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1510         u32 hdr_info = tpa_info->hdr_info;
1511         int iphdr_len, nw_off;
1512
1513         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1514         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1515         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1516
1517         nw_off = inner_ip_off - ETH_HLEN;
1518         skb_set_network_header(skb, nw_off);
1519         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1520                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1521         skb_set_transport_header(skb, nw_off + iphdr_len);
1522
1523         if (inner_mac_off) { /* tunnel */
1524                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1525                                             ETH_HLEN - 2));
1526
1527                 bnxt_gro_tunnel(skb, proto);
1528         }
1529 #endif
1530         return skb;
1531 }
1532
1533 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1534 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1535
1536 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1537                                            int payload_off, int tcp_ts,
1538                                            struct sk_buff *skb)
1539 {
1540 #ifdef CONFIG_INET
1541         struct tcphdr *th;
1542         int len, nw_off, tcp_opt_len = 0;
1543
1544         if (tcp_ts)
1545                 tcp_opt_len = 12;
1546
1547         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1548                 struct iphdr *iph;
1549
1550                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1551                          ETH_HLEN;
1552                 skb_set_network_header(skb, nw_off);
1553                 iph = ip_hdr(skb);
1554                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1555                 len = skb->len - skb_transport_offset(skb);
1556                 th = tcp_hdr(skb);
1557                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1558         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1559                 struct ipv6hdr *iph;
1560
1561                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1562                          ETH_HLEN;
1563                 skb_set_network_header(skb, nw_off);
1564                 iph = ipv6_hdr(skb);
1565                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1566                 len = skb->len - skb_transport_offset(skb);
1567                 th = tcp_hdr(skb);
1568                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1569         } else {
1570                 dev_kfree_skb_any(skb);
1571                 return NULL;
1572         }
1573
1574         if (nw_off) /* tunnel */
1575                 bnxt_gro_tunnel(skb, skb->protocol);
1576 #endif
1577         return skb;
1578 }
1579
1580 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1581                                            struct bnxt_tpa_info *tpa_info,
1582                                            struct rx_tpa_end_cmp *tpa_end,
1583                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1584                                            struct sk_buff *skb)
1585 {
1586 #ifdef CONFIG_INET
1587         int payload_off;
1588         u16 segs;
1589
1590         segs = TPA_END_TPA_SEGS(tpa_end);
1591         if (segs == 1)
1592                 return skb;
1593
1594         NAPI_GRO_CB(skb)->count = segs;
1595         skb_shinfo(skb)->gso_size =
1596                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1597         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1598         if (bp->flags & BNXT_FLAG_CHIP_P5)
1599                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1600         else
1601                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1602         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1603         if (likely(skb))
1604                 tcp_gro_complete(skb);
1605 #endif
1606         return skb;
1607 }
1608
1609 /* Given the cfa_code of a received packet determine which
1610  * netdev (vf-rep or PF) the packet is destined to.
1611  */
1612 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1613 {
1614         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1615
1616         /* if vf-rep dev is NULL, the must belongs to the PF */
1617         return dev ? dev : bp->dev;
1618 }
1619
1620 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1621                                            struct bnxt_cp_ring_info *cpr,
1622                                            u32 *raw_cons,
1623                                            struct rx_tpa_end_cmp *tpa_end,
1624                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1625                                            u8 *event)
1626 {
1627         struct bnxt_napi *bnapi = cpr->bnapi;
1628         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1629         u8 *data_ptr, agg_bufs;
1630         unsigned int len;
1631         struct bnxt_tpa_info *tpa_info;
1632         dma_addr_t mapping;
1633         struct sk_buff *skb;
1634         u16 idx = 0, agg_id;
1635         void *data;
1636         bool gro;
1637
1638         if (unlikely(bnapi->in_reset)) {
1639                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1640
1641                 if (rc < 0)
1642                         return ERR_PTR(-EBUSY);
1643                 return NULL;
1644         }
1645
1646         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1647                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1648                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1649                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1650                 tpa_info = &rxr->rx_tpa[agg_id];
1651                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1652                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1653                                     agg_bufs, tpa_info->agg_count);
1654                         agg_bufs = tpa_info->agg_count;
1655                 }
1656                 tpa_info->agg_count = 0;
1657                 *event |= BNXT_AGG_EVENT;
1658                 bnxt_free_agg_idx(rxr, agg_id);
1659                 idx = agg_id;
1660                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1661         } else {
1662                 agg_id = TPA_END_AGG_ID(tpa_end);
1663                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1664                 tpa_info = &rxr->rx_tpa[agg_id];
1665                 idx = RING_CMP(*raw_cons);
1666                 if (agg_bufs) {
1667                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1668                                 return ERR_PTR(-EBUSY);
1669
1670                         *event |= BNXT_AGG_EVENT;
1671                         idx = NEXT_CMP(idx);
1672                 }
1673                 gro = !!TPA_END_GRO(tpa_end);
1674         }
1675         data = tpa_info->data;
1676         data_ptr = tpa_info->data_ptr;
1677         prefetch(data_ptr);
1678         len = tpa_info->len;
1679         mapping = tpa_info->mapping;
1680
1681         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1682                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1683                 if (agg_bufs > MAX_SKB_FRAGS)
1684                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1685                                     agg_bufs, (int)MAX_SKB_FRAGS);
1686                 return NULL;
1687         }
1688
1689         if (len <= bp->rx_copy_thresh) {
1690                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1691                 if (!skb) {
1692                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1693                         cpr->sw_stats.rx.rx_oom_discards += 1;
1694                         return NULL;
1695                 }
1696         } else {
1697                 u8 *new_data;
1698                 dma_addr_t new_mapping;
1699
1700                 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1701                 if (!new_data) {
1702                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1703                         cpr->sw_stats.rx.rx_oom_discards += 1;
1704                         return NULL;
1705                 }
1706
1707                 tpa_info->data = new_data;
1708                 tpa_info->data_ptr = new_data + bp->rx_offset;
1709                 tpa_info->mapping = new_mapping;
1710
1711                 skb = build_skb(data, bp->rx_buf_size);
1712                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1713                                        bp->rx_buf_use_size, bp->rx_dir,
1714                                        DMA_ATTR_WEAK_ORDERING);
1715
1716                 if (!skb) {
1717                         skb_free_frag(data);
1718                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1719                         cpr->sw_stats.rx.rx_oom_discards += 1;
1720                         return NULL;
1721                 }
1722                 skb_reserve(skb, bp->rx_offset);
1723                 skb_put(skb, len);
1724         }
1725
1726         if (agg_bufs) {
1727                 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1728                 if (!skb) {
1729                         /* Page reuse already handled by bnxt_rx_pages(). */
1730                         cpr->sw_stats.rx.rx_oom_discards += 1;
1731                         return NULL;
1732                 }
1733         }
1734
1735         skb->protocol =
1736                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1737
1738         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1739                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1740
1741         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1742             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1743                 __be16 vlan_proto = htons(tpa_info->metadata >>
1744                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
1745                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1746
1747                 if (eth_type_vlan(vlan_proto)) {
1748                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1749                 } else {
1750                         dev_kfree_skb(skb);
1751                         return NULL;
1752                 }
1753         }
1754
1755         skb_checksum_none_assert(skb);
1756         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1757                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1758                 skb->csum_level =
1759                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1760         }
1761
1762         if (gro)
1763                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1764
1765         return skb;
1766 }
1767
1768 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1769                          struct rx_agg_cmp *rx_agg)
1770 {
1771         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1772         struct bnxt_tpa_info *tpa_info;
1773
1774         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1775         tpa_info = &rxr->rx_tpa[agg_id];
1776         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1777         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1778 }
1779
1780 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1781                              struct sk_buff *skb)
1782 {
1783         if (skb->dev != bp->dev) {
1784                 /* this packet belongs to a vf-rep */
1785                 bnxt_vf_rep_rx(bp, skb);
1786                 return;
1787         }
1788         skb_record_rx_queue(skb, bnapi->index);
1789         napi_gro_receive(&bnapi->napi, skb);
1790 }
1791
1792 /* returns the following:
1793  * 1       - 1 packet successfully received
1794  * 0       - successful TPA_START, packet not completed yet
1795  * -EBUSY  - completion ring does not have all the agg buffers yet
1796  * -ENOMEM - packet aborted due to out of memory
1797  * -EIO    - packet aborted due to hw error indicated in BD
1798  */
1799 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1800                        u32 *raw_cons, u8 *event)
1801 {
1802         struct bnxt_napi *bnapi = cpr->bnapi;
1803         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1804         struct net_device *dev = bp->dev;
1805         struct rx_cmp *rxcmp;
1806         struct rx_cmp_ext *rxcmp1;
1807         u32 tmp_raw_cons = *raw_cons;
1808         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1809         struct bnxt_sw_rx_bd *rx_buf;
1810         unsigned int len;
1811         u8 *data_ptr, agg_bufs, cmp_type;
1812         bool xdp_active = false;
1813         dma_addr_t dma_addr;
1814         struct sk_buff *skb;
1815         struct xdp_buff xdp;
1816         u32 flags, misc;
1817         void *data;
1818         int rc = 0;
1819
1820         rxcmp = (struct rx_cmp *)
1821                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1822
1823         cmp_type = RX_CMP_TYPE(rxcmp);
1824
1825         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1826                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1827                 goto next_rx_no_prod_no_len;
1828         }
1829
1830         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1831         cp_cons = RING_CMP(tmp_raw_cons);
1832         rxcmp1 = (struct rx_cmp_ext *)
1833                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1834
1835         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1836                 return -EBUSY;
1837
1838         /* The valid test of the entry must be done first before
1839          * reading any further.
1840          */
1841         dma_rmb();
1842         prod = rxr->rx_prod;
1843
1844         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1845                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1846                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1847
1848                 *event |= BNXT_RX_EVENT;
1849                 goto next_rx_no_prod_no_len;
1850
1851         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1852                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1853                                    (struct rx_tpa_end_cmp *)rxcmp,
1854                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1855
1856                 if (IS_ERR(skb))
1857                         return -EBUSY;
1858
1859                 rc = -ENOMEM;
1860                 if (likely(skb)) {
1861                         bnxt_deliver_skb(bp, bnapi, skb);
1862                         rc = 1;
1863                 }
1864                 *event |= BNXT_RX_EVENT;
1865                 goto next_rx_no_prod_no_len;
1866         }
1867
1868         cons = rxcmp->rx_cmp_opaque;
1869         if (unlikely(cons != rxr->rx_next_cons)) {
1870                 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1871
1872                 /* 0xffff is forced error, don't print it */
1873                 if (rxr->rx_next_cons != 0xffff)
1874                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1875                                     cons, rxr->rx_next_cons);
1876                 bnxt_sched_reset(bp, rxr);
1877                 if (rc1)
1878                         return rc1;
1879                 goto next_rx_no_prod_no_len;
1880         }
1881         rx_buf = &rxr->rx_buf_ring[cons];
1882         data = rx_buf->data;
1883         data_ptr = rx_buf->data_ptr;
1884         prefetch(data_ptr);
1885
1886         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1887         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1888
1889         if (agg_bufs) {
1890                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1891                         return -EBUSY;
1892
1893                 cp_cons = NEXT_CMP(cp_cons);
1894                 *event |= BNXT_AGG_EVENT;
1895         }
1896         *event |= BNXT_RX_EVENT;
1897
1898         rx_buf->data = NULL;
1899         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1900                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1901
1902                 bnxt_reuse_rx_data(rxr, cons, data);
1903                 if (agg_bufs)
1904                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1905                                                false);
1906
1907                 rc = -EIO;
1908                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1909                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1910                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1911                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1912                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1913                                                  rx_err);
1914                                 bnxt_sched_reset(bp, rxr);
1915                         }
1916                 }
1917                 goto next_rx_no_len;
1918         }
1919
1920         flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
1921         len = flags >> RX_CMP_LEN_SHIFT;
1922         dma_addr = rx_buf->mapping;
1923
1924         if (bnxt_xdp_attached(bp, rxr)) {
1925                 bnxt_xdp_buff_init(bp, rxr, cons, &data_ptr, &len, &xdp);
1926                 if (agg_bufs) {
1927                         u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
1928                                                              cp_cons, agg_bufs,
1929                                                              false);
1930                         if (!frag_len) {
1931                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1932                                 rc = -ENOMEM;
1933                                 goto next_rx;
1934                         }
1935                 }
1936                 xdp_active = true;
1937         }
1938
1939         if (xdp_active) {
1940                 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &len, event)) {
1941                         rc = 1;
1942                         goto next_rx;
1943                 }
1944         }
1945
1946         if (len <= bp->rx_copy_thresh) {
1947                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1948                 bnxt_reuse_rx_data(rxr, cons, data);
1949                 if (!skb) {
1950                         if (agg_bufs) {
1951                                 if (!xdp_active)
1952                                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1953                                                                agg_bufs, false);
1954                                 else
1955                                         bnxt_xdp_buff_frags_free(rxr, &xdp);
1956                         }
1957                         cpr->sw_stats.rx.rx_oom_discards += 1;
1958                         rc = -ENOMEM;
1959                         goto next_rx;
1960                 }
1961         } else {
1962                 u32 payload;
1963
1964                 if (rx_buf->data_ptr == data_ptr)
1965                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1966                 else
1967                         payload = 0;
1968                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1969                                       payload | len);
1970                 if (!skb) {
1971                         cpr->sw_stats.rx.rx_oom_discards += 1;
1972                         rc = -ENOMEM;
1973                         goto next_rx;
1974                 }
1975         }
1976
1977         if (agg_bufs) {
1978                 if (!xdp_active) {
1979                         skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
1980                         if (!skb) {
1981                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1982                                 rc = -ENOMEM;
1983                                 goto next_rx;
1984                         }
1985                 } else {
1986                         skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
1987                         if (!skb) {
1988                                 /* we should be able to free the old skb here */
1989                                 bnxt_xdp_buff_frags_free(rxr, &xdp);
1990                                 cpr->sw_stats.rx.rx_oom_discards += 1;
1991                                 rc = -ENOMEM;
1992                                 goto next_rx;
1993                         }
1994                 }
1995         }
1996
1997         if (RX_CMP_HASH_VALID(rxcmp)) {
1998                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1999                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
2000
2001                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
2002                 if (hash_type != 1 && hash_type != 3)
2003                         type = PKT_HASH_TYPE_L3;
2004                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
2005         }
2006
2007         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
2008         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
2009
2010         if ((rxcmp1->rx_cmp_flags2 &
2011              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
2012             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
2013                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
2014                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
2015                 __be16 vlan_proto = htons(meta_data >>
2016                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
2017
2018                 if (eth_type_vlan(vlan_proto)) {
2019                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
2020                 } else {
2021                         dev_kfree_skb(skb);
2022                         goto next_rx;
2023                 }
2024         }
2025
2026         skb_checksum_none_assert(skb);
2027         if (RX_CMP_L4_CS_OK(rxcmp1)) {
2028                 if (dev->features & NETIF_F_RXCSUM) {
2029                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2030                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2031                 }
2032         } else {
2033                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2034                         if (dev->features & NETIF_F_RXCSUM)
2035                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2036                 }
2037         }
2038
2039         if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) ==
2040                      RX_CMP_FLAGS_ITYPE_PTP_W_TS) || bp->ptp_all_rx_tstamp) {
2041                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2042                         u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
2043                         u64 ns, ts;
2044
2045                         if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2046                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2047
2048                                 spin_lock_bh(&ptp->ptp_lock);
2049                                 ns = timecounter_cyc2time(&ptp->tc, ts);
2050                                 spin_unlock_bh(&ptp->ptp_lock);
2051                                 memset(skb_hwtstamps(skb), 0,
2052                                        sizeof(*skb_hwtstamps(skb)));
2053                                 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2054                         }
2055                 }
2056         }
2057         bnxt_deliver_skb(bp, bnapi, skb);
2058         rc = 1;
2059
2060 next_rx:
2061         cpr->rx_packets += 1;
2062         cpr->rx_bytes += len;
2063
2064 next_rx_no_len:
2065         rxr->rx_prod = NEXT_RX(prod);
2066         rxr->rx_next_cons = NEXT_RX(cons);
2067
2068 next_rx_no_prod_no_len:
2069         *raw_cons = tmp_raw_cons;
2070
2071         return rc;
2072 }
2073
2074 /* In netpoll mode, if we are using a combined completion ring, we need to
2075  * discard the rx packets and recycle the buffers.
2076  */
2077 static int bnxt_force_rx_discard(struct bnxt *bp,
2078                                  struct bnxt_cp_ring_info *cpr,
2079                                  u32 *raw_cons, u8 *event)
2080 {
2081         u32 tmp_raw_cons = *raw_cons;
2082         struct rx_cmp_ext *rxcmp1;
2083         struct rx_cmp *rxcmp;
2084         u16 cp_cons;
2085         u8 cmp_type;
2086         int rc;
2087
2088         cp_cons = RING_CMP(tmp_raw_cons);
2089         rxcmp = (struct rx_cmp *)
2090                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2091
2092         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2093         cp_cons = RING_CMP(tmp_raw_cons);
2094         rxcmp1 = (struct rx_cmp_ext *)
2095                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2096
2097         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2098                 return -EBUSY;
2099
2100         /* The valid test of the entry must be done first before
2101          * reading any further.
2102          */
2103         dma_rmb();
2104         cmp_type = RX_CMP_TYPE(rxcmp);
2105         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
2106                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2107                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2108         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2109                 struct rx_tpa_end_cmp_ext *tpa_end1;
2110
2111                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2112                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2113                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2114         }
2115         rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2116         if (rc && rc != -EBUSY)
2117                 cpr->sw_stats.rx.rx_netpoll_discards += 1;
2118         return rc;
2119 }
2120
2121 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2122 {
2123         struct bnxt_fw_health *fw_health = bp->fw_health;
2124         u32 reg = fw_health->regs[reg_idx];
2125         u32 reg_type, reg_off, val = 0;
2126
2127         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2128         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2129         switch (reg_type) {
2130         case BNXT_FW_HEALTH_REG_TYPE_CFG:
2131                 pci_read_config_dword(bp->pdev, reg_off, &val);
2132                 break;
2133         case BNXT_FW_HEALTH_REG_TYPE_GRC:
2134                 reg_off = fw_health->mapped_regs[reg_idx];
2135                 fallthrough;
2136         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2137                 val = readl(bp->bar0 + reg_off);
2138                 break;
2139         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2140                 val = readl(bp->bar1 + reg_off);
2141                 break;
2142         }
2143         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2144                 val &= fw_health->fw_reset_inprog_reg_mask;
2145         return val;
2146 }
2147
2148 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2149 {
2150         int i;
2151
2152         for (i = 0; i < bp->rx_nr_rings; i++) {
2153                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2154                 struct bnxt_ring_grp_info *grp_info;
2155
2156                 grp_info = &bp->grp_info[grp_idx];
2157                 if (grp_info->agg_fw_ring_id == ring_id)
2158                         return grp_idx;
2159         }
2160         return INVALID_HW_RING_ID;
2161 }
2162
2163 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2164 {
2165         u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2166
2167         switch (err_type) {
2168         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2169                 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2170                            BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2171                 break;
2172         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2173                 netdev_warn(bp->dev, "Pause Storm detected!\n");
2174                 break;
2175         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2176                 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2177                 break;
2178         default:
2179                 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2180                            err_type);
2181                 break;
2182         }
2183 }
2184
2185 #define BNXT_GET_EVENT_PORT(data)       \
2186         ((data) &                       \
2187          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2188
2189 #define BNXT_EVENT_RING_TYPE(data2)     \
2190         ((data2) &                      \
2191          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2192
2193 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
2194         (BNXT_EVENT_RING_TYPE(data2) == \
2195          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2196
2197 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)        \
2198         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2199          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2200
2201 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)        \
2202         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2203          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2204
2205 #define BNXT_PHC_BITS   48
2206
2207 static int bnxt_async_event_process(struct bnxt *bp,
2208                                     struct hwrm_async_event_cmpl *cmpl)
2209 {
2210         u16 event_id = le16_to_cpu(cmpl->event_id);
2211         u32 data1 = le32_to_cpu(cmpl->event_data1);
2212         u32 data2 = le32_to_cpu(cmpl->event_data2);
2213
2214         netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2215                    event_id, data1, data2);
2216
2217         /* TODO CHIMP_FW: Define event id's for link change, error etc */
2218         switch (event_id) {
2219         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2220                 struct bnxt_link_info *link_info = &bp->link_info;
2221
2222                 if (BNXT_VF(bp))
2223                         goto async_event_process_exit;
2224
2225                 /* print unsupported speed warning in forced speed mode only */
2226                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2227                     (data1 & 0x20000)) {
2228                         u16 fw_speed = link_info->force_link_speed;
2229                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2230
2231                         if (speed != SPEED_UNKNOWN)
2232                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2233                                             speed);
2234                 }
2235                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2236         }
2237                 fallthrough;
2238         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2239         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2240                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2241                 fallthrough;
2242         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2243                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2244                 break;
2245         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2246                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2247                 break;
2248         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2249                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2250
2251                 if (BNXT_VF(bp))
2252                         break;
2253
2254                 if (bp->pf.port_id != port_id)
2255                         break;
2256
2257                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2258                 break;
2259         }
2260         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2261                 if (BNXT_PF(bp))
2262                         goto async_event_process_exit;
2263                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2264                 break;
2265         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2266                 char *type_str = "Solicited";
2267
2268                 if (!bp->fw_health)
2269                         goto async_event_process_exit;
2270
2271                 bp->fw_reset_timestamp = jiffies;
2272                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2273                 if (!bp->fw_reset_min_dsecs)
2274                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2275                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2276                 if (!bp->fw_reset_max_dsecs)
2277                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2278                 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2279                         set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2280                 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2281                         type_str = "Fatal";
2282                         bp->fw_health->fatalities++;
2283                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2284                 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2285                            EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2286                         type_str = "Non-fatal";
2287                         bp->fw_health->survivals++;
2288                         set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2289                 }
2290                 netif_warn(bp, hw, bp->dev,
2291                            "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2292                            type_str, data1, data2,
2293                            bp->fw_reset_min_dsecs * 100,
2294                            bp->fw_reset_max_dsecs * 100);
2295                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2296                 break;
2297         }
2298         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2299                 struct bnxt_fw_health *fw_health = bp->fw_health;
2300                 char *status_desc = "healthy";
2301                 u32 status;
2302
2303                 if (!fw_health)
2304                         goto async_event_process_exit;
2305
2306                 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2307                         fw_health->enabled = false;
2308                         netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2309                         break;
2310                 }
2311                 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2312                 fw_health->tmr_multiplier =
2313                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2314                                      bp->current_interval * 10);
2315                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2316                 if (!fw_health->enabled)
2317                         fw_health->last_fw_heartbeat =
2318                                 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2319                 fw_health->last_fw_reset_cnt =
2320                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2321                 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2322                 if (status != BNXT_FW_STATUS_HEALTHY)
2323                         status_desc = "unhealthy";
2324                 netif_info(bp, drv, bp->dev,
2325                            "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2326                            fw_health->primary ? "primary" : "backup", status,
2327                            status_desc, fw_health->last_fw_reset_cnt);
2328                 if (!fw_health->enabled) {
2329                         /* Make sure tmr_counter is set and visible to
2330                          * bnxt_health_check() before setting enabled to true.
2331                          */
2332                         smp_wmb();
2333                         fw_health->enabled = true;
2334                 }
2335                 goto async_event_process_exit;
2336         }
2337         case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2338                 netif_notice(bp, hw, bp->dev,
2339                              "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2340                              data1, data2);
2341                 goto async_event_process_exit;
2342         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2343                 struct bnxt_rx_ring_info *rxr;
2344                 u16 grp_idx;
2345
2346                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2347                         goto async_event_process_exit;
2348
2349                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2350                             BNXT_EVENT_RING_TYPE(data2), data1);
2351                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2352                         goto async_event_process_exit;
2353
2354                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2355                 if (grp_idx == INVALID_HW_RING_ID) {
2356                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2357                                     data1);
2358                         goto async_event_process_exit;
2359                 }
2360                 rxr = bp->bnapi[grp_idx]->rx_ring;
2361                 bnxt_sched_reset(bp, rxr);
2362                 goto async_event_process_exit;
2363         }
2364         case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2365                 struct bnxt_fw_health *fw_health = bp->fw_health;
2366
2367                 netif_notice(bp, hw, bp->dev,
2368                              "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2369                              data1, data2);
2370                 if (fw_health) {
2371                         fw_health->echo_req_data1 = data1;
2372                         fw_health->echo_req_data2 = data2;
2373                         set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2374                         break;
2375                 }
2376                 goto async_event_process_exit;
2377         }
2378         case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2379                 bnxt_ptp_pps_event(bp, data1, data2);
2380                 goto async_event_process_exit;
2381         }
2382         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2383                 bnxt_event_error_report(bp, data1, data2);
2384                 goto async_event_process_exit;
2385         }
2386         case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2387                 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2388                 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2389                         if (bp->fw_cap & BNXT_FW_CAP_PTP_RTC) {
2390                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2391                                 u64 ns;
2392
2393                                 spin_lock_bh(&ptp->ptp_lock);
2394                                 bnxt_ptp_update_current_time(bp);
2395                                 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2396                                        BNXT_PHC_BITS) | ptp->current_time);
2397                                 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2398                                 spin_unlock_bh(&ptp->ptp_lock);
2399                         }
2400                         break;
2401                 }
2402                 goto async_event_process_exit;
2403         }
2404         case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2405                 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2406
2407                 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2408                 goto async_event_process_exit;
2409         }
2410         default:
2411                 goto async_event_process_exit;
2412         }
2413         bnxt_queue_sp_work(bp);
2414 async_event_process_exit:
2415         bnxt_ulp_async_events(bp, cmpl);
2416         return 0;
2417 }
2418
2419 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2420 {
2421         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2422         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2423         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2424                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2425
2426         switch (cmpl_type) {
2427         case CMPL_BASE_TYPE_HWRM_DONE:
2428                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2429                 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2430                 break;
2431
2432         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2433                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2434
2435                 if ((vf_id < bp->pf.first_vf_id) ||
2436                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2437                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2438                                    vf_id);
2439                         return -EINVAL;
2440                 }
2441
2442                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2443                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2444                 bnxt_queue_sp_work(bp);
2445                 break;
2446
2447         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2448                 bnxt_async_event_process(bp,
2449                                          (struct hwrm_async_event_cmpl *)txcmp);
2450                 break;
2451
2452         default:
2453                 break;
2454         }
2455
2456         return 0;
2457 }
2458
2459 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2460 {
2461         struct bnxt_napi *bnapi = dev_instance;
2462         struct bnxt *bp = bnapi->bp;
2463         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2464         u32 cons = RING_CMP(cpr->cp_raw_cons);
2465
2466         cpr->event_ctr++;
2467         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2468         napi_schedule(&bnapi->napi);
2469         return IRQ_HANDLED;
2470 }
2471
2472 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2473 {
2474         u32 raw_cons = cpr->cp_raw_cons;
2475         u16 cons = RING_CMP(raw_cons);
2476         struct tx_cmp *txcmp;
2477
2478         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2479
2480         return TX_CMP_VALID(txcmp, raw_cons);
2481 }
2482
2483 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2484 {
2485         struct bnxt_napi *bnapi = dev_instance;
2486         struct bnxt *bp = bnapi->bp;
2487         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2488         u32 cons = RING_CMP(cpr->cp_raw_cons);
2489         u32 int_status;
2490
2491         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2492
2493         if (!bnxt_has_work(bp, cpr)) {
2494                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2495                 /* return if erroneous interrupt */
2496                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2497                         return IRQ_NONE;
2498         }
2499
2500         /* disable ring IRQ */
2501         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2502
2503         /* Return here if interrupt is shared and is disabled. */
2504         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2505                 return IRQ_HANDLED;
2506
2507         napi_schedule(&bnapi->napi);
2508         return IRQ_HANDLED;
2509 }
2510
2511 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2512                             int budget)
2513 {
2514         struct bnxt_napi *bnapi = cpr->bnapi;
2515         u32 raw_cons = cpr->cp_raw_cons;
2516         u32 cons;
2517         int tx_pkts = 0;
2518         int rx_pkts = 0;
2519         u8 event = 0;
2520         struct tx_cmp *txcmp;
2521
2522         cpr->has_more_work = 0;
2523         cpr->had_work_done = 1;
2524         while (1) {
2525                 int rc;
2526
2527                 cons = RING_CMP(raw_cons);
2528                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2529
2530                 if (!TX_CMP_VALID(txcmp, raw_cons))
2531                         break;
2532
2533                 /* The valid test of the entry must be done first before
2534                  * reading any further.
2535                  */
2536                 dma_rmb();
2537                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2538                         tx_pkts++;
2539                         /* return full budget so NAPI will complete. */
2540                         if (unlikely(tx_pkts >= bp->tx_wake_thresh)) {
2541                                 rx_pkts = budget;
2542                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2543                                 if (budget)
2544                                         cpr->has_more_work = 1;
2545                                 break;
2546                         }
2547                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2548                         if (likely(budget))
2549                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2550                         else
2551                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2552                                                            &event);
2553                         if (likely(rc >= 0))
2554                                 rx_pkts += rc;
2555                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2556                          * the NAPI budget.  Otherwise, we may potentially loop
2557                          * here forever if we consistently cannot allocate
2558                          * buffers.
2559                          */
2560                         else if (rc == -ENOMEM && budget)
2561                                 rx_pkts++;
2562                         else if (rc == -EBUSY)  /* partial completion */
2563                                 break;
2564                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2565                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2566                                     (TX_CMP_TYPE(txcmp) ==
2567                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2568                                     (TX_CMP_TYPE(txcmp) ==
2569                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2570                         bnxt_hwrm_handler(bp, txcmp);
2571                 }
2572                 raw_cons = NEXT_RAW_CMP(raw_cons);
2573
2574                 if (rx_pkts && rx_pkts == budget) {
2575                         cpr->has_more_work = 1;
2576                         break;
2577                 }
2578         }
2579
2580         if (event & BNXT_REDIRECT_EVENT)
2581                 xdp_do_flush();
2582
2583         if (event & BNXT_TX_EVENT) {
2584                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2585                 u16 prod = txr->tx_prod;
2586
2587                 /* Sync BD data before updating doorbell */
2588                 wmb();
2589
2590                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2591         }
2592
2593         cpr->cp_raw_cons = raw_cons;
2594         bnapi->tx_pkts += tx_pkts;
2595         bnapi->events |= event;
2596         return rx_pkts;
2597 }
2598
2599 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2600 {
2601         if (bnapi->tx_pkts) {
2602                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2603                 bnapi->tx_pkts = 0;
2604         }
2605
2606         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2607                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2608
2609                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2610         }
2611         if (bnapi->events & BNXT_AGG_EVENT) {
2612                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2613
2614                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2615         }
2616         bnapi->events = 0;
2617 }
2618
2619 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2620                           int budget)
2621 {
2622         struct bnxt_napi *bnapi = cpr->bnapi;
2623         int rx_pkts;
2624
2625         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2626
2627         /* ACK completion ring before freeing tx ring and producing new
2628          * buffers in rx/agg rings to prevent overflowing the completion
2629          * ring.
2630          */
2631         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2632
2633         __bnxt_poll_work_done(bp, bnapi);
2634         return rx_pkts;
2635 }
2636
2637 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2638 {
2639         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2640         struct bnxt *bp = bnapi->bp;
2641         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2642         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2643         struct tx_cmp *txcmp;
2644         struct rx_cmp_ext *rxcmp1;
2645         u32 cp_cons, tmp_raw_cons;
2646         u32 raw_cons = cpr->cp_raw_cons;
2647         u32 rx_pkts = 0;
2648         u8 event = 0;
2649
2650         while (1) {
2651                 int rc;
2652
2653                 cp_cons = RING_CMP(raw_cons);
2654                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2655
2656                 if (!TX_CMP_VALID(txcmp, raw_cons))
2657                         break;
2658
2659                 /* The valid test of the entry must be done first before
2660                  * reading any further.
2661                  */
2662                 dma_rmb();
2663                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2664                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2665                         cp_cons = RING_CMP(tmp_raw_cons);
2666                         rxcmp1 = (struct rx_cmp_ext *)
2667                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2668
2669                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2670                                 break;
2671
2672                         /* force an error to recycle the buffer */
2673                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2674                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2675
2676                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2677                         if (likely(rc == -EIO) && budget)
2678                                 rx_pkts++;
2679                         else if (rc == -EBUSY)  /* partial completion */
2680                                 break;
2681                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2682                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2683                         bnxt_hwrm_handler(bp, txcmp);
2684                 } else {
2685                         netdev_err(bp->dev,
2686                                    "Invalid completion received on special ring\n");
2687                 }
2688                 raw_cons = NEXT_RAW_CMP(raw_cons);
2689
2690                 if (rx_pkts == budget)
2691                         break;
2692         }
2693
2694         cpr->cp_raw_cons = raw_cons;
2695         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2696         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2697
2698         if (event & BNXT_AGG_EVENT)
2699                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2700
2701         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2702                 napi_complete_done(napi, rx_pkts);
2703                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2704         }
2705         return rx_pkts;
2706 }
2707
2708 static int bnxt_poll(struct napi_struct *napi, int budget)
2709 {
2710         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2711         struct bnxt *bp = bnapi->bp;
2712         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2713         int work_done = 0;
2714
2715         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2716                 napi_complete(napi);
2717                 return 0;
2718         }
2719         while (1) {
2720                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2721
2722                 if (work_done >= budget) {
2723                         if (!budget)
2724                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2725                         break;
2726                 }
2727
2728                 if (!bnxt_has_work(bp, cpr)) {
2729                         if (napi_complete_done(napi, work_done))
2730                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2731                         break;
2732                 }
2733         }
2734         if (bp->flags & BNXT_FLAG_DIM) {
2735                 struct dim_sample dim_sample = {};
2736
2737                 dim_update_sample(cpr->event_ctr,
2738                                   cpr->rx_packets,
2739                                   cpr->rx_bytes,
2740                                   &dim_sample);
2741                 net_dim(&cpr->dim, dim_sample);
2742         }
2743         return work_done;
2744 }
2745
2746 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2747 {
2748         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2749         int i, work_done = 0;
2750
2751         for (i = 0; i < 2; i++) {
2752                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2753
2754                 if (cpr2) {
2755                         work_done += __bnxt_poll_work(bp, cpr2,
2756                                                       budget - work_done);
2757                         cpr->has_more_work |= cpr2->has_more_work;
2758                 }
2759         }
2760         return work_done;
2761 }
2762
2763 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2764                                  u64 dbr_type)
2765 {
2766         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2767         int i;
2768
2769         for (i = 0; i < 2; i++) {
2770                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2771                 struct bnxt_db_info *db;
2772
2773                 if (cpr2 && cpr2->had_work_done) {
2774                         db = &cpr2->cp_db;
2775                         bnxt_writeq(bp, db->db_key64 | dbr_type |
2776                                     RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2777                         cpr2->had_work_done = 0;
2778                 }
2779         }
2780         __bnxt_poll_work_done(bp, bnapi);
2781 }
2782
2783 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2784 {
2785         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2786         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2787         struct bnxt_cp_ring_info *cpr_rx;
2788         u32 raw_cons = cpr->cp_raw_cons;
2789         struct bnxt *bp = bnapi->bp;
2790         struct nqe_cn *nqcmp;
2791         int work_done = 0;
2792         u32 cons;
2793
2794         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2795                 napi_complete(napi);
2796                 return 0;
2797         }
2798         if (cpr->has_more_work) {
2799                 cpr->has_more_work = 0;
2800                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2801         }
2802         while (1) {
2803                 cons = RING_CMP(raw_cons);
2804                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2805
2806                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2807                         if (cpr->has_more_work)
2808                                 break;
2809
2810                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2811                         cpr->cp_raw_cons = raw_cons;
2812                         if (napi_complete_done(napi, work_done))
2813                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2814                                                   cpr->cp_raw_cons);
2815                         goto poll_done;
2816                 }
2817
2818                 /* The valid test of the entry must be done first before
2819                  * reading any further.
2820                  */
2821                 dma_rmb();
2822
2823                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2824                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2825                         struct bnxt_cp_ring_info *cpr2;
2826
2827                         /* No more budget for RX work */
2828                         if (budget && work_done >= budget && idx == BNXT_RX_HDL)
2829                                 break;
2830
2831                         cpr2 = cpr->cp_ring_arr[idx];
2832                         work_done += __bnxt_poll_work(bp, cpr2,
2833                                                       budget - work_done);
2834                         cpr->has_more_work |= cpr2->has_more_work;
2835                 } else {
2836                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2837                 }
2838                 raw_cons = NEXT_RAW_CMP(raw_cons);
2839         }
2840         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2841         if (raw_cons != cpr->cp_raw_cons) {
2842                 cpr->cp_raw_cons = raw_cons;
2843                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2844         }
2845 poll_done:
2846         cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL];
2847         if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) {
2848                 struct dim_sample dim_sample = {};
2849
2850                 dim_update_sample(cpr->event_ctr,
2851                                   cpr_rx->rx_packets,
2852                                   cpr_rx->rx_bytes,
2853                                   &dim_sample);
2854                 net_dim(&cpr->dim, dim_sample);
2855         }
2856         return work_done;
2857 }
2858
2859 static void bnxt_free_tx_skbs(struct bnxt *bp)
2860 {
2861         int i, max_idx;
2862         struct pci_dev *pdev = bp->pdev;
2863
2864         if (!bp->tx_ring)
2865                 return;
2866
2867         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2868         for (i = 0; i < bp->tx_nr_rings; i++) {
2869                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2870                 int j;
2871
2872                 if (!txr->tx_buf_ring)
2873                         continue;
2874
2875                 for (j = 0; j < max_idx;) {
2876                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2877                         struct sk_buff *skb;
2878                         int k, last;
2879
2880                         if (i < bp->tx_nr_rings_xdp &&
2881                             tx_buf->action == XDP_REDIRECT) {
2882                                 dma_unmap_single(&pdev->dev,
2883                                         dma_unmap_addr(tx_buf, mapping),
2884                                         dma_unmap_len(tx_buf, len),
2885                                         DMA_TO_DEVICE);
2886                                 xdp_return_frame(tx_buf->xdpf);
2887                                 tx_buf->action = 0;
2888                                 tx_buf->xdpf = NULL;
2889                                 j++;
2890                                 continue;
2891                         }
2892
2893                         skb = tx_buf->skb;
2894                         if (!skb) {
2895                                 j++;
2896                                 continue;
2897                         }
2898
2899                         tx_buf->skb = NULL;
2900
2901                         if (tx_buf->is_push) {
2902                                 dev_kfree_skb(skb);
2903                                 j += 2;
2904                                 continue;
2905                         }
2906
2907                         dma_unmap_single(&pdev->dev,
2908                                          dma_unmap_addr(tx_buf, mapping),
2909                                          skb_headlen(skb),
2910                                          DMA_TO_DEVICE);
2911
2912                         last = tx_buf->nr_frags;
2913                         j += 2;
2914                         for (k = 0; k < last; k++, j++) {
2915                                 int ring_idx = j & bp->tx_ring_mask;
2916                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2917
2918                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2919                                 dma_unmap_page(
2920                                         &pdev->dev,
2921                                         dma_unmap_addr(tx_buf, mapping),
2922                                         skb_frag_size(frag), DMA_TO_DEVICE);
2923                         }
2924                         dev_kfree_skb(skb);
2925                 }
2926                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2927         }
2928 }
2929
2930 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2931 {
2932         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2933         struct pci_dev *pdev = bp->pdev;
2934         struct bnxt_tpa_idx_map *map;
2935         int i, max_idx, max_agg_idx;
2936
2937         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2938         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2939         if (!rxr->rx_tpa)
2940                 goto skip_rx_tpa_free;
2941
2942         for (i = 0; i < bp->max_tpa; i++) {
2943                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2944                 u8 *data = tpa_info->data;
2945
2946                 if (!data)
2947                         continue;
2948
2949                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2950                                        bp->rx_buf_use_size, bp->rx_dir,
2951                                        DMA_ATTR_WEAK_ORDERING);
2952
2953                 tpa_info->data = NULL;
2954
2955                 skb_free_frag(data);
2956         }
2957
2958 skip_rx_tpa_free:
2959         if (!rxr->rx_buf_ring)
2960                 goto skip_rx_buf_free;
2961
2962         for (i = 0; i < max_idx; i++) {
2963                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2964                 dma_addr_t mapping = rx_buf->mapping;
2965                 void *data = rx_buf->data;
2966
2967                 if (!data)
2968                         continue;
2969
2970                 rx_buf->data = NULL;
2971                 if (BNXT_RX_PAGE_MODE(bp)) {
2972                         mapping -= bp->rx_dma_offset;
2973                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2974                                              bp->rx_dir,
2975                                              DMA_ATTR_WEAK_ORDERING);
2976                         page_pool_recycle_direct(rxr->page_pool, data);
2977                 } else {
2978                         dma_unmap_single_attrs(&pdev->dev, mapping,
2979                                                bp->rx_buf_use_size, bp->rx_dir,
2980                                                DMA_ATTR_WEAK_ORDERING);
2981                         skb_free_frag(data);
2982                 }
2983         }
2984
2985 skip_rx_buf_free:
2986         if (!rxr->rx_agg_ring)
2987                 goto skip_rx_agg_free;
2988
2989         for (i = 0; i < max_agg_idx; i++) {
2990                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2991                 struct page *page = rx_agg_buf->page;
2992
2993                 if (!page)
2994                         continue;
2995
2996                 if (BNXT_RX_PAGE_MODE(bp)) {
2997                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2998                                              BNXT_RX_PAGE_SIZE, bp->rx_dir,
2999                                              DMA_ATTR_WEAK_ORDERING);
3000                         rx_agg_buf->page = NULL;
3001                         __clear_bit(i, rxr->rx_agg_bmap);
3002
3003                         page_pool_recycle_direct(rxr->page_pool, page);
3004                 } else {
3005                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
3006                                              BNXT_RX_PAGE_SIZE, DMA_FROM_DEVICE,
3007                                              DMA_ATTR_WEAK_ORDERING);
3008                         rx_agg_buf->page = NULL;
3009                         __clear_bit(i, rxr->rx_agg_bmap);
3010
3011                         __free_page(page);
3012                 }
3013         }
3014
3015 skip_rx_agg_free:
3016         if (rxr->rx_page) {
3017                 __free_page(rxr->rx_page);
3018                 rxr->rx_page = NULL;
3019         }
3020         map = rxr->rx_tpa_idx_map;
3021         if (map)
3022                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3023 }
3024
3025 static void bnxt_free_rx_skbs(struct bnxt *bp)
3026 {
3027         int i;
3028
3029         if (!bp->rx_ring)
3030                 return;
3031
3032         for (i = 0; i < bp->rx_nr_rings; i++)
3033                 bnxt_free_one_rx_ring_skbs(bp, i);
3034 }
3035
3036 static void bnxt_free_skbs(struct bnxt *bp)
3037 {
3038         bnxt_free_tx_skbs(bp);
3039         bnxt_free_rx_skbs(bp);
3040 }
3041
3042 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
3043 {
3044         u8 init_val = mem_init->init_val;
3045         u16 offset = mem_init->offset;
3046         u8 *p2 = p;
3047         int i;
3048
3049         if (!init_val)
3050                 return;
3051         if (offset == BNXT_MEM_INVALID_OFFSET) {
3052                 memset(p, init_val, len);
3053                 return;
3054         }
3055         for (i = 0; i < len; i += mem_init->size)
3056                 *(p2 + i + offset) = init_val;
3057 }
3058
3059 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3060 {
3061         struct pci_dev *pdev = bp->pdev;
3062         int i;
3063
3064         if (!rmem->pg_arr)
3065                 goto skip_pages;
3066
3067         for (i = 0; i < rmem->nr_pages; i++) {
3068                 if (!rmem->pg_arr[i])
3069                         continue;
3070
3071                 dma_free_coherent(&pdev->dev, rmem->page_size,
3072                                   rmem->pg_arr[i], rmem->dma_arr[i]);
3073
3074                 rmem->pg_arr[i] = NULL;
3075         }
3076 skip_pages:
3077         if (rmem->pg_tbl) {
3078                 size_t pg_tbl_size = rmem->nr_pages * 8;
3079
3080                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3081                         pg_tbl_size = rmem->page_size;
3082                 dma_free_coherent(&pdev->dev, pg_tbl_size,
3083                                   rmem->pg_tbl, rmem->pg_tbl_map);
3084                 rmem->pg_tbl = NULL;
3085         }
3086         if (rmem->vmem_size && *rmem->vmem) {
3087                 vfree(*rmem->vmem);
3088                 *rmem->vmem = NULL;
3089         }
3090 }
3091
3092 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3093 {
3094         struct pci_dev *pdev = bp->pdev;
3095         u64 valid_bit = 0;
3096         int i;
3097
3098         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3099                 valid_bit = PTU_PTE_VALID;
3100         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3101                 size_t pg_tbl_size = rmem->nr_pages * 8;
3102
3103                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3104                         pg_tbl_size = rmem->page_size;
3105                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3106                                                   &rmem->pg_tbl_map,
3107                                                   GFP_KERNEL);
3108                 if (!rmem->pg_tbl)
3109                         return -ENOMEM;
3110         }
3111
3112         for (i = 0; i < rmem->nr_pages; i++) {
3113                 u64 extra_bits = valid_bit;
3114
3115                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3116                                                      rmem->page_size,
3117                                                      &rmem->dma_arr[i],
3118                                                      GFP_KERNEL);
3119                 if (!rmem->pg_arr[i])
3120                         return -ENOMEM;
3121
3122                 if (rmem->mem_init)
3123                         bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
3124                                           rmem->page_size);
3125                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3126                         if (i == rmem->nr_pages - 2 &&
3127                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3128                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3129                         else if (i == rmem->nr_pages - 1 &&
3130                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3131                                 extra_bits |= PTU_PTE_LAST;
3132                         rmem->pg_tbl[i] =
3133                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3134                 }
3135         }
3136
3137         if (rmem->vmem_size) {
3138                 *rmem->vmem = vzalloc(rmem->vmem_size);
3139                 if (!(*rmem->vmem))
3140                         return -ENOMEM;
3141         }
3142         return 0;
3143 }
3144
3145 static void bnxt_free_tpa_info(struct bnxt *bp)
3146 {
3147         int i;
3148
3149         for (i = 0; i < bp->rx_nr_rings; i++) {
3150                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3151
3152                 kfree(rxr->rx_tpa_idx_map);
3153                 rxr->rx_tpa_idx_map = NULL;
3154                 if (rxr->rx_tpa) {
3155                         kfree(rxr->rx_tpa[0].agg_arr);
3156                         rxr->rx_tpa[0].agg_arr = NULL;
3157                 }
3158                 kfree(rxr->rx_tpa);
3159                 rxr->rx_tpa = NULL;
3160         }
3161 }
3162
3163 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3164 {
3165         int i, j, total_aggs = 0;
3166
3167         bp->max_tpa = MAX_TPA;
3168         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3169                 if (!bp->max_tpa_v2)
3170                         return 0;
3171                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3172                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
3173         }
3174
3175         for (i = 0; i < bp->rx_nr_rings; i++) {
3176                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3177                 struct rx_agg_cmp *agg;
3178
3179                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3180                                       GFP_KERNEL);
3181                 if (!rxr->rx_tpa)
3182                         return -ENOMEM;
3183
3184                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3185                         continue;
3186                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
3187                 rxr->rx_tpa[0].agg_arr = agg;
3188                 if (!agg)
3189                         return -ENOMEM;
3190                 for (j = 1; j < bp->max_tpa; j++)
3191                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
3192                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3193                                               GFP_KERNEL);
3194                 if (!rxr->rx_tpa_idx_map)
3195                         return -ENOMEM;
3196         }
3197         return 0;
3198 }
3199
3200 static void bnxt_free_rx_rings(struct bnxt *bp)
3201 {
3202         int i;
3203
3204         if (!bp->rx_ring)
3205                 return;
3206
3207         bnxt_free_tpa_info(bp);
3208         for (i = 0; i < bp->rx_nr_rings; i++) {
3209                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3210                 struct bnxt_ring_struct *ring;
3211
3212                 if (rxr->xdp_prog)
3213                         bpf_prog_put(rxr->xdp_prog);
3214
3215                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3216                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3217
3218                 page_pool_destroy(rxr->page_pool);
3219                 rxr->page_pool = NULL;
3220
3221                 kfree(rxr->rx_agg_bmap);
3222                 rxr->rx_agg_bmap = NULL;
3223
3224                 ring = &rxr->rx_ring_struct;
3225                 bnxt_free_ring(bp, &ring->ring_mem);
3226
3227                 ring = &rxr->rx_agg_ring_struct;
3228                 bnxt_free_ring(bp, &ring->ring_mem);
3229         }
3230 }
3231
3232 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3233                                    struct bnxt_rx_ring_info *rxr)
3234 {
3235         struct page_pool_params pp = { 0 };
3236
3237         pp.pool_size = bp->rx_ring_size;
3238         pp.nid = dev_to_node(&bp->pdev->dev);
3239         pp.dev = &bp->pdev->dev;
3240         pp.dma_dir = DMA_BIDIRECTIONAL;
3241
3242         rxr->page_pool = page_pool_create(&pp);
3243         if (IS_ERR(rxr->page_pool)) {
3244                 int err = PTR_ERR(rxr->page_pool);
3245
3246                 rxr->page_pool = NULL;
3247                 return err;
3248         }
3249         return 0;
3250 }
3251
3252 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3253 {
3254         int i, rc = 0, agg_rings = 0;
3255
3256         if (!bp->rx_ring)
3257                 return -ENOMEM;
3258
3259         if (bp->flags & BNXT_FLAG_AGG_RINGS)
3260                 agg_rings = 1;
3261
3262         for (i = 0; i < bp->rx_nr_rings; i++) {
3263                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3264                 struct bnxt_ring_struct *ring;
3265
3266                 ring = &rxr->rx_ring_struct;
3267
3268                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
3269                 if (rc)
3270                         return rc;
3271
3272                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3273                 if (rc < 0)
3274                         return rc;
3275
3276                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3277                                                 MEM_TYPE_PAGE_POOL,
3278                                                 rxr->page_pool);
3279                 if (rc) {
3280                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3281                         return rc;
3282                 }
3283
3284                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3285                 if (rc)
3286                         return rc;
3287
3288                 ring->grp_idx = i;
3289                 if (agg_rings) {
3290                         u16 mem_size;
3291
3292                         ring = &rxr->rx_agg_ring_struct;
3293                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3294                         if (rc)
3295                                 return rc;
3296
3297                         ring->grp_idx = i;
3298                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3299                         mem_size = rxr->rx_agg_bmap_size / 8;
3300                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3301                         if (!rxr->rx_agg_bmap)
3302                                 return -ENOMEM;
3303                 }
3304         }
3305         if (bp->flags & BNXT_FLAG_TPA)
3306                 rc = bnxt_alloc_tpa_info(bp);
3307         return rc;
3308 }
3309
3310 static void bnxt_free_tx_rings(struct bnxt *bp)
3311 {
3312         int i;
3313         struct pci_dev *pdev = bp->pdev;
3314
3315         if (!bp->tx_ring)
3316                 return;
3317
3318         for (i = 0; i < bp->tx_nr_rings; i++) {
3319                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3320                 struct bnxt_ring_struct *ring;
3321
3322                 if (txr->tx_push) {
3323                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
3324                                           txr->tx_push, txr->tx_push_mapping);
3325                         txr->tx_push = NULL;
3326                 }
3327
3328                 ring = &txr->tx_ring_struct;
3329
3330                 bnxt_free_ring(bp, &ring->ring_mem);
3331         }
3332 }
3333
3334 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3335 {
3336         int i, j, rc;
3337         struct pci_dev *pdev = bp->pdev;
3338
3339         bp->tx_push_size = 0;
3340         if (bp->tx_push_thresh) {
3341                 int push_size;
3342
3343                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3344                                         bp->tx_push_thresh);
3345
3346                 if (push_size > 256) {
3347                         push_size = 0;
3348                         bp->tx_push_thresh = 0;
3349                 }
3350
3351                 bp->tx_push_size = push_size;
3352         }
3353
3354         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3355                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3356                 struct bnxt_ring_struct *ring;
3357                 u8 qidx;
3358
3359                 ring = &txr->tx_ring_struct;
3360
3361                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3362                 if (rc)
3363                         return rc;
3364
3365                 ring->grp_idx = txr->bnapi->index;
3366                 if (bp->tx_push_size) {
3367                         dma_addr_t mapping;
3368
3369                         /* One pre-allocated DMA buffer to backup
3370                          * TX push operation
3371                          */
3372                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
3373                                                 bp->tx_push_size,
3374                                                 &txr->tx_push_mapping,
3375                                                 GFP_KERNEL);
3376
3377                         if (!txr->tx_push)
3378                                 return -ENOMEM;
3379
3380                         mapping = txr->tx_push_mapping +
3381                                 sizeof(struct tx_push_bd);
3382                         txr->data_mapping = cpu_to_le64(mapping);
3383                 }
3384                 qidx = bp->tc_to_qidx[j];
3385                 ring->queue_id = bp->q_info[qidx].queue_id;
3386                 spin_lock_init(&txr->xdp_tx_lock);
3387                 if (i < bp->tx_nr_rings_xdp)
3388                         continue;
3389                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3390                         j++;
3391         }
3392         return 0;
3393 }
3394
3395 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3396 {
3397         struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3398
3399         kfree(cpr->cp_desc_ring);
3400         cpr->cp_desc_ring = NULL;
3401         ring->ring_mem.pg_arr = NULL;
3402         kfree(cpr->cp_desc_mapping);
3403         cpr->cp_desc_mapping = NULL;
3404         ring->ring_mem.dma_arr = NULL;
3405 }
3406
3407 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3408 {
3409         cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3410         if (!cpr->cp_desc_ring)
3411                 return -ENOMEM;
3412         cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3413                                        GFP_KERNEL);
3414         if (!cpr->cp_desc_mapping)
3415                 return -ENOMEM;
3416         return 0;
3417 }
3418
3419 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3420 {
3421         int i;
3422
3423         if (!bp->bnapi)
3424                 return;
3425         for (i = 0; i < bp->cp_nr_rings; i++) {
3426                 struct bnxt_napi *bnapi = bp->bnapi[i];
3427
3428                 if (!bnapi)
3429                         continue;
3430                 bnxt_free_cp_arrays(&bnapi->cp_ring);
3431         }
3432 }
3433
3434 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3435 {
3436         int i, n = bp->cp_nr_pages;
3437
3438         for (i = 0; i < bp->cp_nr_rings; i++) {
3439                 struct bnxt_napi *bnapi = bp->bnapi[i];
3440                 int rc;
3441
3442                 if (!bnapi)
3443                         continue;
3444                 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3445                 if (rc)
3446                         return rc;
3447         }
3448         return 0;
3449 }
3450
3451 static void bnxt_free_cp_rings(struct bnxt *bp)
3452 {
3453         int i;
3454
3455         if (!bp->bnapi)
3456                 return;
3457
3458         for (i = 0; i < bp->cp_nr_rings; i++) {
3459                 struct bnxt_napi *bnapi = bp->bnapi[i];
3460                 struct bnxt_cp_ring_info *cpr;
3461                 struct bnxt_ring_struct *ring;
3462                 int j;
3463
3464                 if (!bnapi)
3465                         continue;
3466
3467                 cpr = &bnapi->cp_ring;
3468                 ring = &cpr->cp_ring_struct;
3469
3470                 bnxt_free_ring(bp, &ring->ring_mem);
3471
3472                 for (j = 0; j < 2; j++) {
3473                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3474
3475                         if (cpr2) {
3476                                 ring = &cpr2->cp_ring_struct;
3477                                 bnxt_free_ring(bp, &ring->ring_mem);
3478                                 bnxt_free_cp_arrays(cpr2);
3479                                 kfree(cpr2);
3480                                 cpr->cp_ring_arr[j] = NULL;
3481                         }
3482                 }
3483         }
3484 }
3485
3486 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3487 {
3488         struct bnxt_ring_mem_info *rmem;
3489         struct bnxt_ring_struct *ring;
3490         struct bnxt_cp_ring_info *cpr;
3491         int rc;
3492
3493         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3494         if (!cpr)
3495                 return NULL;
3496
3497         rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3498         if (rc) {
3499                 bnxt_free_cp_arrays(cpr);
3500                 kfree(cpr);
3501                 return NULL;
3502         }
3503         ring = &cpr->cp_ring_struct;
3504         rmem = &ring->ring_mem;
3505         rmem->nr_pages = bp->cp_nr_pages;
3506         rmem->page_size = HW_CMPD_RING_SIZE;
3507         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3508         rmem->dma_arr = cpr->cp_desc_mapping;
3509         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3510         rc = bnxt_alloc_ring(bp, rmem);
3511         if (rc) {
3512                 bnxt_free_ring(bp, rmem);
3513                 bnxt_free_cp_arrays(cpr);
3514                 kfree(cpr);
3515                 cpr = NULL;
3516         }
3517         return cpr;
3518 }
3519
3520 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3521 {
3522         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3523         int i, rc, ulp_base_vec, ulp_msix;
3524
3525         ulp_msix = bnxt_get_ulp_msix_num(bp);
3526         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3527         for (i = 0; i < bp->cp_nr_rings; i++) {
3528                 struct bnxt_napi *bnapi = bp->bnapi[i];
3529                 struct bnxt_cp_ring_info *cpr;
3530                 struct bnxt_ring_struct *ring;
3531
3532                 if (!bnapi)
3533                         continue;
3534
3535                 cpr = &bnapi->cp_ring;
3536                 cpr->bnapi = bnapi;
3537                 ring = &cpr->cp_ring_struct;
3538
3539                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3540                 if (rc)
3541                         return rc;
3542
3543                 if (ulp_msix && i >= ulp_base_vec)
3544                         ring->map_idx = i + ulp_msix;
3545                 else
3546                         ring->map_idx = i;
3547
3548                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3549                         continue;
3550
3551                 if (i < bp->rx_nr_rings) {
3552                         struct bnxt_cp_ring_info *cpr2 =
3553                                 bnxt_alloc_cp_sub_ring(bp);
3554
3555                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3556                         if (!cpr2)
3557                                 return -ENOMEM;
3558                         cpr2->bnapi = bnapi;
3559                 }
3560                 if ((sh && i < bp->tx_nr_rings) ||
3561                     (!sh && i >= bp->rx_nr_rings)) {
3562                         struct bnxt_cp_ring_info *cpr2 =
3563                                 bnxt_alloc_cp_sub_ring(bp);
3564
3565                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3566                         if (!cpr2)
3567                                 return -ENOMEM;
3568                         cpr2->bnapi = bnapi;
3569                 }
3570         }
3571         return 0;
3572 }
3573
3574 static void bnxt_init_ring_struct(struct bnxt *bp)
3575 {
3576         int i;
3577
3578         for (i = 0; i < bp->cp_nr_rings; i++) {
3579                 struct bnxt_napi *bnapi = bp->bnapi[i];
3580                 struct bnxt_ring_mem_info *rmem;
3581                 struct bnxt_cp_ring_info *cpr;
3582                 struct bnxt_rx_ring_info *rxr;
3583                 struct bnxt_tx_ring_info *txr;
3584                 struct bnxt_ring_struct *ring;
3585
3586                 if (!bnapi)
3587                         continue;
3588
3589                 cpr = &bnapi->cp_ring;
3590                 ring = &cpr->cp_ring_struct;
3591                 rmem = &ring->ring_mem;
3592                 rmem->nr_pages = bp->cp_nr_pages;
3593                 rmem->page_size = HW_CMPD_RING_SIZE;
3594                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3595                 rmem->dma_arr = cpr->cp_desc_mapping;
3596                 rmem->vmem_size = 0;
3597
3598                 rxr = bnapi->rx_ring;
3599                 if (!rxr)
3600                         goto skip_rx;
3601
3602                 ring = &rxr->rx_ring_struct;
3603                 rmem = &ring->ring_mem;
3604                 rmem->nr_pages = bp->rx_nr_pages;
3605                 rmem->page_size = HW_RXBD_RING_SIZE;
3606                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3607                 rmem->dma_arr = rxr->rx_desc_mapping;
3608                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3609                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3610
3611                 ring = &rxr->rx_agg_ring_struct;
3612                 rmem = &ring->ring_mem;
3613                 rmem->nr_pages = bp->rx_agg_nr_pages;
3614                 rmem->page_size = HW_RXBD_RING_SIZE;
3615                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3616                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3617                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3618                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3619
3620 skip_rx:
3621                 txr = bnapi->tx_ring;
3622                 if (!txr)
3623                         continue;
3624
3625                 ring = &txr->tx_ring_struct;
3626                 rmem = &ring->ring_mem;
3627                 rmem->nr_pages = bp->tx_nr_pages;
3628                 rmem->page_size = HW_RXBD_RING_SIZE;
3629                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3630                 rmem->dma_arr = txr->tx_desc_mapping;
3631                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3632                 rmem->vmem = (void **)&txr->tx_buf_ring;
3633         }
3634 }
3635
3636 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3637 {
3638         int i;
3639         u32 prod;
3640         struct rx_bd **rx_buf_ring;
3641
3642         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3643         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3644                 int j;
3645                 struct rx_bd *rxbd;
3646
3647                 rxbd = rx_buf_ring[i];
3648                 if (!rxbd)
3649                         continue;
3650
3651                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3652                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3653                         rxbd->rx_bd_opaque = prod;
3654                 }
3655         }
3656 }
3657
3658 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3659 {
3660         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3661         struct net_device *dev = bp->dev;
3662         u32 prod;
3663         int i;
3664
3665         prod = rxr->rx_prod;
3666         for (i = 0; i < bp->rx_ring_size; i++) {
3667                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3668                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3669                                     ring_nr, i, bp->rx_ring_size);
3670                         break;
3671                 }
3672                 prod = NEXT_RX(prod);
3673         }
3674         rxr->rx_prod = prod;
3675
3676         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3677                 return 0;
3678
3679         prod = rxr->rx_agg_prod;
3680         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3681                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3682                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3683                                     ring_nr, i, bp->rx_ring_size);
3684                         break;
3685                 }
3686                 prod = NEXT_RX_AGG(prod);
3687         }
3688         rxr->rx_agg_prod = prod;
3689
3690         if (rxr->rx_tpa) {
3691                 dma_addr_t mapping;
3692                 u8 *data;
3693
3694                 for (i = 0; i < bp->max_tpa; i++) {
3695                         data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
3696                         if (!data)
3697                                 return -ENOMEM;
3698
3699                         rxr->rx_tpa[i].data = data;
3700                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3701                         rxr->rx_tpa[i].mapping = mapping;
3702                 }
3703         }
3704         return 0;
3705 }
3706
3707 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3708 {
3709         struct bnxt_rx_ring_info *rxr;
3710         struct bnxt_ring_struct *ring;
3711         u32 type;
3712
3713         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3714                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3715
3716         if (NET_IP_ALIGN == 2)
3717                 type |= RX_BD_FLAGS_SOP;
3718
3719         rxr = &bp->rx_ring[ring_nr];
3720         ring = &rxr->rx_ring_struct;
3721         bnxt_init_rxbd_pages(ring, type);
3722
3723         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3724                 bpf_prog_add(bp->xdp_prog, 1);
3725                 rxr->xdp_prog = bp->xdp_prog;
3726         }
3727         ring->fw_ring_id = INVALID_HW_RING_ID;
3728
3729         ring = &rxr->rx_agg_ring_struct;
3730         ring->fw_ring_id = INVALID_HW_RING_ID;
3731
3732         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3733                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3734                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3735
3736                 bnxt_init_rxbd_pages(ring, type);
3737         }
3738
3739         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3740 }
3741
3742 static void bnxt_init_cp_rings(struct bnxt *bp)
3743 {
3744         int i, j;
3745
3746         for (i = 0; i < bp->cp_nr_rings; i++) {
3747                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3748                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3749
3750                 ring->fw_ring_id = INVALID_HW_RING_ID;
3751                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3752                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3753                 for (j = 0; j < 2; j++) {
3754                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3755
3756                         if (!cpr2)
3757                                 continue;
3758
3759                         ring = &cpr2->cp_ring_struct;
3760                         ring->fw_ring_id = INVALID_HW_RING_ID;
3761                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3762                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3763                 }
3764         }
3765 }
3766
3767 static int bnxt_init_rx_rings(struct bnxt *bp)
3768 {
3769         int i, rc = 0;
3770
3771         if (BNXT_RX_PAGE_MODE(bp)) {
3772                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3773                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3774         } else {
3775                 bp->rx_offset = BNXT_RX_OFFSET;
3776                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3777         }
3778
3779         for (i = 0; i < bp->rx_nr_rings; i++) {
3780                 rc = bnxt_init_one_rx_ring(bp, i);
3781                 if (rc)
3782                         break;
3783         }
3784
3785         return rc;
3786 }
3787
3788 static int bnxt_init_tx_rings(struct bnxt *bp)
3789 {
3790         u16 i;
3791
3792         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3793                                    BNXT_MIN_TX_DESC_CNT);
3794
3795         for (i = 0; i < bp->tx_nr_rings; i++) {
3796                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3797                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3798
3799                 ring->fw_ring_id = INVALID_HW_RING_ID;
3800         }
3801
3802         return 0;
3803 }
3804
3805 static void bnxt_free_ring_grps(struct bnxt *bp)
3806 {
3807         kfree(bp->grp_info);
3808         bp->grp_info = NULL;
3809 }
3810
3811 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3812 {
3813         int i;
3814
3815         if (irq_re_init) {
3816                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3817                                        sizeof(struct bnxt_ring_grp_info),
3818                                        GFP_KERNEL);
3819                 if (!bp->grp_info)
3820                         return -ENOMEM;
3821         }
3822         for (i = 0; i < bp->cp_nr_rings; i++) {
3823                 if (irq_re_init)
3824                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3825                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3826                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3827                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3828                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3829         }
3830         return 0;
3831 }
3832
3833 static void bnxt_free_vnics(struct bnxt *bp)
3834 {
3835         kfree(bp->vnic_info);
3836         bp->vnic_info = NULL;
3837         bp->nr_vnics = 0;
3838 }
3839
3840 static int bnxt_alloc_vnics(struct bnxt *bp)
3841 {
3842         int num_vnics = 1;
3843
3844 #ifdef CONFIG_RFS_ACCEL
3845         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3846                 num_vnics += bp->rx_nr_rings;
3847 #endif
3848
3849         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3850                 num_vnics++;
3851
3852         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3853                                 GFP_KERNEL);
3854         if (!bp->vnic_info)
3855                 return -ENOMEM;
3856
3857         bp->nr_vnics = num_vnics;
3858         return 0;
3859 }
3860
3861 static void bnxt_init_vnics(struct bnxt *bp)
3862 {
3863         int i;
3864
3865         for (i = 0; i < bp->nr_vnics; i++) {
3866                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3867                 int j;
3868
3869                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3870                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3871                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3872
3873                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3874
3875                 if (bp->vnic_info[i].rss_hash_key) {
3876                         if (i == 0)
3877                                 prandom_bytes(vnic->rss_hash_key,
3878                                               HW_HASH_KEY_SIZE);
3879                         else
3880                                 memcpy(vnic->rss_hash_key,
3881                                        bp->vnic_info[0].rss_hash_key,
3882                                        HW_HASH_KEY_SIZE);
3883                 }
3884         }
3885 }
3886
3887 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3888 {
3889         int pages;
3890
3891         pages = ring_size / desc_per_pg;
3892
3893         if (!pages)
3894                 return 1;
3895
3896         pages++;
3897
3898         while (pages & (pages - 1))
3899                 pages++;
3900
3901         return pages;
3902 }
3903
3904 void bnxt_set_tpa_flags(struct bnxt *bp)
3905 {
3906         bp->flags &= ~BNXT_FLAG_TPA;
3907         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3908                 return;
3909         if (bp->dev->features & NETIF_F_LRO)
3910                 bp->flags |= BNXT_FLAG_LRO;
3911         else if (bp->dev->features & NETIF_F_GRO_HW)
3912                 bp->flags |= BNXT_FLAG_GRO;
3913 }
3914
3915 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3916  * be set on entry.
3917  */
3918 void bnxt_set_ring_params(struct bnxt *bp)
3919 {
3920         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3921         u32 agg_factor = 0, agg_ring_size = 0;
3922
3923         /* 8 for CRC and VLAN */
3924         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3925
3926         rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
3927                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3928
3929         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3930         ring_size = bp->rx_ring_size;
3931         bp->rx_agg_ring_size = 0;
3932         bp->rx_agg_nr_pages = 0;
3933
3934         if (bp->flags & BNXT_FLAG_TPA)
3935                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3936
3937         bp->flags &= ~BNXT_FLAG_JUMBO;
3938         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3939                 u32 jumbo_factor;
3940
3941                 bp->flags |= BNXT_FLAG_JUMBO;
3942                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3943                 if (jumbo_factor > agg_factor)
3944                         agg_factor = jumbo_factor;
3945         }
3946         if (agg_factor) {
3947                 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
3948                         ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
3949                         netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
3950                                     bp->rx_ring_size, ring_size);
3951                         bp->rx_ring_size = ring_size;
3952                 }
3953                 agg_ring_size = ring_size * agg_factor;
3954
3955                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3956                                                         RX_DESC_CNT);
3957                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3958                         u32 tmp = agg_ring_size;
3959
3960                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3961                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3962                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3963                                     tmp, agg_ring_size);
3964                 }
3965                 bp->rx_agg_ring_size = agg_ring_size;
3966                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3967
3968                 if (BNXT_RX_PAGE_MODE(bp)) {
3969                         rx_space = BNXT_PAGE_MODE_BUF_SIZE;
3970                         rx_size = BNXT_MAX_PAGE_MODE_MTU;
3971                 } else {
3972                         rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3973                         rx_space = rx_size + NET_SKB_PAD +
3974                                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3975                 }
3976         }
3977
3978         bp->rx_buf_use_size = rx_size;
3979         bp->rx_buf_size = rx_space;
3980
3981         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3982         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3983
3984         ring_size = bp->tx_ring_size;
3985         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3986         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3987
3988         max_rx_cmpl = bp->rx_ring_size;
3989         /* MAX TPA needs to be added because TPA_START completions are
3990          * immediately recycled, so the TPA completions are not bound by
3991          * the RX ring size.
3992          */
3993         if (bp->flags & BNXT_FLAG_TPA)
3994                 max_rx_cmpl += bp->max_tpa;
3995         /* RX and TPA completions are 32-byte, all others are 16-byte */
3996         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3997         bp->cp_ring_size = ring_size;
3998
3999         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
4000         if (bp->cp_nr_pages > MAX_CP_PAGES) {
4001                 bp->cp_nr_pages = MAX_CP_PAGES;
4002                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
4003                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
4004                             ring_size, bp->cp_ring_size);
4005         }
4006         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
4007         bp->cp_ring_mask = bp->cp_bit - 1;
4008 }
4009
4010 /* Changing allocation mode of RX rings.
4011  * TODO: Update when extending xdp_rxq_info to support allocation modes.
4012  */
4013 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
4014 {
4015         if (page_mode) {
4016                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
4017                 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
4018
4019                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4020                         bp->flags |= BNXT_FLAG_JUMBO;
4021                         bp->rx_skb_func = bnxt_rx_multi_page_skb;
4022                         bp->dev->max_mtu =
4023                                 min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4024                 } else {
4025                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4026                         bp->rx_skb_func = bnxt_rx_page_skb;
4027                         bp->dev->max_mtu =
4028                                 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4029                 }
4030                 bp->rx_dir = DMA_BIDIRECTIONAL;
4031                 /* Disable LRO or GRO_HW */
4032                 netdev_update_features(bp->dev);
4033         } else {
4034                 bp->dev->max_mtu = bp->max_mtu;
4035                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4036                 bp->rx_dir = DMA_FROM_DEVICE;
4037                 bp->rx_skb_func = bnxt_rx_skb;
4038         }
4039         return 0;
4040 }
4041
4042 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4043 {
4044         int i;
4045         struct bnxt_vnic_info *vnic;
4046         struct pci_dev *pdev = bp->pdev;
4047
4048         if (!bp->vnic_info)
4049                 return;
4050
4051         for (i = 0; i < bp->nr_vnics; i++) {
4052                 vnic = &bp->vnic_info[i];
4053
4054                 kfree(vnic->fw_grp_ids);
4055                 vnic->fw_grp_ids = NULL;
4056
4057                 kfree(vnic->uc_list);
4058                 vnic->uc_list = NULL;
4059
4060                 if (vnic->mc_list) {
4061                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4062                                           vnic->mc_list, vnic->mc_list_mapping);
4063                         vnic->mc_list = NULL;
4064                 }
4065
4066                 if (vnic->rss_table) {
4067                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4068                                           vnic->rss_table,
4069                                           vnic->rss_table_dma_addr);
4070                         vnic->rss_table = NULL;
4071                 }
4072
4073                 vnic->rss_hash_key = NULL;
4074                 vnic->flags = 0;
4075         }
4076 }
4077
4078 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4079 {
4080         int i, rc = 0, size;
4081         struct bnxt_vnic_info *vnic;
4082         struct pci_dev *pdev = bp->pdev;
4083         int max_rings;
4084
4085         for (i = 0; i < bp->nr_vnics; i++) {
4086                 vnic = &bp->vnic_info[i];
4087
4088                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4089                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4090
4091                         if (mem_size > 0) {
4092                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4093                                 if (!vnic->uc_list) {
4094                                         rc = -ENOMEM;
4095                                         goto out;
4096                                 }
4097                         }
4098                 }
4099
4100                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4101                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4102                         vnic->mc_list =
4103                                 dma_alloc_coherent(&pdev->dev,
4104                                                    vnic->mc_list_size,
4105                                                    &vnic->mc_list_mapping,
4106                                                    GFP_KERNEL);
4107                         if (!vnic->mc_list) {
4108                                 rc = -ENOMEM;
4109                                 goto out;
4110                         }
4111                 }
4112
4113                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4114                         goto vnic_skip_grps;
4115
4116                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4117                         max_rings = bp->rx_nr_rings;
4118                 else
4119                         max_rings = 1;
4120
4121                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4122                 if (!vnic->fw_grp_ids) {
4123                         rc = -ENOMEM;
4124                         goto out;
4125                 }
4126 vnic_skip_grps:
4127                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
4128                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4129                         continue;
4130
4131                 /* Allocate rss table and hash key */
4132                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4133                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4134                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4135
4136                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4137                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4138                                                      vnic->rss_table_size,
4139                                                      &vnic->rss_table_dma_addr,
4140                                                      GFP_KERNEL);
4141                 if (!vnic->rss_table) {
4142                         rc = -ENOMEM;
4143                         goto out;
4144                 }
4145
4146                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4147                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4148         }
4149         return 0;
4150
4151 out:
4152         return rc;
4153 }
4154
4155 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4156 {
4157         struct bnxt_hwrm_wait_token *token;
4158
4159         dma_pool_destroy(bp->hwrm_dma_pool);
4160         bp->hwrm_dma_pool = NULL;
4161
4162         rcu_read_lock();
4163         hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4164                 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4165         rcu_read_unlock();
4166 }
4167
4168 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4169 {
4170         bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4171                                             BNXT_HWRM_DMA_SIZE,
4172                                             BNXT_HWRM_DMA_ALIGN, 0);
4173         if (!bp->hwrm_dma_pool)
4174                 return -ENOMEM;
4175
4176         INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4177
4178         return 0;
4179 }
4180
4181 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4182 {
4183         kfree(stats->hw_masks);
4184         stats->hw_masks = NULL;
4185         kfree(stats->sw_stats);
4186         stats->sw_stats = NULL;
4187         if (stats->hw_stats) {
4188                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4189                                   stats->hw_stats_map);
4190                 stats->hw_stats = NULL;
4191         }
4192 }
4193
4194 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4195                                 bool alloc_masks)
4196 {
4197         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4198                                              &stats->hw_stats_map, GFP_KERNEL);
4199         if (!stats->hw_stats)
4200                 return -ENOMEM;
4201
4202         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4203         if (!stats->sw_stats)
4204                 goto stats_mem_err;
4205
4206         if (alloc_masks) {
4207                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4208                 if (!stats->hw_masks)
4209                         goto stats_mem_err;
4210         }
4211         return 0;
4212
4213 stats_mem_err:
4214         bnxt_free_stats_mem(bp, stats);
4215         return -ENOMEM;
4216 }
4217
4218 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4219 {
4220         int i;
4221
4222         for (i = 0; i < count; i++)
4223                 mask_arr[i] = mask;
4224 }
4225
4226 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4227 {
4228         int i;
4229
4230         for (i = 0; i < count; i++)
4231                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4232 }
4233
4234 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4235                                     struct bnxt_stats_mem *stats)
4236 {
4237         struct hwrm_func_qstats_ext_output *resp;
4238         struct hwrm_func_qstats_ext_input *req;
4239         __le64 *hw_masks;
4240         int rc;
4241
4242         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4243             !(bp->flags & BNXT_FLAG_CHIP_P5))
4244                 return -EOPNOTSUPP;
4245
4246         rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4247         if (rc)
4248                 return rc;
4249
4250         req->fid = cpu_to_le16(0xffff);
4251         req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4252
4253         resp = hwrm_req_hold(bp, req);
4254         rc = hwrm_req_send(bp, req);
4255         if (!rc) {
4256                 hw_masks = &resp->rx_ucast_pkts;
4257                 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4258         }
4259         hwrm_req_drop(bp, req);
4260         return rc;
4261 }
4262
4263 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4264 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4265
4266 static void bnxt_init_stats(struct bnxt *bp)
4267 {
4268         struct bnxt_napi *bnapi = bp->bnapi[0];
4269         struct bnxt_cp_ring_info *cpr;
4270         struct bnxt_stats_mem *stats;
4271         __le64 *rx_stats, *tx_stats;
4272         int rc, rx_count, tx_count;
4273         u64 *rx_masks, *tx_masks;
4274         u64 mask;
4275         u8 flags;
4276
4277         cpr = &bnapi->cp_ring;
4278         stats = &cpr->stats;
4279         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4280         if (rc) {
4281                 if (bp->flags & BNXT_FLAG_CHIP_P5)
4282                         mask = (1ULL << 48) - 1;
4283                 else
4284                         mask = -1ULL;
4285                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4286         }
4287         if (bp->flags & BNXT_FLAG_PORT_STATS) {
4288                 stats = &bp->port_stats;
4289                 rx_stats = stats->hw_stats;
4290                 rx_masks = stats->hw_masks;
4291                 rx_count = sizeof(struct rx_port_stats) / 8;
4292                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4293                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4294                 tx_count = sizeof(struct tx_port_stats) / 8;
4295
4296                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4297                 rc = bnxt_hwrm_port_qstats(bp, flags);
4298                 if (rc) {
4299                         mask = (1ULL << 40) - 1;
4300
4301                         bnxt_fill_masks(rx_masks, mask, rx_count);
4302                         bnxt_fill_masks(tx_masks, mask, tx_count);
4303                 } else {
4304                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4305                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4306                         bnxt_hwrm_port_qstats(bp, 0);
4307                 }
4308         }
4309         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4310                 stats = &bp->rx_port_stats_ext;
4311                 rx_stats = stats->hw_stats;
4312                 rx_masks = stats->hw_masks;
4313                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4314                 stats = &bp->tx_port_stats_ext;
4315                 tx_stats = stats->hw_stats;
4316                 tx_masks = stats->hw_masks;
4317                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4318
4319                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4320                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4321                 if (rc) {
4322                         mask = (1ULL << 40) - 1;
4323
4324                         bnxt_fill_masks(rx_masks, mask, rx_count);
4325                         if (tx_stats)
4326                                 bnxt_fill_masks(tx_masks, mask, tx_count);
4327                 } else {
4328                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4329                         if (tx_stats)
4330                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
4331                                                    tx_count);
4332                         bnxt_hwrm_port_qstats_ext(bp, 0);
4333                 }
4334         }
4335 }
4336
4337 static void bnxt_free_port_stats(struct bnxt *bp)
4338 {
4339         bp->flags &= ~BNXT_FLAG_PORT_STATS;
4340         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4341
4342         bnxt_free_stats_mem(bp, &bp->port_stats);
4343         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4344         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4345 }
4346
4347 static void bnxt_free_ring_stats(struct bnxt *bp)
4348 {
4349         int i;
4350
4351         if (!bp->bnapi)
4352                 return;
4353
4354         for (i = 0; i < bp->cp_nr_rings; i++) {
4355                 struct bnxt_napi *bnapi = bp->bnapi[i];
4356                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4357
4358                 bnxt_free_stats_mem(bp, &cpr->stats);
4359         }
4360 }
4361
4362 static int bnxt_alloc_stats(struct bnxt *bp)
4363 {
4364         u32 size, i;
4365         int rc;
4366
4367         size = bp->hw_ring_stats_size;
4368
4369         for (i = 0; i < bp->cp_nr_rings; i++) {
4370                 struct bnxt_napi *bnapi = bp->bnapi[i];
4371                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4372
4373                 cpr->stats.len = size;
4374                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4375                 if (rc)
4376                         return rc;
4377
4378                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4379         }
4380
4381         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4382                 return 0;
4383
4384         if (bp->port_stats.hw_stats)
4385                 goto alloc_ext_stats;
4386
4387         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4388         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4389         if (rc)
4390                 return rc;
4391
4392         bp->flags |= BNXT_FLAG_PORT_STATS;
4393
4394 alloc_ext_stats:
4395         /* Display extended statistics only if FW supports it */
4396         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4397                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4398                         return 0;
4399
4400         if (bp->rx_port_stats_ext.hw_stats)
4401                 goto alloc_tx_ext_stats;
4402
4403         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4404         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4405         /* Extended stats are optional */
4406         if (rc)
4407                 return 0;
4408
4409 alloc_tx_ext_stats:
4410         if (bp->tx_port_stats_ext.hw_stats)
4411                 return 0;
4412
4413         if (bp->hwrm_spec_code >= 0x10902 ||
4414             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4415                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4416                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4417                 /* Extended stats are optional */
4418                 if (rc)
4419                         return 0;
4420         }
4421         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4422         return 0;
4423 }
4424
4425 static void bnxt_clear_ring_indices(struct bnxt *bp)
4426 {
4427         int i;
4428
4429         if (!bp->bnapi)
4430                 return;
4431
4432         for (i = 0; i < bp->cp_nr_rings; i++) {
4433                 struct bnxt_napi *bnapi = bp->bnapi[i];
4434                 struct bnxt_cp_ring_info *cpr;
4435                 struct bnxt_rx_ring_info *rxr;
4436                 struct bnxt_tx_ring_info *txr;
4437
4438                 if (!bnapi)
4439                         continue;
4440
4441                 cpr = &bnapi->cp_ring;
4442                 cpr->cp_raw_cons = 0;
4443
4444                 txr = bnapi->tx_ring;
4445                 if (txr) {
4446                         txr->tx_prod = 0;
4447                         txr->tx_cons = 0;
4448                 }
4449
4450                 rxr = bnapi->rx_ring;
4451                 if (rxr) {
4452                         rxr->rx_prod = 0;
4453                         rxr->rx_agg_prod = 0;
4454                         rxr->rx_sw_agg_prod = 0;
4455                         rxr->rx_next_cons = 0;
4456                 }
4457         }
4458 }
4459
4460 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4461 {
4462 #ifdef CONFIG_RFS_ACCEL
4463         int i;
4464
4465         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4466          * safe to delete the hash table.
4467          */
4468         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4469                 struct hlist_head *head;
4470                 struct hlist_node *tmp;
4471                 struct bnxt_ntuple_filter *fltr;
4472
4473                 head = &bp->ntp_fltr_hash_tbl[i];
4474                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4475                         hlist_del(&fltr->hash);
4476                         kfree(fltr);
4477                 }
4478         }
4479         if (irq_reinit) {
4480                 kfree(bp->ntp_fltr_bmap);
4481                 bp->ntp_fltr_bmap = NULL;
4482         }
4483         bp->ntp_fltr_count = 0;
4484 #endif
4485 }
4486
4487 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4488 {
4489 #ifdef CONFIG_RFS_ACCEL
4490         int i, rc = 0;
4491
4492         if (!(bp->flags & BNXT_FLAG_RFS))
4493                 return 0;
4494
4495         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4496                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4497
4498         bp->ntp_fltr_count = 0;
4499         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4500                                     sizeof(long),
4501                                     GFP_KERNEL);
4502
4503         if (!bp->ntp_fltr_bmap)
4504                 rc = -ENOMEM;
4505
4506         return rc;
4507 #else
4508         return 0;
4509 #endif
4510 }
4511
4512 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4513 {
4514         bnxt_free_vnic_attributes(bp);
4515         bnxt_free_tx_rings(bp);
4516         bnxt_free_rx_rings(bp);
4517         bnxt_free_cp_rings(bp);
4518         bnxt_free_all_cp_arrays(bp);
4519         bnxt_free_ntp_fltrs(bp, irq_re_init);
4520         if (irq_re_init) {
4521                 bnxt_free_ring_stats(bp);
4522                 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4523                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4524                         bnxt_free_port_stats(bp);
4525                 bnxt_free_ring_grps(bp);
4526                 bnxt_free_vnics(bp);
4527                 kfree(bp->tx_ring_map);
4528                 bp->tx_ring_map = NULL;
4529                 kfree(bp->tx_ring);
4530                 bp->tx_ring = NULL;
4531                 kfree(bp->rx_ring);
4532                 bp->rx_ring = NULL;
4533                 kfree(bp->bnapi);
4534                 bp->bnapi = NULL;
4535         } else {
4536                 bnxt_clear_ring_indices(bp);
4537         }
4538 }
4539
4540 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4541 {
4542         int i, j, rc, size, arr_size;
4543         void *bnapi;
4544
4545         if (irq_re_init) {
4546                 /* Allocate bnapi mem pointer array and mem block for
4547                  * all queues
4548                  */
4549                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4550                                 bp->cp_nr_rings);
4551                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4552                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4553                 if (!bnapi)
4554                         return -ENOMEM;
4555
4556                 bp->bnapi = bnapi;
4557                 bnapi += arr_size;
4558                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4559                         bp->bnapi[i] = bnapi;
4560                         bp->bnapi[i]->index = i;
4561                         bp->bnapi[i]->bp = bp;
4562                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4563                                 struct bnxt_cp_ring_info *cpr =
4564                                         &bp->bnapi[i]->cp_ring;
4565
4566                                 cpr->cp_ring_struct.ring_mem.flags =
4567                                         BNXT_RMEM_RING_PTE_FLAG;
4568                         }
4569                 }
4570
4571                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4572                                       sizeof(struct bnxt_rx_ring_info),
4573                                       GFP_KERNEL);
4574                 if (!bp->rx_ring)
4575                         return -ENOMEM;
4576
4577                 for (i = 0; i < bp->rx_nr_rings; i++) {
4578                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4579
4580                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4581                                 rxr->rx_ring_struct.ring_mem.flags =
4582                                         BNXT_RMEM_RING_PTE_FLAG;
4583                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4584                                         BNXT_RMEM_RING_PTE_FLAG;
4585                         }
4586                         rxr->bnapi = bp->bnapi[i];
4587                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4588                 }
4589
4590                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4591                                       sizeof(struct bnxt_tx_ring_info),
4592                                       GFP_KERNEL);
4593                 if (!bp->tx_ring)
4594                         return -ENOMEM;
4595
4596                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4597                                           GFP_KERNEL);
4598
4599                 if (!bp->tx_ring_map)
4600                         return -ENOMEM;
4601
4602                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4603                         j = 0;
4604                 else
4605                         j = bp->rx_nr_rings;
4606
4607                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4608                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4609
4610                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4611                                 txr->tx_ring_struct.ring_mem.flags =
4612                                         BNXT_RMEM_RING_PTE_FLAG;
4613                         txr->bnapi = bp->bnapi[j];
4614                         bp->bnapi[j]->tx_ring = txr;
4615                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4616                         if (i >= bp->tx_nr_rings_xdp) {
4617                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4618                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4619                         } else {
4620                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4621                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4622                         }
4623                 }
4624
4625                 rc = bnxt_alloc_stats(bp);
4626                 if (rc)
4627                         goto alloc_mem_err;
4628                 bnxt_init_stats(bp);
4629
4630                 rc = bnxt_alloc_ntp_fltrs(bp);
4631                 if (rc)
4632                         goto alloc_mem_err;
4633
4634                 rc = bnxt_alloc_vnics(bp);
4635                 if (rc)
4636                         goto alloc_mem_err;
4637         }
4638
4639         rc = bnxt_alloc_all_cp_arrays(bp);
4640         if (rc)
4641                 goto alloc_mem_err;
4642
4643         bnxt_init_ring_struct(bp);
4644
4645         rc = bnxt_alloc_rx_rings(bp);
4646         if (rc)
4647                 goto alloc_mem_err;
4648
4649         rc = bnxt_alloc_tx_rings(bp);
4650         if (rc)
4651                 goto alloc_mem_err;
4652
4653         rc = bnxt_alloc_cp_rings(bp);
4654         if (rc)
4655                 goto alloc_mem_err;
4656
4657         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4658                                   BNXT_VNIC_UCAST_FLAG;
4659         rc = bnxt_alloc_vnic_attributes(bp);
4660         if (rc)
4661                 goto alloc_mem_err;
4662         return 0;
4663
4664 alloc_mem_err:
4665         bnxt_free_mem(bp, true);
4666         return rc;
4667 }
4668
4669 static void bnxt_disable_int(struct bnxt *bp)
4670 {
4671         int i;
4672
4673         if (!bp->bnapi)
4674                 return;
4675
4676         for (i = 0; i < bp->cp_nr_rings; i++) {
4677                 struct bnxt_napi *bnapi = bp->bnapi[i];
4678                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4679                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4680
4681                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4682                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4683         }
4684 }
4685
4686 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4687 {
4688         struct bnxt_napi *bnapi = bp->bnapi[n];
4689         struct bnxt_cp_ring_info *cpr;
4690
4691         cpr = &bnapi->cp_ring;
4692         return cpr->cp_ring_struct.map_idx;
4693 }
4694
4695 static void bnxt_disable_int_sync(struct bnxt *bp)
4696 {
4697         int i;
4698
4699         if (!bp->irq_tbl)
4700                 return;
4701
4702         atomic_inc(&bp->intr_sem);
4703
4704         bnxt_disable_int(bp);
4705         for (i = 0; i < bp->cp_nr_rings; i++) {
4706                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4707
4708                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4709         }
4710 }
4711
4712 static void bnxt_enable_int(struct bnxt *bp)
4713 {
4714         int i;
4715
4716         atomic_set(&bp->intr_sem, 0);
4717         for (i = 0; i < bp->cp_nr_rings; i++) {
4718                 struct bnxt_napi *bnapi = bp->bnapi[i];
4719                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4720
4721                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4722         }
4723 }
4724
4725 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4726                             bool async_only)
4727 {
4728         DECLARE_BITMAP(async_events_bmap, 256);
4729         u32 *events = (u32 *)async_events_bmap;
4730         struct hwrm_func_drv_rgtr_output *resp;
4731         struct hwrm_func_drv_rgtr_input *req;
4732         u32 flags;
4733         int rc, i;
4734
4735         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
4736         if (rc)
4737                 return rc;
4738
4739         req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4740                                    FUNC_DRV_RGTR_REQ_ENABLES_VER |
4741                                    FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4742
4743         req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4744         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4745         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4746                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4747         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4748                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4749                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4750         req->flags = cpu_to_le32(flags);
4751         req->ver_maj_8b = DRV_VER_MAJ;
4752         req->ver_min_8b = DRV_VER_MIN;
4753         req->ver_upd_8b = DRV_VER_UPD;
4754         req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
4755         req->ver_min = cpu_to_le16(DRV_VER_MIN);
4756         req->ver_upd = cpu_to_le16(DRV_VER_UPD);
4757
4758         if (BNXT_PF(bp)) {
4759                 u32 data[8];
4760                 int i;
4761
4762                 memset(data, 0, sizeof(data));
4763                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4764                         u16 cmd = bnxt_vf_req_snif[i];
4765                         unsigned int bit, idx;
4766
4767                         idx = cmd / 32;
4768                         bit = cmd % 32;
4769                         data[idx] |= 1 << bit;
4770                 }
4771
4772                 for (i = 0; i < 8; i++)
4773                         req->vf_req_fwd[i] = cpu_to_le32(data[i]);
4774
4775                 req->enables |=
4776                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4777         }
4778
4779         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4780                 req->flags |= cpu_to_le32(
4781                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4782
4783         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4784         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4785                 u16 event_id = bnxt_async_events_arr[i];
4786
4787                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4788                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4789                         continue;
4790                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4791         }
4792         if (bmap && bmap_size) {
4793                 for (i = 0; i < bmap_size; i++) {
4794                         if (test_bit(i, bmap))
4795                                 __set_bit(i, async_events_bmap);
4796                 }
4797         }
4798         for (i = 0; i < 8; i++)
4799                 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
4800
4801         if (async_only)
4802                 req->enables =
4803                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4804
4805         resp = hwrm_req_hold(bp, req);
4806         rc = hwrm_req_send(bp, req);
4807         if (!rc) {
4808                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4809                 if (resp->flags &
4810                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4811                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4812         }
4813         hwrm_req_drop(bp, req);
4814         return rc;
4815 }
4816
4817 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4818 {
4819         struct hwrm_func_drv_unrgtr_input *req;
4820         int rc;
4821
4822         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4823                 return 0;
4824
4825         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
4826         if (rc)
4827                 return rc;
4828         return hwrm_req_send(bp, req);
4829 }
4830
4831 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4832 {
4833         struct hwrm_tunnel_dst_port_free_input *req;
4834         int rc;
4835
4836         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
4837             bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
4838                 return 0;
4839         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
4840             bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
4841                 return 0;
4842
4843         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
4844         if (rc)
4845                 return rc;
4846
4847         req->tunnel_type = tunnel_type;
4848
4849         switch (tunnel_type) {
4850         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4851                 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4852                 bp->vxlan_port = 0;
4853                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4854                 break;
4855         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4856                 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4857                 bp->nge_port = 0;
4858                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4859                 break;
4860         default:
4861                 break;
4862         }
4863
4864         rc = hwrm_req_send(bp, req);
4865         if (rc)
4866                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4867                            rc);
4868         return rc;
4869 }
4870
4871 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4872                                            u8 tunnel_type)
4873 {
4874         struct hwrm_tunnel_dst_port_alloc_output *resp;
4875         struct hwrm_tunnel_dst_port_alloc_input *req;
4876         int rc;
4877
4878         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
4879         if (rc)
4880                 return rc;
4881
4882         req->tunnel_type = tunnel_type;
4883         req->tunnel_dst_port_val = port;
4884
4885         resp = hwrm_req_hold(bp, req);
4886         rc = hwrm_req_send(bp, req);
4887         if (rc) {
4888                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4889                            rc);
4890                 goto err_out;
4891         }
4892
4893         switch (tunnel_type) {
4894         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4895                 bp->vxlan_port = port;
4896                 bp->vxlan_fw_dst_port_id =
4897                         le16_to_cpu(resp->tunnel_dst_port_id);
4898                 break;
4899         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4900                 bp->nge_port = port;
4901                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4902                 break;
4903         default:
4904                 break;
4905         }
4906
4907 err_out:
4908         hwrm_req_drop(bp, req);
4909         return rc;
4910 }
4911
4912 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4913 {
4914         struct hwrm_cfa_l2_set_rx_mask_input *req;
4915         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4916         int rc;
4917
4918         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
4919         if (rc)
4920                 return rc;
4921
4922         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4923         if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
4924                 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4925                 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4926         }
4927         req->mask = cpu_to_le32(vnic->rx_mask);
4928         return hwrm_req_send_silent(bp, req);
4929 }
4930
4931 #ifdef CONFIG_RFS_ACCEL
4932 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4933                                             struct bnxt_ntuple_filter *fltr)
4934 {
4935         struct hwrm_cfa_ntuple_filter_free_input *req;
4936         int rc;
4937
4938         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
4939         if (rc)
4940                 return rc;
4941
4942         req->ntuple_filter_id = fltr->filter_id;
4943         return hwrm_req_send(bp, req);
4944 }
4945
4946 #define BNXT_NTP_FLTR_FLAGS                                     \
4947         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4948          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4949          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4950          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4951          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4952          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4953          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4954          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4955          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4956          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4957          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4958          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4959          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4960          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4961
4962 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4963                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4964
4965 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4966                                              struct bnxt_ntuple_filter *fltr)
4967 {
4968         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4969         struct hwrm_cfa_ntuple_filter_alloc_input *req;
4970         struct flow_keys *keys = &fltr->fkeys;
4971         struct bnxt_vnic_info *vnic;
4972         u32 flags = 0;
4973         int rc;
4974
4975         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
4976         if (rc)
4977                 return rc;
4978
4979         req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4980
4981         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4982                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4983                 req->dst_id = cpu_to_le16(fltr->rxq);
4984         } else {
4985                 vnic = &bp->vnic_info[fltr->rxq + 1];
4986                 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
4987         }
4988         req->flags = cpu_to_le32(flags);
4989         req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4990
4991         req->ethertype = htons(ETH_P_IP);
4992         memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4993         req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4994         req->ip_protocol = keys->basic.ip_proto;
4995
4996         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4997                 int i;
4998
4999                 req->ethertype = htons(ETH_P_IPV6);
5000                 req->ip_addr_type =
5001                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
5002                 *(struct in6_addr *)&req->src_ipaddr[0] =
5003                         keys->addrs.v6addrs.src;
5004                 *(struct in6_addr *)&req->dst_ipaddr[0] =
5005                         keys->addrs.v6addrs.dst;
5006                 for (i = 0; i < 4; i++) {
5007                         req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
5008                         req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
5009                 }
5010         } else {
5011                 req->src_ipaddr[0] = keys->addrs.v4addrs.src;
5012                 req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
5013                 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
5014                 req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
5015         }
5016         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
5017                 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
5018                 req->tunnel_type =
5019                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
5020         }
5021
5022         req->src_port = keys->ports.src;
5023         req->src_port_mask = cpu_to_be16(0xffff);
5024         req->dst_port = keys->ports.dst;
5025         req->dst_port_mask = cpu_to_be16(0xffff);
5026
5027         resp = hwrm_req_hold(bp, req);
5028         rc = hwrm_req_send(bp, req);
5029         if (!rc)
5030                 fltr->filter_id = resp->ntuple_filter_id;
5031         hwrm_req_drop(bp, req);
5032         return rc;
5033 }
5034 #endif
5035
5036 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5037                                      const u8 *mac_addr)
5038 {
5039         struct hwrm_cfa_l2_filter_alloc_output *resp;
5040         struct hwrm_cfa_l2_filter_alloc_input *req;
5041         int rc;
5042
5043         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5044         if (rc)
5045                 return rc;
5046
5047         req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5048         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5049                 req->flags |=
5050                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5051         req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
5052         req->enables =
5053                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5054                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5055                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5056         memcpy(req->l2_addr, mac_addr, ETH_ALEN);
5057         req->l2_addr_mask[0] = 0xff;
5058         req->l2_addr_mask[1] = 0xff;
5059         req->l2_addr_mask[2] = 0xff;
5060         req->l2_addr_mask[3] = 0xff;
5061         req->l2_addr_mask[4] = 0xff;
5062         req->l2_addr_mask[5] = 0xff;
5063
5064         resp = hwrm_req_hold(bp, req);
5065         rc = hwrm_req_send(bp, req);
5066         if (!rc)
5067                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
5068                                                         resp->l2_filter_id;
5069         hwrm_req_drop(bp, req);
5070         return rc;
5071 }
5072
5073 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5074 {
5075         struct hwrm_cfa_l2_filter_free_input *req;
5076         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5077         int rc;
5078
5079         /* Any associated ntuple filters will also be cleared by firmware. */
5080         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5081         if (rc)
5082                 return rc;
5083         hwrm_req_hold(bp, req);
5084         for (i = 0; i < num_of_vnics; i++) {
5085                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5086
5087                 for (j = 0; j < vnic->uc_filter_count; j++) {
5088                         req->l2_filter_id = vnic->fw_l2_filter_id[j];
5089
5090                         rc = hwrm_req_send(bp, req);
5091                 }
5092                 vnic->uc_filter_count = 0;
5093         }
5094         hwrm_req_drop(bp, req);
5095         return rc;
5096 }
5097
5098 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5099 {
5100         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5101         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5102         struct hwrm_vnic_tpa_cfg_input *req;
5103         int rc;
5104
5105         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5106                 return 0;
5107
5108         rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5109         if (rc)
5110                 return rc;
5111
5112         if (tpa_flags) {
5113                 u16 mss = bp->dev->mtu - 40;
5114                 u32 nsegs, n, segs = 0, flags;
5115
5116                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5117                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5118                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5119                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5120                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5121                 if (tpa_flags & BNXT_FLAG_GRO)
5122                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5123
5124                 req->flags = cpu_to_le32(flags);
5125
5126                 req->enables =
5127                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5128                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5129                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5130
5131                 /* Number of segs are log2 units, and first packet is not
5132                  * included as part of this units.
5133                  */
5134                 if (mss <= BNXT_RX_PAGE_SIZE) {
5135                         n = BNXT_RX_PAGE_SIZE / mss;
5136                         nsegs = (MAX_SKB_FRAGS - 1) * n;
5137                 } else {
5138                         n = mss / BNXT_RX_PAGE_SIZE;
5139                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
5140                                 n++;
5141                         nsegs = (MAX_SKB_FRAGS - n) / n;
5142                 }
5143
5144                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5145                         segs = MAX_TPA_SEGS_P5;
5146                         max_aggs = bp->max_tpa;
5147                 } else {
5148                         segs = ilog2(nsegs);
5149                 }
5150                 req->max_agg_segs = cpu_to_le16(segs);
5151                 req->max_aggs = cpu_to_le16(max_aggs);
5152
5153                 req->min_agg_len = cpu_to_le32(512);
5154         }
5155         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5156
5157         return hwrm_req_send(bp, req);
5158 }
5159
5160 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5161 {
5162         struct bnxt_ring_grp_info *grp_info;
5163
5164         grp_info = &bp->grp_info[ring->grp_idx];
5165         return grp_info->cp_fw_ring_id;
5166 }
5167
5168 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5169 {
5170         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5171                 struct bnxt_napi *bnapi = rxr->bnapi;
5172                 struct bnxt_cp_ring_info *cpr;
5173
5174                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5175                 return cpr->cp_ring_struct.fw_ring_id;
5176         } else {
5177                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5178         }
5179 }
5180
5181 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5182 {
5183         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5184                 struct bnxt_napi *bnapi = txr->bnapi;
5185                 struct bnxt_cp_ring_info *cpr;
5186
5187                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5188                 return cpr->cp_ring_struct.fw_ring_id;
5189         } else {
5190                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5191         }
5192 }
5193
5194 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5195 {
5196         int entries;
5197
5198         if (bp->flags & BNXT_FLAG_CHIP_P5)
5199                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5200         else
5201                 entries = HW_HASH_INDEX_SIZE;
5202
5203         bp->rss_indir_tbl_entries = entries;
5204         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5205                                           GFP_KERNEL);
5206         if (!bp->rss_indir_tbl)
5207                 return -ENOMEM;
5208         return 0;
5209 }
5210
5211 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5212 {
5213         u16 max_rings, max_entries, pad, i;
5214
5215         if (!bp->rx_nr_rings)
5216                 return;
5217
5218         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5219                 max_rings = bp->rx_nr_rings - 1;
5220         else
5221                 max_rings = bp->rx_nr_rings;
5222
5223         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5224
5225         for (i = 0; i < max_entries; i++)
5226                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5227
5228         pad = bp->rss_indir_tbl_entries - max_entries;
5229         if (pad)
5230                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5231 }
5232
5233 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5234 {
5235         u16 i, tbl_size, max_ring = 0;
5236
5237         if (!bp->rss_indir_tbl)
5238                 return 0;
5239
5240         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5241         for (i = 0; i < tbl_size; i++)
5242                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5243         return max_ring;
5244 }
5245
5246 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5247 {
5248         if (bp->flags & BNXT_FLAG_CHIP_P5)
5249                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5250         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5251                 return 2;
5252         return 1;
5253 }
5254
5255 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5256 {
5257         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5258         u16 i, j;
5259
5260         /* Fill the RSS indirection table with ring group ids */
5261         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5262                 if (!no_rss)
5263                         j = bp->rss_indir_tbl[i];
5264                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5265         }
5266 }
5267
5268 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5269                                       struct bnxt_vnic_info *vnic)
5270 {
5271         __le16 *ring_tbl = vnic->rss_table;
5272         struct bnxt_rx_ring_info *rxr;
5273         u16 tbl_size, i;
5274
5275         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5276
5277         for (i = 0; i < tbl_size; i++) {
5278                 u16 ring_id, j;
5279
5280                 j = bp->rss_indir_tbl[i];
5281                 rxr = &bp->rx_ring[j];
5282
5283                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5284                 *ring_tbl++ = cpu_to_le16(ring_id);
5285                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5286                 *ring_tbl++ = cpu_to_le16(ring_id);
5287         }
5288 }
5289
5290 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5291 {
5292         if (bp->flags & BNXT_FLAG_CHIP_P5)
5293                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5294         else
5295                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5296 }
5297
5298 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5299 {
5300         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5301         struct hwrm_vnic_rss_cfg_input *req;
5302         int rc;
5303
5304         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5305             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5306                 return 0;
5307
5308         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5309         if (rc)
5310                 return rc;
5311
5312         if (set_rss) {
5313                 bnxt_fill_hw_rss_tbl(bp, vnic);
5314                 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
5315                 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5316                 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5317                 req->hash_key_tbl_addr =
5318                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5319         }
5320         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5321         return hwrm_req_send(bp, req);
5322 }
5323
5324 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5325 {
5326         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5327         struct hwrm_vnic_rss_cfg_input *req;
5328         dma_addr_t ring_tbl_map;
5329         u32 i, nr_ctxs;
5330         int rc;
5331
5332         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5333         if (rc)
5334                 return rc;
5335
5336         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5337         if (!set_rss)
5338                 return hwrm_req_send(bp, req);
5339
5340         bnxt_fill_hw_rss_tbl(bp, vnic);
5341         req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
5342         req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5343         req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5344         ring_tbl_map = vnic->rss_table_dma_addr;
5345         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5346
5347         hwrm_req_hold(bp, req);
5348         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5349                 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5350                 req->ring_table_pair_index = i;
5351                 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5352                 rc = hwrm_req_send(bp, req);
5353                 if (rc)
5354                         goto exit;
5355         }
5356
5357 exit:
5358         hwrm_req_drop(bp, req);
5359         return rc;
5360 }
5361
5362 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5363 {
5364         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5365         struct hwrm_vnic_plcmodes_cfg_input *req;
5366         int rc;
5367
5368         rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
5369         if (rc)
5370                 return rc;
5371
5372         req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
5373         req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
5374
5375         if (BNXT_RX_PAGE_MODE(bp) && !BNXT_RX_JUMBO_MODE(bp)) {
5376                 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5377                                           VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5378                 req->enables |=
5379                         cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5380         }
5381         /* thresholds not implemented in firmware yet */
5382         req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5383         req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5384         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5385         return hwrm_req_send(bp, req);
5386 }
5387
5388 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5389                                         u16 ctx_idx)
5390 {
5391         struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
5392
5393         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
5394                 return;
5395
5396         req->rss_cos_lb_ctx_id =
5397                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5398
5399         hwrm_req_send(bp, req);
5400         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5401 }
5402
5403 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5404 {
5405         int i, j;
5406
5407         for (i = 0; i < bp->nr_vnics; i++) {
5408                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5409
5410                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5411                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5412                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5413                 }
5414         }
5415         bp->rsscos_nr_ctxs = 0;
5416 }
5417
5418 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5419 {
5420         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
5421         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
5422         int rc;
5423
5424         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
5425         if (rc)
5426                 return rc;
5427
5428         resp = hwrm_req_hold(bp, req);
5429         rc = hwrm_req_send(bp, req);
5430         if (!rc)
5431                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5432                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5433         hwrm_req_drop(bp, req);
5434
5435         return rc;
5436 }
5437
5438 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5439 {
5440         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5441                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5442         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5443 }
5444
5445 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5446 {
5447         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5448         struct hwrm_vnic_cfg_input *req;
5449         unsigned int ring = 0, grp_idx;
5450         u16 def_vlan = 0;
5451         int rc;
5452
5453         rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
5454         if (rc)
5455                 return rc;
5456
5457         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5458                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5459
5460                 req->default_rx_ring_id =
5461                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5462                 req->default_cmpl_ring_id =
5463                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5464                 req->enables =
5465                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5466                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5467                 goto vnic_mru;
5468         }
5469         req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5470         /* Only RSS support for now TBD: COS & LB */
5471         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5472                 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5473                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5474                                            VNIC_CFG_REQ_ENABLES_MRU);
5475         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5476                 req->rss_rule =
5477                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5478                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5479                                            VNIC_CFG_REQ_ENABLES_MRU);
5480                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5481         } else {
5482                 req->rss_rule = cpu_to_le16(0xffff);
5483         }
5484
5485         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5486             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5487                 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5488                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5489         } else {
5490                 req->cos_rule = cpu_to_le16(0xffff);
5491         }
5492
5493         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5494                 ring = 0;
5495         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5496                 ring = vnic_id - 1;
5497         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5498                 ring = bp->rx_nr_rings - 1;
5499
5500         grp_idx = bp->rx_ring[ring].bnapi->index;
5501         req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5502         req->lb_rule = cpu_to_le16(0xffff);
5503 vnic_mru:
5504         req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5505
5506         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5507 #ifdef CONFIG_BNXT_SRIOV
5508         if (BNXT_VF(bp))
5509                 def_vlan = bp->vf.vlan;
5510 #endif
5511         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5512                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5513         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5514                 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5515
5516         return hwrm_req_send(bp, req);
5517 }
5518
5519 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5520 {
5521         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5522                 struct hwrm_vnic_free_input *req;
5523
5524                 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
5525                         return;
5526
5527                 req->vnic_id =
5528                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5529
5530                 hwrm_req_send(bp, req);
5531                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5532         }
5533 }
5534
5535 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5536 {
5537         u16 i;
5538
5539         for (i = 0; i < bp->nr_vnics; i++)
5540                 bnxt_hwrm_vnic_free_one(bp, i);
5541 }
5542
5543 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5544                                 unsigned int start_rx_ring_idx,
5545                                 unsigned int nr_rings)
5546 {
5547         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5548         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5549         struct hwrm_vnic_alloc_output *resp;
5550         struct hwrm_vnic_alloc_input *req;
5551         int rc;
5552
5553         rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
5554         if (rc)
5555                 return rc;
5556
5557         if (bp->flags & BNXT_FLAG_CHIP_P5)
5558                 goto vnic_no_ring_grps;
5559
5560         /* map ring groups to this vnic */
5561         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5562                 grp_idx = bp->rx_ring[i].bnapi->index;
5563                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5564                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5565                                    j, nr_rings);
5566                         break;
5567                 }
5568                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5569         }
5570
5571 vnic_no_ring_grps:
5572         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5573                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5574         if (vnic_id == 0)
5575                 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5576
5577         resp = hwrm_req_hold(bp, req);
5578         rc = hwrm_req_send(bp, req);
5579         if (!rc)
5580                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5581         hwrm_req_drop(bp, req);
5582         return rc;
5583 }
5584
5585 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5586 {
5587         struct hwrm_vnic_qcaps_output *resp;
5588         struct hwrm_vnic_qcaps_input *req;
5589         int rc;
5590
5591         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5592         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5593         if (bp->hwrm_spec_code < 0x10600)
5594                 return 0;
5595
5596         rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
5597         if (rc)
5598                 return rc;
5599
5600         resp = hwrm_req_hold(bp, req);
5601         rc = hwrm_req_send(bp, req);
5602         if (!rc) {
5603                 u32 flags = le32_to_cpu(resp->flags);
5604
5605                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5606                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5607                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5608                 if (flags &
5609                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5610                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5611
5612                 /* Older P5 fw before EXT_HW_STATS support did not set
5613                  * VLAN_STRIP_CAP properly.
5614                  */
5615                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5616                     (BNXT_CHIP_P5_THOR(bp) &&
5617                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5618                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5619                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5620                 if (bp->max_tpa_v2) {
5621                         if (BNXT_CHIP_P5_THOR(bp))
5622                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5623                         else
5624                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5625                 }
5626         }
5627         hwrm_req_drop(bp, req);
5628         return rc;
5629 }
5630
5631 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5632 {
5633         struct hwrm_ring_grp_alloc_output *resp;
5634         struct hwrm_ring_grp_alloc_input *req;
5635         int rc;
5636         u16 i;
5637
5638         if (bp->flags & BNXT_FLAG_CHIP_P5)
5639                 return 0;
5640
5641         rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
5642         if (rc)
5643                 return rc;
5644
5645         resp = hwrm_req_hold(bp, req);
5646         for (i = 0; i < bp->rx_nr_rings; i++) {
5647                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5648
5649                 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5650                 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5651                 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5652                 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5653
5654                 rc = hwrm_req_send(bp, req);
5655
5656                 if (rc)
5657                         break;
5658
5659                 bp->grp_info[grp_idx].fw_grp_id =
5660                         le32_to_cpu(resp->ring_group_id);
5661         }
5662         hwrm_req_drop(bp, req);
5663         return rc;
5664 }
5665
5666 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5667 {
5668         struct hwrm_ring_grp_free_input *req;
5669         u16 i;
5670
5671         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5672                 return;
5673
5674         if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
5675                 return;
5676
5677         hwrm_req_hold(bp, req);
5678         for (i = 0; i < bp->cp_nr_rings; i++) {
5679                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5680                         continue;
5681                 req->ring_group_id =
5682                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5683
5684                 hwrm_req_send(bp, req);
5685                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5686         }
5687         hwrm_req_drop(bp, req);
5688 }
5689
5690 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5691                                     struct bnxt_ring_struct *ring,
5692                                     u32 ring_type, u32 map_index)
5693 {
5694         struct hwrm_ring_alloc_output *resp;
5695         struct hwrm_ring_alloc_input *req;
5696         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5697         struct bnxt_ring_grp_info *grp_info;
5698         int rc, err = 0;
5699         u16 ring_id;
5700
5701         rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
5702         if (rc)
5703                 goto exit;
5704
5705         req->enables = 0;
5706         if (rmem->nr_pages > 1) {
5707                 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5708                 /* Page size is in log2 units */
5709                 req->page_size = BNXT_PAGE_SHIFT;
5710                 req->page_tbl_depth = 1;
5711         } else {
5712                 req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5713         }
5714         req->fbo = 0;
5715         /* Association of ring index with doorbell index and MSIX number */
5716         req->logical_id = cpu_to_le16(map_index);
5717
5718         switch (ring_type) {
5719         case HWRM_RING_ALLOC_TX: {
5720                 struct bnxt_tx_ring_info *txr;
5721
5722                 txr = container_of(ring, struct bnxt_tx_ring_info,
5723                                    tx_ring_struct);
5724                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5725                 /* Association of transmit ring with completion ring */
5726                 grp_info = &bp->grp_info[ring->grp_idx];
5727                 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5728                 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
5729                 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5730                 req->queue_id = cpu_to_le16(ring->queue_id);
5731                 break;
5732         }
5733         case HWRM_RING_ALLOC_RX:
5734                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5735                 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
5736                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5737                         u16 flags = 0;
5738
5739                         /* Association of rx ring with stats context */
5740                         grp_info = &bp->grp_info[ring->grp_idx];
5741                         req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5742                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5743                         req->enables |= cpu_to_le32(
5744                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5745                         if (NET_IP_ALIGN == 2)
5746                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5747                         req->flags = cpu_to_le16(flags);
5748                 }
5749                 break;
5750         case HWRM_RING_ALLOC_AGG:
5751                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5752                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5753                         /* Association of agg ring with rx ring */
5754                         grp_info = &bp->grp_info[ring->grp_idx];
5755                         req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5756                         req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5757                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5758                         req->enables |= cpu_to_le32(
5759                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5760                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5761                 } else {
5762                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5763                 }
5764                 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5765                 break;
5766         case HWRM_RING_ALLOC_CMPL:
5767                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5768                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5769                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5770                         /* Association of cp ring with nq */
5771                         grp_info = &bp->grp_info[map_index];
5772                         req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5773                         req->cq_handle = cpu_to_le64(ring->handle);
5774                         req->enables |= cpu_to_le32(
5775                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5776                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5777                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5778                 }
5779                 break;
5780         case HWRM_RING_ALLOC_NQ:
5781                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5782                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5783                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5784                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5785                 break;
5786         default:
5787                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5788                            ring_type);
5789                 return -1;
5790         }
5791
5792         resp = hwrm_req_hold(bp, req);
5793         rc = hwrm_req_send(bp, req);
5794         err = le16_to_cpu(resp->error_code);
5795         ring_id = le16_to_cpu(resp->ring_id);
5796         hwrm_req_drop(bp, req);
5797
5798 exit:
5799         if (rc || err) {
5800                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5801                            ring_type, rc, err);
5802                 return -EIO;
5803         }
5804         ring->fw_ring_id = ring_id;
5805         return rc;
5806 }
5807
5808 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5809 {
5810         int rc;
5811
5812         if (BNXT_PF(bp)) {
5813                 struct hwrm_func_cfg_input *req;
5814
5815                 rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
5816                 if (rc)
5817                         return rc;
5818
5819                 req->fid = cpu_to_le16(0xffff);
5820                 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5821                 req->async_event_cr = cpu_to_le16(idx);
5822                 return hwrm_req_send(bp, req);
5823         } else {
5824                 struct hwrm_func_vf_cfg_input *req;
5825
5826                 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
5827                 if (rc)
5828                         return rc;
5829
5830                 req->enables =
5831                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5832                 req->async_event_cr = cpu_to_le16(idx);
5833                 return hwrm_req_send(bp, req);
5834         }
5835 }
5836
5837 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5838                         u32 map_idx, u32 xid)
5839 {
5840         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5841                 if (BNXT_PF(bp))
5842                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5843                 else
5844                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5845                 switch (ring_type) {
5846                 case HWRM_RING_ALLOC_TX:
5847                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5848                         break;
5849                 case HWRM_RING_ALLOC_RX:
5850                 case HWRM_RING_ALLOC_AGG:
5851                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5852                         break;
5853                 case HWRM_RING_ALLOC_CMPL:
5854                         db->db_key64 = DBR_PATH_L2;
5855                         break;
5856                 case HWRM_RING_ALLOC_NQ:
5857                         db->db_key64 = DBR_PATH_L2;
5858                         break;
5859                 }
5860                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5861         } else {
5862                 db->doorbell = bp->bar1 + map_idx * 0x80;
5863                 switch (ring_type) {
5864                 case HWRM_RING_ALLOC_TX:
5865                         db->db_key32 = DB_KEY_TX;
5866                         break;
5867                 case HWRM_RING_ALLOC_RX:
5868                 case HWRM_RING_ALLOC_AGG:
5869                         db->db_key32 = DB_KEY_RX;
5870                         break;
5871                 case HWRM_RING_ALLOC_CMPL:
5872                         db->db_key32 = DB_KEY_CP;
5873                         break;
5874                 }
5875         }
5876 }
5877
5878 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5879 {
5880         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5881         int i, rc = 0;
5882         u32 type;
5883
5884         if (bp->flags & BNXT_FLAG_CHIP_P5)
5885                 type = HWRM_RING_ALLOC_NQ;
5886         else
5887                 type = HWRM_RING_ALLOC_CMPL;
5888         for (i = 0; i < bp->cp_nr_rings; i++) {
5889                 struct bnxt_napi *bnapi = bp->bnapi[i];
5890                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5891                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5892                 u32 map_idx = ring->map_idx;
5893                 unsigned int vector;
5894
5895                 vector = bp->irq_tbl[map_idx].vector;
5896                 disable_irq_nosync(vector);
5897                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5898                 if (rc) {
5899                         enable_irq(vector);
5900                         goto err_out;
5901                 }
5902                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5903                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5904                 enable_irq(vector);
5905                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5906
5907                 if (!i) {
5908                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5909                         if (rc)
5910                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5911                 }
5912         }
5913
5914         type = HWRM_RING_ALLOC_TX;
5915         for (i = 0; i < bp->tx_nr_rings; i++) {
5916                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5917                 struct bnxt_ring_struct *ring;
5918                 u32 map_idx;
5919
5920                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5921                         struct bnxt_napi *bnapi = txr->bnapi;
5922                         struct bnxt_cp_ring_info *cpr, *cpr2;
5923                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5924
5925                         cpr = &bnapi->cp_ring;
5926                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5927                         ring = &cpr2->cp_ring_struct;
5928                         ring->handle = BNXT_TX_HDL;
5929                         map_idx = bnapi->index;
5930                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5931                         if (rc)
5932                                 goto err_out;
5933                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5934                                     ring->fw_ring_id);
5935                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5936                 }
5937                 ring = &txr->tx_ring_struct;
5938                 map_idx = i;
5939                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5940                 if (rc)
5941                         goto err_out;
5942                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5943         }
5944
5945         type = HWRM_RING_ALLOC_RX;
5946         for (i = 0; i < bp->rx_nr_rings; i++) {
5947                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5948                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5949                 struct bnxt_napi *bnapi = rxr->bnapi;
5950                 u32 map_idx = bnapi->index;
5951
5952                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5953                 if (rc)
5954                         goto err_out;
5955                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5956                 /* If we have agg rings, post agg buffers first. */
5957                 if (!agg_rings)
5958                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5959                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5960                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5961                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5962                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5963                         struct bnxt_cp_ring_info *cpr2;
5964
5965                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5966                         ring = &cpr2->cp_ring_struct;
5967                         ring->handle = BNXT_RX_HDL;
5968                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5969                         if (rc)
5970                                 goto err_out;
5971                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5972                                     ring->fw_ring_id);
5973                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5974                 }
5975         }
5976
5977         if (agg_rings) {
5978                 type = HWRM_RING_ALLOC_AGG;
5979                 for (i = 0; i < bp->rx_nr_rings; i++) {
5980                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5981                         struct bnxt_ring_struct *ring =
5982                                                 &rxr->rx_agg_ring_struct;
5983                         u32 grp_idx = ring->grp_idx;
5984                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5985
5986                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5987                         if (rc)
5988                                 goto err_out;
5989
5990                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5991                                     ring->fw_ring_id);
5992                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5993                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5994                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5995                 }
5996         }
5997 err_out:
5998         return rc;
5999 }
6000
6001 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6002                                    struct bnxt_ring_struct *ring,
6003                                    u32 ring_type, int cmpl_ring_id)
6004 {
6005         struct hwrm_ring_free_output *resp;
6006         struct hwrm_ring_free_input *req;
6007         u16 error_code = 0;
6008         int rc;
6009
6010         if (BNXT_NO_FW_ACCESS(bp))
6011                 return 0;
6012
6013         rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6014         if (rc)
6015                 goto exit;
6016
6017         req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6018         req->ring_type = ring_type;
6019         req->ring_id = cpu_to_le16(ring->fw_ring_id);
6020
6021         resp = hwrm_req_hold(bp, req);
6022         rc = hwrm_req_send(bp, req);
6023         error_code = le16_to_cpu(resp->error_code);
6024         hwrm_req_drop(bp, req);
6025 exit:
6026         if (rc || error_code) {
6027                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6028                            ring_type, rc, error_code);
6029                 return -EIO;
6030         }
6031         return 0;
6032 }
6033
6034 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6035 {
6036         u32 type;
6037         int i;
6038
6039         if (!bp->bnapi)
6040                 return;
6041
6042         for (i = 0; i < bp->tx_nr_rings; i++) {
6043                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6044                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6045
6046                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6047                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6048
6049                         hwrm_ring_free_send_msg(bp, ring,
6050                                                 RING_FREE_REQ_RING_TYPE_TX,
6051                                                 close_path ? cmpl_ring_id :
6052                                                 INVALID_HW_RING_ID);
6053                         ring->fw_ring_id = INVALID_HW_RING_ID;
6054                 }
6055         }
6056
6057         for (i = 0; i < bp->rx_nr_rings; i++) {
6058                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6059                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6060                 u32 grp_idx = rxr->bnapi->index;
6061
6062                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6063                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6064
6065                         hwrm_ring_free_send_msg(bp, ring,
6066                                                 RING_FREE_REQ_RING_TYPE_RX,
6067                                                 close_path ? cmpl_ring_id :
6068                                                 INVALID_HW_RING_ID);
6069                         ring->fw_ring_id = INVALID_HW_RING_ID;
6070                         bp->grp_info[grp_idx].rx_fw_ring_id =
6071                                 INVALID_HW_RING_ID;
6072                 }
6073         }
6074
6075         if (bp->flags & BNXT_FLAG_CHIP_P5)
6076                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6077         else
6078                 type = RING_FREE_REQ_RING_TYPE_RX;
6079         for (i = 0; i < bp->rx_nr_rings; i++) {
6080                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6081                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6082                 u32 grp_idx = rxr->bnapi->index;
6083
6084                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6085                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6086
6087                         hwrm_ring_free_send_msg(bp, ring, type,
6088                                                 close_path ? cmpl_ring_id :
6089                                                 INVALID_HW_RING_ID);
6090                         ring->fw_ring_id = INVALID_HW_RING_ID;
6091                         bp->grp_info[grp_idx].agg_fw_ring_id =
6092                                 INVALID_HW_RING_ID;
6093                 }
6094         }
6095
6096         /* The completion rings are about to be freed.  After that the
6097          * IRQ doorbell will not work anymore.  So we need to disable
6098          * IRQ here.
6099          */
6100         bnxt_disable_int_sync(bp);
6101
6102         if (bp->flags & BNXT_FLAG_CHIP_P5)
6103                 type = RING_FREE_REQ_RING_TYPE_NQ;
6104         else
6105                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6106         for (i = 0; i < bp->cp_nr_rings; i++) {
6107                 struct bnxt_napi *bnapi = bp->bnapi[i];
6108                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6109                 struct bnxt_ring_struct *ring;
6110                 int j;
6111
6112                 for (j = 0; j < 2; j++) {
6113                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
6114
6115                         if (cpr2) {
6116                                 ring = &cpr2->cp_ring_struct;
6117                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
6118                                         continue;
6119                                 hwrm_ring_free_send_msg(bp, ring,
6120                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
6121                                         INVALID_HW_RING_ID);
6122                                 ring->fw_ring_id = INVALID_HW_RING_ID;
6123                         }
6124                 }
6125                 ring = &cpr->cp_ring_struct;
6126                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6127                         hwrm_ring_free_send_msg(bp, ring, type,
6128                                                 INVALID_HW_RING_ID);
6129                         ring->fw_ring_id = INVALID_HW_RING_ID;
6130                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6131                 }
6132         }
6133 }
6134
6135 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6136                            bool shared);
6137
6138 static int bnxt_hwrm_get_rings(struct bnxt *bp)
6139 {
6140         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6141         struct hwrm_func_qcfg_output *resp;
6142         struct hwrm_func_qcfg_input *req;
6143         int rc;
6144
6145         if (bp->hwrm_spec_code < 0x10601)
6146                 return 0;
6147
6148         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6149         if (rc)
6150                 return rc;
6151
6152         req->fid = cpu_to_le16(0xffff);
6153         resp = hwrm_req_hold(bp, req);
6154         rc = hwrm_req_send(bp, req);
6155         if (rc) {
6156                 hwrm_req_drop(bp, req);
6157                 return rc;
6158         }
6159
6160         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6161         if (BNXT_NEW_RM(bp)) {
6162                 u16 cp, stats;
6163
6164                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6165                 hw_resc->resv_hw_ring_grps =
6166                         le32_to_cpu(resp->alloc_hw_ring_grps);
6167                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6168                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
6169                 stats = le16_to_cpu(resp->alloc_stat_ctx);
6170                 hw_resc->resv_irqs = cp;
6171                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6172                         int rx = hw_resc->resv_rx_rings;
6173                         int tx = hw_resc->resv_tx_rings;
6174
6175                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6176                                 rx >>= 1;
6177                         if (cp < (rx + tx)) {
6178                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
6179                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6180                                         rx <<= 1;
6181                                 hw_resc->resv_rx_rings = rx;
6182                                 hw_resc->resv_tx_rings = tx;
6183                         }
6184                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6185                         hw_resc->resv_hw_ring_grps = rx;
6186                 }
6187                 hw_resc->resv_cp_rings = cp;
6188                 hw_resc->resv_stat_ctxs = stats;
6189         }
6190         hwrm_req_drop(bp, req);
6191         return 0;
6192 }
6193
6194 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6195 {
6196         struct hwrm_func_qcfg_output *resp;
6197         struct hwrm_func_qcfg_input *req;
6198         int rc;
6199
6200         if (bp->hwrm_spec_code < 0x10601)
6201                 return 0;
6202
6203         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6204         if (rc)
6205                 return rc;
6206
6207         req->fid = cpu_to_le16(fid);
6208         resp = hwrm_req_hold(bp, req);
6209         rc = hwrm_req_send(bp, req);
6210         if (!rc)
6211                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6212
6213         hwrm_req_drop(bp, req);
6214         return rc;
6215 }
6216
6217 static bool bnxt_rfs_supported(struct bnxt *bp);
6218
6219 static struct hwrm_func_cfg_input *
6220 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6221                              int ring_grps, int cp_rings, int stats, int vnics)
6222 {
6223         struct hwrm_func_cfg_input *req;
6224         u32 enables = 0;
6225
6226         if (hwrm_req_init(bp, req, HWRM_FUNC_CFG))
6227                 return NULL;
6228
6229         req->fid = cpu_to_le16(0xffff);
6230         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6231         req->num_tx_rings = cpu_to_le16(tx_rings);
6232         if (BNXT_NEW_RM(bp)) {
6233                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6234                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6235                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6236                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6237                         enables |= tx_rings + ring_grps ?
6238                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6239                         enables |= rx_rings ?
6240                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6241                 } else {
6242                         enables |= cp_rings ?
6243                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6244                         enables |= ring_grps ?
6245                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6246                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6247                 }
6248                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6249
6250                 req->num_rx_rings = cpu_to_le16(rx_rings);
6251                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6252                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6253                         req->num_msix = cpu_to_le16(cp_rings);
6254                         req->num_rsscos_ctxs =
6255                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6256                 } else {
6257                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6258                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6259                         req->num_rsscos_ctxs = cpu_to_le16(1);
6260                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6261                             bnxt_rfs_supported(bp))
6262                                 req->num_rsscos_ctxs =
6263                                         cpu_to_le16(ring_grps + 1);
6264                 }
6265                 req->num_stat_ctxs = cpu_to_le16(stats);
6266                 req->num_vnics = cpu_to_le16(vnics);
6267         }
6268         req->enables = cpu_to_le32(enables);
6269         return req;
6270 }
6271
6272 static struct hwrm_func_vf_cfg_input *
6273 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6274                              int ring_grps, int cp_rings, int stats, int vnics)
6275 {
6276         struct hwrm_func_vf_cfg_input *req;
6277         u32 enables = 0;
6278
6279         if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
6280                 return NULL;
6281
6282         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6283         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6284                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6285         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6286         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6287                 enables |= tx_rings + ring_grps ?
6288                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6289         } else {
6290                 enables |= cp_rings ?
6291                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6292                 enables |= ring_grps ?
6293                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6294         }
6295         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6296         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6297
6298         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6299         req->num_tx_rings = cpu_to_le16(tx_rings);
6300         req->num_rx_rings = cpu_to_le16(rx_rings);
6301         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6302                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6303                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6304         } else {
6305                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6306                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6307                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6308         }
6309         req->num_stat_ctxs = cpu_to_le16(stats);
6310         req->num_vnics = cpu_to_le16(vnics);
6311
6312         req->enables = cpu_to_le32(enables);
6313         return req;
6314 }
6315
6316 static int
6317 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6318                            int ring_grps, int cp_rings, int stats, int vnics)
6319 {
6320         struct hwrm_func_cfg_input *req;
6321         int rc;
6322
6323         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6324                                            cp_rings, stats, vnics);
6325         if (!req)
6326                 return -ENOMEM;
6327
6328         if (!req->enables) {
6329                 hwrm_req_drop(bp, req);
6330                 return 0;
6331         }
6332
6333         rc = hwrm_req_send(bp, req);
6334         if (rc)
6335                 return rc;
6336
6337         if (bp->hwrm_spec_code < 0x10601)
6338                 bp->hw_resc.resv_tx_rings = tx_rings;
6339
6340         return bnxt_hwrm_get_rings(bp);
6341 }
6342
6343 static int
6344 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6345                            int ring_grps, int cp_rings, int stats, int vnics)
6346 {
6347         struct hwrm_func_vf_cfg_input *req;
6348         int rc;
6349
6350         if (!BNXT_NEW_RM(bp)) {
6351                 bp->hw_resc.resv_tx_rings = tx_rings;
6352                 return 0;
6353         }
6354
6355         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6356                                            cp_rings, stats, vnics);
6357         if (!req)
6358                 return -ENOMEM;
6359
6360         rc = hwrm_req_send(bp, req);
6361         if (rc)
6362                 return rc;
6363
6364         return bnxt_hwrm_get_rings(bp);
6365 }
6366
6367 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6368                                    int cp, int stat, int vnic)
6369 {
6370         if (BNXT_PF(bp))
6371                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6372                                                   vnic);
6373         else
6374                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6375                                                   vnic);
6376 }
6377
6378 int bnxt_nq_rings_in_use(struct bnxt *bp)
6379 {
6380         int cp = bp->cp_nr_rings;
6381         int ulp_msix, ulp_base;
6382
6383         ulp_msix = bnxt_get_ulp_msix_num(bp);
6384         if (ulp_msix) {
6385                 ulp_base = bnxt_get_ulp_msix_base(bp);
6386                 cp += ulp_msix;
6387                 if ((ulp_base + ulp_msix) > cp)
6388                         cp = ulp_base + ulp_msix;
6389         }
6390         return cp;
6391 }
6392
6393 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6394 {
6395         int cp;
6396
6397         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6398                 return bnxt_nq_rings_in_use(bp);
6399
6400         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6401         return cp;
6402 }
6403
6404 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6405 {
6406         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6407         int cp = bp->cp_nr_rings;
6408
6409         if (!ulp_stat)
6410                 return cp;
6411
6412         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6413                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6414
6415         return cp + ulp_stat;
6416 }
6417
6418 /* Check if a default RSS map needs to be setup.  This function is only
6419  * used on older firmware that does not require reserving RX rings.
6420  */
6421 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6422 {
6423         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6424
6425         /* The RSS map is valid for RX rings set to resv_rx_rings */
6426         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6427                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6428                 if (!netif_is_rxfh_configured(bp->dev))
6429                         bnxt_set_dflt_rss_indir_tbl(bp);
6430         }
6431 }
6432
6433 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6434 {
6435         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6436         int cp = bnxt_cp_rings_in_use(bp);
6437         int nq = bnxt_nq_rings_in_use(bp);
6438         int rx = bp->rx_nr_rings, stat;
6439         int vnic = 1, grp = rx;
6440
6441         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6442             bp->hwrm_spec_code >= 0x10601)
6443                 return true;
6444
6445         /* Old firmware does not need RX ring reservations but we still
6446          * need to setup a default RSS map when needed.  With new firmware
6447          * we go through RX ring reservations first and then set up the
6448          * RSS map for the successfully reserved RX rings when needed.
6449          */
6450         if (!BNXT_NEW_RM(bp)) {
6451                 bnxt_check_rss_tbl_no_rmgr(bp);
6452                 return false;
6453         }
6454         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6455                 vnic = rx + 1;
6456         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6457                 rx <<= 1;
6458         stat = bnxt_get_func_stat_ctxs(bp);
6459         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6460             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6461             (hw_resc->resv_hw_ring_grps != grp &&
6462              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6463                 return true;
6464         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6465             hw_resc->resv_irqs != nq)
6466                 return true;
6467         return false;
6468 }
6469
6470 static int __bnxt_reserve_rings(struct bnxt *bp)
6471 {
6472         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6473         int cp = bnxt_nq_rings_in_use(bp);
6474         int tx = bp->tx_nr_rings;
6475         int rx = bp->rx_nr_rings;
6476         int grp, rx_rings, rc;
6477         int vnic = 1, stat;
6478         bool sh = false;
6479
6480         if (!bnxt_need_reserve_rings(bp))
6481                 return 0;
6482
6483         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6484                 sh = true;
6485         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6486                 vnic = rx + 1;
6487         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6488                 rx <<= 1;
6489         grp = bp->rx_nr_rings;
6490         stat = bnxt_get_func_stat_ctxs(bp);
6491
6492         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6493         if (rc)
6494                 return rc;
6495
6496         tx = hw_resc->resv_tx_rings;
6497         if (BNXT_NEW_RM(bp)) {
6498                 rx = hw_resc->resv_rx_rings;
6499                 cp = hw_resc->resv_irqs;
6500                 grp = hw_resc->resv_hw_ring_grps;
6501                 vnic = hw_resc->resv_vnics;
6502                 stat = hw_resc->resv_stat_ctxs;
6503         }
6504
6505         rx_rings = rx;
6506         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6507                 if (rx >= 2) {
6508                         rx_rings = rx >> 1;
6509                 } else {
6510                         if (netif_running(bp->dev))
6511                                 return -ENOMEM;
6512
6513                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6514                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6515                         bp->dev->hw_features &= ~NETIF_F_LRO;
6516                         bp->dev->features &= ~NETIF_F_LRO;
6517                         bnxt_set_ring_params(bp);
6518                 }
6519         }
6520         rx_rings = min_t(int, rx_rings, grp);
6521         cp = min_t(int, cp, bp->cp_nr_rings);
6522         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6523                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6524         cp = min_t(int, cp, stat);
6525         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6526         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6527                 rx = rx_rings << 1;
6528         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6529         bp->tx_nr_rings = tx;
6530
6531         /* If we cannot reserve all the RX rings, reset the RSS map only
6532          * if absolutely necessary
6533          */
6534         if (rx_rings != bp->rx_nr_rings) {
6535                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6536                             rx_rings, bp->rx_nr_rings);
6537                 if (netif_is_rxfh_configured(bp->dev) &&
6538                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6539                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6540                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6541                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6542                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6543                 }
6544         }
6545         bp->rx_nr_rings = rx_rings;
6546         bp->cp_nr_rings = cp;
6547
6548         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6549                 return -ENOMEM;
6550
6551         if (!netif_is_rxfh_configured(bp->dev))
6552                 bnxt_set_dflt_rss_indir_tbl(bp);
6553
6554         return rc;
6555 }
6556
6557 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6558                                     int ring_grps, int cp_rings, int stats,
6559                                     int vnics)
6560 {
6561         struct hwrm_func_vf_cfg_input *req;
6562         u32 flags;
6563
6564         if (!BNXT_NEW_RM(bp))
6565                 return 0;
6566
6567         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6568                                            cp_rings, stats, vnics);
6569         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6570                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6571                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6572                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6573                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6574                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6575         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6576                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6577
6578         req->flags = cpu_to_le32(flags);
6579         return hwrm_req_send_silent(bp, req);
6580 }
6581
6582 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6583                                     int ring_grps, int cp_rings, int stats,
6584                                     int vnics)
6585 {
6586         struct hwrm_func_cfg_input *req;
6587         u32 flags;
6588
6589         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6590                                            cp_rings, stats, vnics);
6591         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6592         if (BNXT_NEW_RM(bp)) {
6593                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6594                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6595                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6596                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6597                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6598                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6599                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6600                 else
6601                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6602         }
6603
6604         req->flags = cpu_to_le32(flags);
6605         return hwrm_req_send_silent(bp, req);
6606 }
6607
6608 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6609                                  int ring_grps, int cp_rings, int stats,
6610                                  int vnics)
6611 {
6612         if (bp->hwrm_spec_code < 0x10801)
6613                 return 0;
6614
6615         if (BNXT_PF(bp))
6616                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6617                                                 ring_grps, cp_rings, stats,
6618                                                 vnics);
6619
6620         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6621                                         cp_rings, stats, vnics);
6622 }
6623
6624 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6625 {
6626         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6627         struct hwrm_ring_aggint_qcaps_output *resp;
6628         struct hwrm_ring_aggint_qcaps_input *req;
6629         int rc;
6630
6631         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6632         coal_cap->num_cmpl_dma_aggr_max = 63;
6633         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6634         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6635         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6636         coal_cap->int_lat_tmr_min_max = 65535;
6637         coal_cap->int_lat_tmr_max_max = 65535;
6638         coal_cap->num_cmpl_aggr_int_max = 65535;
6639         coal_cap->timer_units = 80;
6640
6641         if (bp->hwrm_spec_code < 0x10902)
6642                 return;
6643
6644         if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
6645                 return;
6646
6647         resp = hwrm_req_hold(bp, req);
6648         rc = hwrm_req_send_silent(bp, req);
6649         if (!rc) {
6650                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6651                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6652                 coal_cap->num_cmpl_dma_aggr_max =
6653                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6654                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6655                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6656                 coal_cap->cmpl_aggr_dma_tmr_max =
6657                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6658                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6659                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6660                 coal_cap->int_lat_tmr_min_max =
6661                         le16_to_cpu(resp->int_lat_tmr_min_max);
6662                 coal_cap->int_lat_tmr_max_max =
6663                         le16_to_cpu(resp->int_lat_tmr_max_max);
6664                 coal_cap->num_cmpl_aggr_int_max =
6665                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6666                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6667         }
6668         hwrm_req_drop(bp, req);
6669 }
6670
6671 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6672 {
6673         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6674
6675         return usec * 1000 / coal_cap->timer_units;
6676 }
6677
6678 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6679         struct bnxt_coal *hw_coal,
6680         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6681 {
6682         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6683         u16 val, tmr, max, flags = hw_coal->flags;
6684         u32 cmpl_params = coal_cap->cmpl_params;
6685
6686         max = hw_coal->bufs_per_record * 128;
6687         if (hw_coal->budget)
6688                 max = hw_coal->bufs_per_record * hw_coal->budget;
6689         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6690
6691         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6692         req->num_cmpl_aggr_int = cpu_to_le16(val);
6693
6694         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6695         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6696
6697         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6698                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6699         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6700
6701         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6702         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6703         req->int_lat_tmr_max = cpu_to_le16(tmr);
6704
6705         /* min timer set to 1/2 of interrupt timer */
6706         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6707                 val = tmr / 2;
6708                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6709                 req->int_lat_tmr_min = cpu_to_le16(val);
6710                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6711         }
6712
6713         /* buf timer set to 1/4 of interrupt timer */
6714         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6715         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6716
6717         if (cmpl_params &
6718             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6719                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6720                 val = clamp_t(u16, tmr, 1,
6721                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6722                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6723                 req->enables |=
6724                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6725         }
6726
6727         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6728             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6729                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6730         req->flags = cpu_to_le16(flags);
6731         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6732 }
6733
6734 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6735                                    struct bnxt_coal *hw_coal)
6736 {
6737         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
6738         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6739         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6740         u32 nq_params = coal_cap->nq_params;
6741         u16 tmr;
6742         int rc;
6743
6744         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6745                 return 0;
6746
6747         rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6748         if (rc)
6749                 return rc;
6750
6751         req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6752         req->flags =
6753                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6754
6755         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6756         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6757         req->int_lat_tmr_min = cpu_to_le16(tmr);
6758         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6759         return hwrm_req_send(bp, req);
6760 }
6761
6762 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6763 {
6764         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
6765         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6766         struct bnxt_coal coal;
6767         int rc;
6768
6769         /* Tick values in micro seconds.
6770          * 1 coal_buf x bufs_per_record = 1 completion record.
6771          */
6772         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6773
6774         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6775         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6776
6777         if (!bnapi->rx_ring)
6778                 return -ENODEV;
6779
6780         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6781         if (rc)
6782                 return rc;
6783
6784         bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
6785
6786         req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6787
6788         return hwrm_req_send(bp, req_rx);
6789 }
6790
6791 int bnxt_hwrm_set_coal(struct bnxt *bp)
6792 {
6793         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx,
6794                                                            *req;
6795         int i, rc;
6796
6797         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6798         if (rc)
6799                 return rc;
6800
6801         rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6802         if (rc) {
6803                 hwrm_req_drop(bp, req_rx);
6804                 return rc;
6805         }
6806
6807         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
6808         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
6809
6810         hwrm_req_hold(bp, req_rx);
6811         hwrm_req_hold(bp, req_tx);
6812         for (i = 0; i < bp->cp_nr_rings; i++) {
6813                 struct bnxt_napi *bnapi = bp->bnapi[i];
6814                 struct bnxt_coal *hw_coal;
6815                 u16 ring_id;
6816
6817                 req = req_rx;
6818                 if (!bnapi->rx_ring) {
6819                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6820                         req = req_tx;
6821                 } else {
6822                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6823                 }
6824                 req->ring_id = cpu_to_le16(ring_id);
6825
6826                 rc = hwrm_req_send(bp, req);
6827                 if (rc)
6828                         break;
6829
6830                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6831                         continue;
6832
6833                 if (bnapi->rx_ring && bnapi->tx_ring) {
6834                         req = req_tx;
6835                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6836                         req->ring_id = cpu_to_le16(ring_id);
6837                         rc = hwrm_req_send(bp, req);
6838                         if (rc)
6839                                 break;
6840                 }
6841                 if (bnapi->rx_ring)
6842                         hw_coal = &bp->rx_coal;
6843                 else
6844                         hw_coal = &bp->tx_coal;
6845                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6846         }
6847         hwrm_req_drop(bp, req_rx);
6848         hwrm_req_drop(bp, req_tx);
6849         return rc;
6850 }
6851
6852 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6853 {
6854         struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
6855         struct hwrm_stat_ctx_free_input *req;
6856         int i;
6857
6858         if (!bp->bnapi)
6859                 return;
6860
6861         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6862                 return;
6863
6864         if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
6865                 return;
6866         if (BNXT_FW_MAJ(bp) <= 20) {
6867                 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
6868                         hwrm_req_drop(bp, req);
6869                         return;
6870                 }
6871                 hwrm_req_hold(bp, req0);
6872         }
6873         hwrm_req_hold(bp, req);
6874         for (i = 0; i < bp->cp_nr_rings; i++) {
6875                 struct bnxt_napi *bnapi = bp->bnapi[i];
6876                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6877
6878                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6879                         req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6880                         if (req0) {
6881                                 req0->stat_ctx_id = req->stat_ctx_id;
6882                                 hwrm_req_send(bp, req0);
6883                         }
6884                         hwrm_req_send(bp, req);
6885
6886                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6887                 }
6888         }
6889         hwrm_req_drop(bp, req);
6890         if (req0)
6891                 hwrm_req_drop(bp, req0);
6892 }
6893
6894 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6895 {
6896         struct hwrm_stat_ctx_alloc_output *resp;
6897         struct hwrm_stat_ctx_alloc_input *req;
6898         int rc, i;
6899
6900         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6901                 return 0;
6902
6903         rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
6904         if (rc)
6905                 return rc;
6906
6907         req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6908         req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6909
6910         resp = hwrm_req_hold(bp, req);
6911         for (i = 0; i < bp->cp_nr_rings; i++) {
6912                 struct bnxt_napi *bnapi = bp->bnapi[i];
6913                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6914
6915                 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6916
6917                 rc = hwrm_req_send(bp, req);
6918                 if (rc)
6919                         break;
6920
6921                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6922
6923                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6924         }
6925         hwrm_req_drop(bp, req);
6926         return rc;
6927 }
6928
6929 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6930 {
6931         struct hwrm_func_qcfg_output *resp;
6932         struct hwrm_func_qcfg_input *req;
6933         u32 min_db_offset = 0;
6934         u16 flags;
6935         int rc;
6936
6937         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6938         if (rc)
6939                 return rc;
6940
6941         req->fid = cpu_to_le16(0xffff);
6942         resp = hwrm_req_hold(bp, req);
6943         rc = hwrm_req_send(bp, req);
6944         if (rc)
6945                 goto func_qcfg_exit;
6946
6947 #ifdef CONFIG_BNXT_SRIOV
6948         if (BNXT_VF(bp)) {
6949                 struct bnxt_vf_info *vf = &bp->vf;
6950
6951                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6952         } else {
6953                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6954         }
6955 #endif
6956         flags = le16_to_cpu(resp->flags);
6957         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6958                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6959                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6960                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6961                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6962         }
6963         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6964                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6965         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6966                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6967
6968         switch (resp->port_partition_type) {
6969         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6970         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6971         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6972                 bp->port_partition_type = resp->port_partition_type;
6973                 break;
6974         }
6975         if (bp->hwrm_spec_code < 0x10707 ||
6976             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6977                 bp->br_mode = BRIDGE_MODE_VEB;
6978         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6979                 bp->br_mode = BRIDGE_MODE_VEPA;
6980         else
6981                 bp->br_mode = BRIDGE_MODE_UNDEF;
6982
6983         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6984         if (!bp->max_mtu)
6985                 bp->max_mtu = BNXT_MAX_MTU;
6986
6987         if (bp->db_size)
6988                 goto func_qcfg_exit;
6989
6990         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6991                 if (BNXT_PF(bp))
6992                         min_db_offset = DB_PF_OFFSET_P5;
6993                 else
6994                         min_db_offset = DB_VF_OFFSET_P5;
6995         }
6996         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6997                                  1024);
6998         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6999             bp->db_size <= min_db_offset)
7000                 bp->db_size = pci_resource_len(bp->pdev, 2);
7001
7002 func_qcfg_exit:
7003         hwrm_req_drop(bp, req);
7004         return rc;
7005 }
7006
7007 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
7008                         struct hwrm_func_backing_store_qcaps_output *resp)
7009 {
7010         struct bnxt_mem_init *mem_init;
7011         u16 init_mask;
7012         u8 init_val;
7013         u8 *offset;
7014         int i;
7015
7016         init_val = resp->ctx_kind_initializer;
7017         init_mask = le16_to_cpu(resp->ctx_init_mask);
7018         offset = &resp->qp_init_offset;
7019         mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7020         for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
7021                 mem_init->init_val = init_val;
7022                 mem_init->offset = BNXT_MEM_INVALID_OFFSET;
7023                 if (!init_mask)
7024                         continue;
7025                 if (i == BNXT_CTX_MEM_INIT_STAT)
7026                         offset = &resp->stat_init_offset;
7027                 if (init_mask & (1 << i))
7028                         mem_init->offset = *offset * 4;
7029                 else
7030                         mem_init->init_val = 0;
7031         }
7032         ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
7033         ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
7034         ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
7035         ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
7036         ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
7037         ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
7038 }
7039
7040 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7041 {
7042         struct hwrm_func_backing_store_qcaps_output *resp;
7043         struct hwrm_func_backing_store_qcaps_input *req;
7044         int rc;
7045
7046         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7047                 return 0;
7048
7049         rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7050         if (rc)
7051                 return rc;
7052
7053         resp = hwrm_req_hold(bp, req);
7054         rc = hwrm_req_send_silent(bp, req);
7055         if (!rc) {
7056                 struct bnxt_ctx_pg_info *ctx_pg;
7057                 struct bnxt_ctx_mem_info *ctx;
7058                 int i, tqm_rings;
7059
7060                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7061                 if (!ctx) {
7062                         rc = -ENOMEM;
7063                         goto ctx_err;
7064                 }
7065                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
7066                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7067                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7068                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
7069                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7070                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
7071                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
7072                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7073                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
7074                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
7075                 ctx->vnic_max_vnic_entries =
7076                         le16_to_cpu(resp->vnic_max_vnic_entries);
7077                 ctx->vnic_max_ring_table_entries =
7078                         le16_to_cpu(resp->vnic_max_ring_table_entries);
7079                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
7080                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
7081                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
7082                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
7083                 ctx->tqm_min_entries_per_ring =
7084                         le32_to_cpu(resp->tqm_min_entries_per_ring);
7085                 ctx->tqm_max_entries_per_ring =
7086                         le32_to_cpu(resp->tqm_max_entries_per_ring);
7087                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
7088                 if (!ctx->tqm_entries_multiple)
7089                         ctx->tqm_entries_multiple = 1;
7090                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
7091                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
7092                 ctx->mrav_num_entries_units =
7093                         le16_to_cpu(resp->mrav_num_entries_units);
7094                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
7095                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
7096
7097                 bnxt_init_ctx_initializer(ctx, resp);
7098
7099                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7100                 if (!ctx->tqm_fp_rings_count)
7101                         ctx->tqm_fp_rings_count = bp->max_q;
7102                 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7103                         ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7104
7105                 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
7106                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
7107                 if (!ctx_pg) {
7108                         kfree(ctx);
7109                         rc = -ENOMEM;
7110                         goto ctx_err;
7111                 }
7112                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
7113                         ctx->tqm_mem[i] = ctx_pg;
7114                 bp->ctx = ctx;
7115         } else {
7116                 rc = 0;
7117         }
7118 ctx_err:
7119         hwrm_req_drop(bp, req);
7120         return rc;
7121 }
7122
7123 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
7124                                   __le64 *pg_dir)
7125 {
7126         if (!rmem->nr_pages)
7127                 return;
7128
7129         BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
7130         if (rmem->depth >= 1) {
7131                 if (rmem->depth == 2)
7132                         *pg_attr |= 2;
7133                 else
7134                         *pg_attr |= 1;
7135                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
7136         } else {
7137                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
7138         }
7139 }
7140
7141 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
7142         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
7143          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
7144          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
7145          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
7146          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
7147
7148 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
7149 {
7150         struct hwrm_func_backing_store_cfg_input *req;
7151         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7152         struct bnxt_ctx_pg_info *ctx_pg;
7153         void **__req = (void **)&req;
7154         u32 req_len = sizeof(*req);
7155         __le32 *num_entries;
7156         __le64 *pg_dir;
7157         u32 flags = 0;
7158         u8 *pg_attr;
7159         u32 ena;
7160         int rc;
7161         int i;
7162
7163         if (!ctx)
7164                 return 0;
7165
7166         if (req_len > bp->hwrm_max_ext_req_len)
7167                 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
7168         rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
7169         if (rc)
7170                 return rc;
7171
7172         req->enables = cpu_to_le32(enables);
7173         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
7174                 ctx_pg = &ctx->qp_mem;
7175                 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
7176                 req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
7177                 req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
7178                 req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
7179                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7180                                       &req->qpc_pg_size_qpc_lvl,
7181                                       &req->qpc_page_dir);
7182         }
7183         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
7184                 ctx_pg = &ctx->srq_mem;
7185                 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
7186                 req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
7187                 req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
7188                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7189                                       &req->srq_pg_size_srq_lvl,
7190                                       &req->srq_page_dir);
7191         }
7192         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
7193                 ctx_pg = &ctx->cq_mem;
7194                 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
7195                 req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
7196                 req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
7197                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7198                                       &req->cq_pg_size_cq_lvl,
7199                                       &req->cq_page_dir);
7200         }
7201         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
7202                 ctx_pg = &ctx->vnic_mem;
7203                 req->vnic_num_vnic_entries =
7204                         cpu_to_le16(ctx->vnic_max_vnic_entries);
7205                 req->vnic_num_ring_table_entries =
7206                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
7207                 req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
7208                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7209                                       &req->vnic_pg_size_vnic_lvl,
7210                                       &req->vnic_page_dir);
7211         }
7212         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
7213                 ctx_pg = &ctx->stat_mem;
7214                 req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
7215                 req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
7216                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7217                                       &req->stat_pg_size_stat_lvl,
7218                                       &req->stat_page_dir);
7219         }
7220         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7221                 ctx_pg = &ctx->mrav_mem;
7222                 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7223                 if (ctx->mrav_num_entries_units)
7224                         flags |=
7225                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7226                 req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7227                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7228                                       &req->mrav_pg_size_mrav_lvl,
7229                                       &req->mrav_page_dir);
7230         }
7231         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7232                 ctx_pg = &ctx->tim_mem;
7233                 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
7234                 req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7235                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7236                                       &req->tim_pg_size_tim_lvl,
7237                                       &req->tim_page_dir);
7238         }
7239         for (i = 0, num_entries = &req->tqm_sp_num_entries,
7240              pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
7241              pg_dir = &req->tqm_sp_page_dir,
7242              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7243              i < BNXT_MAX_TQM_RINGS;
7244              i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7245                 if (!(enables & ena))
7246                         continue;
7247
7248                 req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7249                 ctx_pg = ctx->tqm_mem[i];
7250                 *num_entries = cpu_to_le32(ctx_pg->entries);
7251                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7252         }
7253         req->flags = cpu_to_le32(flags);
7254         return hwrm_req_send(bp, req);
7255 }
7256
7257 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7258                                   struct bnxt_ctx_pg_info *ctx_pg)
7259 {
7260         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7261
7262         rmem->page_size = BNXT_PAGE_SIZE;
7263         rmem->pg_arr = ctx_pg->ctx_pg_arr;
7264         rmem->dma_arr = ctx_pg->ctx_dma_arr;
7265         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7266         if (rmem->depth >= 1)
7267                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7268         return bnxt_alloc_ring(bp, rmem);
7269 }
7270
7271 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7272                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7273                                   u8 depth, struct bnxt_mem_init *mem_init)
7274 {
7275         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7276         int rc;
7277
7278         if (!mem_size)
7279                 return -EINVAL;
7280
7281         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7282         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7283                 ctx_pg->nr_pages = 0;
7284                 return -EINVAL;
7285         }
7286         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7287                 int nr_tbls, i;
7288
7289                 rmem->depth = 2;
7290                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7291                                              GFP_KERNEL);
7292                 if (!ctx_pg->ctx_pg_tbl)
7293                         return -ENOMEM;
7294                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7295                 rmem->nr_pages = nr_tbls;
7296                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7297                 if (rc)
7298                         return rc;
7299                 for (i = 0; i < nr_tbls; i++) {
7300                         struct bnxt_ctx_pg_info *pg_tbl;
7301
7302                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7303                         if (!pg_tbl)
7304                                 return -ENOMEM;
7305                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7306                         rmem = &pg_tbl->ring_mem;
7307                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7308                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7309                         rmem->depth = 1;
7310                         rmem->nr_pages = MAX_CTX_PAGES;
7311                         rmem->mem_init = mem_init;
7312                         if (i == (nr_tbls - 1)) {
7313                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7314
7315                                 if (rem)
7316                                         rmem->nr_pages = rem;
7317                         }
7318                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7319                         if (rc)
7320                                 break;
7321                 }
7322         } else {
7323                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7324                 if (rmem->nr_pages > 1 || depth)
7325                         rmem->depth = 1;
7326                 rmem->mem_init = mem_init;
7327                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7328         }
7329         return rc;
7330 }
7331
7332 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7333                                   struct bnxt_ctx_pg_info *ctx_pg)
7334 {
7335         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7336
7337         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7338             ctx_pg->ctx_pg_tbl) {
7339                 int i, nr_tbls = rmem->nr_pages;
7340
7341                 for (i = 0; i < nr_tbls; i++) {
7342                         struct bnxt_ctx_pg_info *pg_tbl;
7343                         struct bnxt_ring_mem_info *rmem2;
7344
7345                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7346                         if (!pg_tbl)
7347                                 continue;
7348                         rmem2 = &pg_tbl->ring_mem;
7349                         bnxt_free_ring(bp, rmem2);
7350                         ctx_pg->ctx_pg_arr[i] = NULL;
7351                         kfree(pg_tbl);
7352                         ctx_pg->ctx_pg_tbl[i] = NULL;
7353                 }
7354                 kfree(ctx_pg->ctx_pg_tbl);
7355                 ctx_pg->ctx_pg_tbl = NULL;
7356         }
7357         bnxt_free_ring(bp, rmem);
7358         ctx_pg->nr_pages = 0;
7359 }
7360
7361 void bnxt_free_ctx_mem(struct bnxt *bp)
7362 {
7363         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7364         int i;
7365
7366         if (!ctx)
7367                 return;
7368
7369         if (ctx->tqm_mem[0]) {
7370                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7371                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7372                 kfree(ctx->tqm_mem[0]);
7373                 ctx->tqm_mem[0] = NULL;
7374         }
7375
7376         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7377         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7378         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7379         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7380         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7381         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7382         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7383         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7384 }
7385
7386 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7387 {
7388         struct bnxt_ctx_pg_info *ctx_pg;
7389         struct bnxt_ctx_mem_info *ctx;
7390         struct bnxt_mem_init *init;
7391         u32 mem_size, ena, entries;
7392         u32 entries_sp, min;
7393         u32 num_mr, num_ah;
7394         u32 extra_srqs = 0;
7395         u32 extra_qps = 0;
7396         u8 pg_lvl = 1;
7397         int i, rc;
7398
7399         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7400         if (rc) {
7401                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7402                            rc);
7403                 return rc;
7404         }
7405         ctx = bp->ctx;
7406         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7407                 return 0;
7408
7409         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7410                 pg_lvl = 2;
7411                 extra_qps = 65536;
7412                 extra_srqs = 8192;
7413         }
7414
7415         ctx_pg = &ctx->qp_mem;
7416         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7417                           extra_qps;
7418         if (ctx->qp_entry_size) {
7419                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
7420                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7421                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7422                 if (rc)
7423                         return rc;
7424         }
7425
7426         ctx_pg = &ctx->srq_mem;
7427         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7428         if (ctx->srq_entry_size) {
7429                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
7430                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7431                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7432                 if (rc)
7433                         return rc;
7434         }
7435
7436         ctx_pg = &ctx->cq_mem;
7437         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7438         if (ctx->cq_entry_size) {
7439                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
7440                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7441                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7442                 if (rc)
7443                         return rc;
7444         }
7445
7446         ctx_pg = &ctx->vnic_mem;
7447         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7448                           ctx->vnic_max_ring_table_entries;
7449         if (ctx->vnic_entry_size) {
7450                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7451                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7452                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7453                 if (rc)
7454                         return rc;
7455         }
7456
7457         ctx_pg = &ctx->stat_mem;
7458         ctx_pg->entries = ctx->stat_max_entries;
7459         if (ctx->stat_entry_size) {
7460                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
7461                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7462                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7463                 if (rc)
7464                         return rc;
7465         }
7466
7467         ena = 0;
7468         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7469                 goto skip_rdma;
7470
7471         ctx_pg = &ctx->mrav_mem;
7472         /* 128K extra is needed to accommodate static AH context
7473          * allocation by f/w.
7474          */
7475         num_mr = 1024 * 256;
7476         num_ah = 1024 * 128;
7477         ctx_pg->entries = num_mr + num_ah;
7478         if (ctx->mrav_entry_size) {
7479                 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7480                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7481                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7482                 if (rc)
7483                         return rc;
7484         }
7485         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7486         if (ctx->mrav_num_entries_units)
7487                 ctx_pg->entries =
7488                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7489                          (num_ah / ctx->mrav_num_entries_units);
7490
7491         ctx_pg = &ctx->tim_mem;
7492         ctx_pg->entries = ctx->qp_mem.entries;
7493         if (ctx->tim_entry_size) {
7494                 mem_size = ctx->tim_entry_size * ctx_pg->entries;
7495                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7496                 if (rc)
7497                         return rc;
7498         }
7499         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7500
7501 skip_rdma:
7502         min = ctx->tqm_min_entries_per_ring;
7503         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7504                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7505         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7506         entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries);
7507         entries = roundup(entries, ctx->tqm_entries_multiple);
7508         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7509         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7510                 ctx_pg = ctx->tqm_mem[i];
7511                 ctx_pg->entries = i ? entries : entries_sp;
7512                 if (ctx->tqm_entry_size) {
7513                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7514                         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7515                                                     NULL);
7516                         if (rc)
7517                                 return rc;
7518                 }
7519                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7520         }
7521         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7522         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7523         if (rc) {
7524                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7525                            rc);
7526                 return rc;
7527         }
7528         ctx->flags |= BNXT_CTX_FLAG_INITED;
7529         return 0;
7530 }
7531
7532 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7533 {
7534         struct hwrm_func_resource_qcaps_output *resp;
7535         struct hwrm_func_resource_qcaps_input *req;
7536         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7537         int rc;
7538
7539         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
7540         if (rc)
7541                 return rc;
7542
7543         req->fid = cpu_to_le16(0xffff);
7544         resp = hwrm_req_hold(bp, req);
7545         rc = hwrm_req_send_silent(bp, req);
7546         if (rc)
7547                 goto hwrm_func_resc_qcaps_exit;
7548
7549         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7550         if (!all)
7551                 goto hwrm_func_resc_qcaps_exit;
7552
7553         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7554         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7555         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7556         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7557         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7558         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7559         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7560         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7561         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7562         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7563         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7564         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7565         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7566         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7567         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7568         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7569
7570         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7571                 u16 max_msix = le16_to_cpu(resp->max_msix);
7572
7573                 hw_resc->max_nqs = max_msix;
7574                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7575         }
7576
7577         if (BNXT_PF(bp)) {
7578                 struct bnxt_pf_info *pf = &bp->pf;
7579
7580                 pf->vf_resv_strategy =
7581                         le16_to_cpu(resp->vf_reservation_strategy);
7582                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7583                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7584         }
7585 hwrm_func_resc_qcaps_exit:
7586         hwrm_req_drop(bp, req);
7587         return rc;
7588 }
7589
7590 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
7591 {
7592         struct hwrm_port_mac_ptp_qcfg_output *resp;
7593         struct hwrm_port_mac_ptp_qcfg_input *req;
7594         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
7595         bool phc_cfg;
7596         u8 flags;
7597         int rc;
7598
7599         if (bp->hwrm_spec_code < 0x10801) {
7600                 rc = -ENODEV;
7601                 goto no_ptp;
7602         }
7603
7604         rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
7605         if (rc)
7606                 goto no_ptp;
7607
7608         req->port_id = cpu_to_le16(bp->pf.port_id);
7609         resp = hwrm_req_hold(bp, req);
7610         rc = hwrm_req_send(bp, req);
7611         if (rc)
7612                 goto exit;
7613
7614         flags = resp->flags;
7615         if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
7616                 rc = -ENODEV;
7617                 goto exit;
7618         }
7619         if (!ptp) {
7620                 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
7621                 if (!ptp) {
7622                         rc = -ENOMEM;
7623                         goto exit;
7624                 }
7625                 ptp->bp = bp;
7626                 bp->ptp_cfg = ptp;
7627         }
7628         if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
7629                 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
7630                 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
7631         } else if (bp->flags & BNXT_FLAG_CHIP_P5) {
7632                 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
7633                 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
7634         } else {
7635                 rc = -ENODEV;
7636                 goto exit;
7637         }
7638         phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
7639         rc = bnxt_ptp_init(bp, phc_cfg);
7640         if (rc)
7641                 netdev_warn(bp->dev, "PTP initialization failed.\n");
7642 exit:
7643         hwrm_req_drop(bp, req);
7644         if (!rc)
7645                 return 0;
7646
7647 no_ptp:
7648         bnxt_ptp_clear(bp);
7649         kfree(ptp);
7650         bp->ptp_cfg = NULL;
7651         return rc;
7652 }
7653
7654 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7655 {
7656         struct hwrm_func_qcaps_output *resp;
7657         struct hwrm_func_qcaps_input *req;
7658         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7659         u32 flags, flags_ext, flags_ext2;
7660         int rc;
7661
7662         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
7663         if (rc)
7664                 return rc;
7665
7666         req->fid = cpu_to_le16(0xffff);
7667         resp = hwrm_req_hold(bp, req);
7668         rc = hwrm_req_send(bp, req);
7669         if (rc)
7670                 goto hwrm_func_qcaps_exit;
7671
7672         flags = le32_to_cpu(resp->flags);
7673         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7674                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7675         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7676                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7677         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7678                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7679         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7680                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7681         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7682                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7683         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7684                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7685         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7686                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7687         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7688                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7689         if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
7690                 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
7691
7692         flags_ext = le32_to_cpu(resp->flags_ext);
7693         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7694                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7695         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
7696                 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
7697         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
7698                 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
7699         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
7700                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
7701         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
7702                 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
7703
7704         flags_ext2 = le32_to_cpu(resp->flags_ext2);
7705         if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
7706                 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
7707
7708         bp->tx_push_thresh = 0;
7709         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7710             BNXT_FW_MAJ(bp) > 217)
7711                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7712
7713         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7714         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7715         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7716         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7717         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7718         if (!hw_resc->max_hw_ring_grps)
7719                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7720         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7721         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7722         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7723
7724         if (BNXT_PF(bp)) {
7725                 struct bnxt_pf_info *pf = &bp->pf;
7726
7727                 pf->fw_fid = le16_to_cpu(resp->fid);
7728                 pf->port_id = le16_to_cpu(resp->port_id);
7729                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7730                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7731                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7732                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7733                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7734                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7735                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7736                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7737                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7738                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7739                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7740                         bp->flags |= BNXT_FLAG_WOL_CAP;
7741                 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
7742                         __bnxt_hwrm_ptp_qcfg(bp);
7743                 } else {
7744                         bnxt_ptp_clear(bp);
7745                         kfree(bp->ptp_cfg);
7746                         bp->ptp_cfg = NULL;
7747                 }
7748         } else {
7749 #ifdef CONFIG_BNXT_SRIOV
7750                 struct bnxt_vf_info *vf = &bp->vf;
7751
7752                 vf->fw_fid = le16_to_cpu(resp->fid);
7753                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7754 #endif
7755         }
7756
7757 hwrm_func_qcaps_exit:
7758         hwrm_req_drop(bp, req);
7759         return rc;
7760 }
7761
7762 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
7763 {
7764         struct hwrm_dbg_qcaps_output *resp;
7765         struct hwrm_dbg_qcaps_input *req;
7766         int rc;
7767
7768         bp->fw_dbg_cap = 0;
7769         if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
7770                 return;
7771
7772         rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
7773         if (rc)
7774                 return;
7775
7776         req->fid = cpu_to_le16(0xffff);
7777         resp = hwrm_req_hold(bp, req);
7778         rc = hwrm_req_send(bp, req);
7779         if (rc)
7780                 goto hwrm_dbg_qcaps_exit;
7781
7782         bp->fw_dbg_cap = le32_to_cpu(resp->flags);
7783
7784 hwrm_dbg_qcaps_exit:
7785         hwrm_req_drop(bp, req);
7786 }
7787
7788 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7789
7790 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7791 {
7792         int rc;
7793
7794         rc = __bnxt_hwrm_func_qcaps(bp);
7795         if (rc)
7796                 return rc;
7797
7798         bnxt_hwrm_dbg_qcaps(bp);
7799
7800         rc = bnxt_hwrm_queue_qportcfg(bp);
7801         if (rc) {
7802                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7803                 return rc;
7804         }
7805         if (bp->hwrm_spec_code >= 0x10803) {
7806                 rc = bnxt_alloc_ctx_mem(bp);
7807                 if (rc)
7808                         return rc;
7809                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7810                 if (!rc)
7811                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7812         }
7813         return 0;
7814 }
7815
7816 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7817 {
7818         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7819         struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
7820         u32 flags;
7821         int rc;
7822
7823         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7824                 return 0;
7825
7826         rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
7827         if (rc)
7828                 return rc;
7829
7830         resp = hwrm_req_hold(bp, req);
7831         rc = hwrm_req_send(bp, req);
7832         if (rc)
7833                 goto hwrm_cfa_adv_qcaps_exit;
7834
7835         flags = le32_to_cpu(resp->flags);
7836         if (flags &
7837             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7838                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7839
7840 hwrm_cfa_adv_qcaps_exit:
7841         hwrm_req_drop(bp, req);
7842         return rc;
7843 }
7844
7845 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7846 {
7847         if (bp->fw_health)
7848                 return 0;
7849
7850         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7851         if (!bp->fw_health)
7852                 return -ENOMEM;
7853
7854         mutex_init(&bp->fw_health->lock);
7855         return 0;
7856 }
7857
7858 static int bnxt_alloc_fw_health(struct bnxt *bp)
7859 {
7860         int rc;
7861
7862         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7863             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7864                 return 0;
7865
7866         rc = __bnxt_alloc_fw_health(bp);
7867         if (rc) {
7868                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7869                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7870                 return rc;
7871         }
7872
7873         return 0;
7874 }
7875
7876 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7877 {
7878         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7879                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7880                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7881 }
7882
7883 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
7884 {
7885         struct bnxt_fw_health *fw_health = bp->fw_health;
7886         u32 reg_type;
7887
7888         if (!fw_health)
7889                 return;
7890
7891         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
7892         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7893                 fw_health->status_reliable = false;
7894
7895         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
7896         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7897                 fw_health->resets_reliable = false;
7898 }
7899
7900 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7901 {
7902         void __iomem *hs;
7903         u32 status_loc;
7904         u32 reg_type;
7905         u32 sig;
7906
7907         if (bp->fw_health)
7908                 bp->fw_health->status_reliable = false;
7909
7910         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7911         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7912
7913         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7914         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7915                 if (!bp->chip_num) {
7916                         __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7917                         bp->chip_num = readl(bp->bar0 +
7918                                              BNXT_FW_HEALTH_WIN_BASE +
7919                                              BNXT_GRC_REG_CHIP_NUM);
7920                 }
7921                 if (!BNXT_CHIP_P5(bp))
7922                         return;
7923
7924                 status_loc = BNXT_GRC_REG_STATUS_P5 |
7925                              BNXT_FW_HEALTH_REG_TYPE_BAR0;
7926         } else {
7927                 status_loc = readl(hs + offsetof(struct hcomm_status,
7928                                                  fw_status_loc));
7929         }
7930
7931         if (__bnxt_alloc_fw_health(bp)) {
7932                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7933                 return;
7934         }
7935
7936         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7937         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7938         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7939                 __bnxt_map_fw_health_reg(bp, status_loc);
7940                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7941                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7942         }
7943
7944         bp->fw_health->status_reliable = true;
7945 }
7946
7947 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7948 {
7949         struct bnxt_fw_health *fw_health = bp->fw_health;
7950         u32 reg_base = 0xffffffff;
7951         int i;
7952
7953         bp->fw_health->status_reliable = false;
7954         bp->fw_health->resets_reliable = false;
7955         /* Only pre-map the monitoring GRC registers using window 3 */
7956         for (i = 0; i < 4; i++) {
7957                 u32 reg = fw_health->regs[i];
7958
7959                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7960                         continue;
7961                 if (reg_base == 0xffffffff)
7962                         reg_base = reg & BNXT_GRC_BASE_MASK;
7963                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7964                         return -ERANGE;
7965                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7966         }
7967         bp->fw_health->status_reliable = true;
7968         bp->fw_health->resets_reliable = true;
7969         if (reg_base == 0xffffffff)
7970                 return 0;
7971
7972         __bnxt_map_fw_health_reg(bp, reg_base);
7973         return 0;
7974 }
7975
7976 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
7977 {
7978         if (!bp->fw_health)
7979                 return;
7980
7981         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
7982                 bp->fw_health->status_reliable = true;
7983                 bp->fw_health->resets_reliable = true;
7984         } else {
7985                 bnxt_try_map_fw_health_reg(bp);
7986         }
7987 }
7988
7989 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7990 {
7991         struct bnxt_fw_health *fw_health = bp->fw_health;
7992         struct hwrm_error_recovery_qcfg_output *resp;
7993         struct hwrm_error_recovery_qcfg_input *req;
7994         int rc, i;
7995
7996         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7997                 return 0;
7998
7999         rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8000         if (rc)
8001                 return rc;
8002
8003         resp = hwrm_req_hold(bp, req);
8004         rc = hwrm_req_send(bp, req);
8005         if (rc)
8006                 goto err_recovery_out;
8007         fw_health->flags = le32_to_cpu(resp->flags);
8008         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8009             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8010                 rc = -EINVAL;
8011                 goto err_recovery_out;
8012         }
8013         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
8014         fw_health->master_func_wait_dsecs =
8015                 le32_to_cpu(resp->master_func_wait_period);
8016         fw_health->normal_func_wait_dsecs =
8017                 le32_to_cpu(resp->normal_func_wait_period);
8018         fw_health->post_reset_wait_dsecs =
8019                 le32_to_cpu(resp->master_func_wait_period_after_reset);
8020         fw_health->post_reset_max_wait_dsecs =
8021                 le32_to_cpu(resp->max_bailout_time_after_reset);
8022         fw_health->regs[BNXT_FW_HEALTH_REG] =
8023                 le32_to_cpu(resp->fw_health_status_reg);
8024         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
8025                 le32_to_cpu(resp->fw_heartbeat_reg);
8026         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
8027                 le32_to_cpu(resp->fw_reset_cnt_reg);
8028         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
8029                 le32_to_cpu(resp->reset_inprogress_reg);
8030         fw_health->fw_reset_inprog_reg_mask =
8031                 le32_to_cpu(resp->reset_inprogress_reg_mask);
8032         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
8033         if (fw_health->fw_reset_seq_cnt >= 16) {
8034                 rc = -EINVAL;
8035                 goto err_recovery_out;
8036         }
8037         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
8038                 fw_health->fw_reset_seq_regs[i] =
8039                         le32_to_cpu(resp->reset_reg[i]);
8040                 fw_health->fw_reset_seq_vals[i] =
8041                         le32_to_cpu(resp->reset_reg_val[i]);
8042                 fw_health->fw_reset_seq_delay_msec[i] =
8043                         resp->delay_after_reset[i];
8044         }
8045 err_recovery_out:
8046         hwrm_req_drop(bp, req);
8047         if (!rc)
8048                 rc = bnxt_map_fw_health_regs(bp);
8049         if (rc)
8050                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8051         return rc;
8052 }
8053
8054 static int bnxt_hwrm_func_reset(struct bnxt *bp)
8055 {
8056         struct hwrm_func_reset_input *req;
8057         int rc;
8058
8059         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
8060         if (rc)
8061                 return rc;
8062
8063         req->enables = 0;
8064         hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
8065         return hwrm_req_send(bp, req);
8066 }
8067
8068 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
8069 {
8070         struct hwrm_nvm_get_dev_info_output nvm_info;
8071
8072         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
8073                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
8074                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
8075                          nvm_info.nvm_cfg_ver_upd);
8076 }
8077
8078 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
8079 {
8080         struct hwrm_queue_qportcfg_output *resp;
8081         struct hwrm_queue_qportcfg_input *req;
8082         u8 i, j, *qptr;
8083         bool no_rdma;
8084         int rc = 0;
8085
8086         rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
8087         if (rc)
8088                 return rc;
8089
8090         resp = hwrm_req_hold(bp, req);
8091         rc = hwrm_req_send(bp, req);
8092         if (rc)
8093                 goto qportcfg_exit;
8094
8095         if (!resp->max_configurable_queues) {
8096                 rc = -EINVAL;
8097                 goto qportcfg_exit;
8098         }
8099         bp->max_tc = resp->max_configurable_queues;
8100         bp->max_lltc = resp->max_configurable_lossless_queues;
8101         if (bp->max_tc > BNXT_MAX_QUEUE)
8102                 bp->max_tc = BNXT_MAX_QUEUE;
8103
8104         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
8105         qptr = &resp->queue_id0;
8106         for (i = 0, j = 0; i < bp->max_tc; i++) {
8107                 bp->q_info[j].queue_id = *qptr;
8108                 bp->q_ids[i] = *qptr++;
8109                 bp->q_info[j].queue_profile = *qptr++;
8110                 bp->tc_to_qidx[j] = j;
8111                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
8112                     (no_rdma && BNXT_PF(bp)))
8113                         j++;
8114         }
8115         bp->max_q = bp->max_tc;
8116         bp->max_tc = max_t(u8, j, 1);
8117
8118         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
8119                 bp->max_tc = 1;
8120
8121         if (bp->max_lltc > bp->max_tc)
8122                 bp->max_lltc = bp->max_tc;
8123
8124 qportcfg_exit:
8125         hwrm_req_drop(bp, req);
8126         return rc;
8127 }
8128
8129 static int bnxt_hwrm_poll(struct bnxt *bp)
8130 {
8131         struct hwrm_ver_get_input *req;
8132         int rc;
8133
8134         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8135         if (rc)
8136                 return rc;
8137
8138         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8139         req->hwrm_intf_min = HWRM_VERSION_MINOR;
8140         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8141
8142         hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
8143         rc = hwrm_req_send(bp, req);
8144         return rc;
8145 }
8146
8147 static int bnxt_hwrm_ver_get(struct bnxt *bp)
8148 {
8149         struct hwrm_ver_get_output *resp;
8150         struct hwrm_ver_get_input *req;
8151         u16 fw_maj, fw_min, fw_bld, fw_rsv;
8152         u32 dev_caps_cfg, hwrm_ver;
8153         int rc, len;
8154
8155         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8156         if (rc)
8157                 return rc;
8158
8159         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
8160         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
8161         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8162         req->hwrm_intf_min = HWRM_VERSION_MINOR;
8163         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8164
8165         resp = hwrm_req_hold(bp, req);
8166         rc = hwrm_req_send(bp, req);
8167         if (rc)
8168                 goto hwrm_ver_get_exit;
8169
8170         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
8171
8172         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
8173                              resp->hwrm_intf_min_8b << 8 |
8174                              resp->hwrm_intf_upd_8b;
8175         if (resp->hwrm_intf_maj_8b < 1) {
8176                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
8177                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8178                             resp->hwrm_intf_upd_8b);
8179                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
8180         }
8181
8182         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
8183                         HWRM_VERSION_UPDATE;
8184
8185         if (bp->hwrm_spec_code > hwrm_ver)
8186                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8187                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
8188                          HWRM_VERSION_UPDATE);
8189         else
8190                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8191                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8192                          resp->hwrm_intf_upd_8b);
8193
8194         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
8195         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
8196                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
8197                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
8198                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
8199                 len = FW_VER_STR_LEN;
8200         } else {
8201                 fw_maj = resp->hwrm_fw_maj_8b;
8202                 fw_min = resp->hwrm_fw_min_8b;
8203                 fw_bld = resp->hwrm_fw_bld_8b;
8204                 fw_rsv = resp->hwrm_fw_rsvd_8b;
8205                 len = BC_HWRM_STR_LEN;
8206         }
8207         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
8208         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
8209                  fw_rsv);
8210
8211         if (strlen(resp->active_pkg_name)) {
8212                 int fw_ver_len = strlen(bp->fw_ver_str);
8213
8214                 snprintf(bp->fw_ver_str + fw_ver_len,
8215                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
8216                          resp->active_pkg_name);
8217                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
8218         }
8219
8220         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
8221         if (!bp->hwrm_cmd_timeout)
8222                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
8223         bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
8224         if (!bp->hwrm_cmd_max_timeout)
8225                 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
8226         else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
8227                 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
8228                             bp->hwrm_cmd_max_timeout / 1000);
8229
8230         if (resp->hwrm_intf_maj_8b >= 1) {
8231                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
8232                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
8233         }
8234         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
8235                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
8236
8237         bp->chip_num = le16_to_cpu(resp->chip_num);
8238         bp->chip_rev = resp->chip_rev;
8239         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
8240             !resp->chip_metal)
8241                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
8242
8243         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
8244         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
8245             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
8246                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
8247
8248         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
8249                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
8250
8251         if (dev_caps_cfg &
8252             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
8253                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
8254
8255         if (dev_caps_cfg &
8256             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
8257                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
8258
8259         if (dev_caps_cfg &
8260             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
8261                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
8262
8263 hwrm_ver_get_exit:
8264         hwrm_req_drop(bp, req);
8265         return rc;
8266 }
8267
8268 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
8269 {
8270         struct hwrm_fw_set_time_input *req;
8271         struct tm tm;
8272         time64_t now = ktime_get_real_seconds();
8273         int rc;
8274
8275         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
8276             bp->hwrm_spec_code < 0x10400)
8277                 return -EOPNOTSUPP;
8278
8279         time64_to_tm(now, 0, &tm);
8280         rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
8281         if (rc)
8282                 return rc;
8283
8284         req->year = cpu_to_le16(1900 + tm.tm_year);
8285         req->month = 1 + tm.tm_mon;
8286         req->day = tm.tm_mday;
8287         req->hour = tm.tm_hour;
8288         req->minute = tm.tm_min;
8289         req->second = tm.tm_sec;
8290         return hwrm_req_send(bp, req);
8291 }
8292
8293 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
8294 {
8295         u64 sw_tmp;
8296
8297         hw &= mask;
8298         sw_tmp = (*sw & ~mask) | hw;
8299         if (hw < (*sw & mask))
8300                 sw_tmp += mask + 1;
8301         WRITE_ONCE(*sw, sw_tmp);
8302 }
8303
8304 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
8305                                     int count, bool ignore_zero)
8306 {
8307         int i;
8308
8309         for (i = 0; i < count; i++) {
8310                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
8311
8312                 if (ignore_zero && !hw)
8313                         continue;
8314
8315                 if (masks[i] == -1ULL)
8316                         sw_stats[i] = hw;
8317                 else
8318                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
8319         }
8320 }
8321
8322 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
8323 {
8324         if (!stats->hw_stats)
8325                 return;
8326
8327         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8328                                 stats->hw_masks, stats->len / 8, false);
8329 }
8330
8331 static void bnxt_accumulate_all_stats(struct bnxt *bp)
8332 {
8333         struct bnxt_stats_mem *ring0_stats;
8334         bool ignore_zero = false;
8335         int i;
8336
8337         /* Chip bug.  Counter intermittently becomes 0. */
8338         if (bp->flags & BNXT_FLAG_CHIP_P5)
8339                 ignore_zero = true;
8340
8341         for (i = 0; i < bp->cp_nr_rings; i++) {
8342                 struct bnxt_napi *bnapi = bp->bnapi[i];
8343                 struct bnxt_cp_ring_info *cpr;
8344                 struct bnxt_stats_mem *stats;
8345
8346                 cpr = &bnapi->cp_ring;
8347                 stats = &cpr->stats;
8348                 if (!i)
8349                         ring0_stats = stats;
8350                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8351                                         ring0_stats->hw_masks,
8352                                         ring0_stats->len / 8, ignore_zero);
8353         }
8354         if (bp->flags & BNXT_FLAG_PORT_STATS) {
8355                 struct bnxt_stats_mem *stats = &bp->port_stats;
8356                 __le64 *hw_stats = stats->hw_stats;
8357                 u64 *sw_stats = stats->sw_stats;
8358                 u64 *masks = stats->hw_masks;
8359                 int cnt;
8360
8361                 cnt = sizeof(struct rx_port_stats) / 8;
8362                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8363
8364                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8365                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8366                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8367                 cnt = sizeof(struct tx_port_stats) / 8;
8368                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8369         }
8370         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
8371                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
8372                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
8373         }
8374 }
8375
8376 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
8377 {
8378         struct hwrm_port_qstats_input *req;
8379         struct bnxt_pf_info *pf = &bp->pf;
8380         int rc;
8381
8382         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
8383                 return 0;
8384
8385         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8386                 return -EOPNOTSUPP;
8387
8388         rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
8389         if (rc)
8390                 return rc;
8391
8392         req->flags = flags;
8393         req->port_id = cpu_to_le16(pf->port_id);
8394         req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
8395                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
8396         req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
8397         return hwrm_req_send(bp, req);
8398 }
8399
8400 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
8401 {
8402         struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
8403         struct hwrm_queue_pri2cos_qcfg_input *req_qc;
8404         struct hwrm_port_qstats_ext_output *resp_qs;
8405         struct hwrm_port_qstats_ext_input *req_qs;
8406         struct bnxt_pf_info *pf = &bp->pf;
8407         u32 tx_stat_size;
8408         int rc;
8409
8410         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
8411                 return 0;
8412
8413         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8414                 return -EOPNOTSUPP;
8415
8416         rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
8417         if (rc)
8418                 return rc;
8419
8420         req_qs->flags = flags;
8421         req_qs->port_id = cpu_to_le16(pf->port_id);
8422         req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8423         req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8424         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8425                        sizeof(struct tx_port_stats_ext) : 0;
8426         req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
8427         req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8428         resp_qs = hwrm_req_hold(bp, req_qs);
8429         rc = hwrm_req_send(bp, req_qs);
8430         if (!rc) {
8431                 bp->fw_rx_stats_ext_size =
8432                         le16_to_cpu(resp_qs->rx_stat_size) / 8;
8433                 if (BNXT_FW_MAJ(bp) < 220 &&
8434                     bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
8435                         bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
8436
8437                 bp->fw_tx_stats_ext_size = tx_stat_size ?
8438                         le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
8439         } else {
8440                 bp->fw_rx_stats_ext_size = 0;
8441                 bp->fw_tx_stats_ext_size = 0;
8442         }
8443         hwrm_req_drop(bp, req_qs);
8444
8445         if (flags)
8446                 return rc;
8447
8448         if (bp->fw_tx_stats_ext_size <=
8449             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8450                 bp->pri2cos_valid = 0;
8451                 return rc;
8452         }
8453
8454         rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
8455         if (rc)
8456                 return rc;
8457
8458         req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8459
8460         resp_qc = hwrm_req_hold(bp, req_qc);
8461         rc = hwrm_req_send(bp, req_qc);
8462         if (!rc) {
8463                 u8 *pri2cos;
8464                 int i, j;
8465
8466                 pri2cos = &resp_qc->pri0_cos_queue_id;
8467                 for (i = 0; i < 8; i++) {
8468                         u8 queue_id = pri2cos[i];
8469                         u8 queue_idx;
8470
8471                         /* Per port queue IDs start from 0, 10, 20, etc */
8472                         queue_idx = queue_id % 10;
8473                         if (queue_idx > BNXT_MAX_QUEUE) {
8474                                 bp->pri2cos_valid = false;
8475                                 hwrm_req_drop(bp, req_qc);
8476                                 return rc;
8477                         }
8478                         for (j = 0; j < bp->max_q; j++) {
8479                                 if (bp->q_ids[j] == queue_id)
8480                                         bp->pri2cos_idx[i] = queue_idx;
8481                         }
8482                 }
8483                 bp->pri2cos_valid = true;
8484         }
8485         hwrm_req_drop(bp, req_qc);
8486
8487         return rc;
8488 }
8489
8490 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8491 {
8492         bnxt_hwrm_tunnel_dst_port_free(bp,
8493                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8494         bnxt_hwrm_tunnel_dst_port_free(bp,
8495                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8496 }
8497
8498 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8499 {
8500         int rc, i;
8501         u32 tpa_flags = 0;
8502
8503         if (set_tpa)
8504                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
8505         else if (BNXT_NO_FW_ACCESS(bp))
8506                 return 0;
8507         for (i = 0; i < bp->nr_vnics; i++) {
8508                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8509                 if (rc) {
8510                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8511                                    i, rc);
8512                         return rc;
8513                 }
8514         }
8515         return 0;
8516 }
8517
8518 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8519 {
8520         int i;
8521
8522         for (i = 0; i < bp->nr_vnics; i++)
8523                 bnxt_hwrm_vnic_set_rss(bp, i, false);
8524 }
8525
8526 static void bnxt_clear_vnic(struct bnxt *bp)
8527 {
8528         if (!bp->vnic_info)
8529                 return;
8530
8531         bnxt_hwrm_clear_vnic_filter(bp);
8532         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8533                 /* clear all RSS setting before free vnic ctx */
8534                 bnxt_hwrm_clear_vnic_rss(bp);
8535                 bnxt_hwrm_vnic_ctx_free(bp);
8536         }
8537         /* before free the vnic, undo the vnic tpa settings */
8538         if (bp->flags & BNXT_FLAG_TPA)
8539                 bnxt_set_tpa(bp, false);
8540         bnxt_hwrm_vnic_free(bp);
8541         if (bp->flags & BNXT_FLAG_CHIP_P5)
8542                 bnxt_hwrm_vnic_ctx_free(bp);
8543 }
8544
8545 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8546                                     bool irq_re_init)
8547 {
8548         bnxt_clear_vnic(bp);
8549         bnxt_hwrm_ring_free(bp, close_path);
8550         bnxt_hwrm_ring_grp_free(bp);
8551         if (irq_re_init) {
8552                 bnxt_hwrm_stat_ctx_free(bp);
8553                 bnxt_hwrm_free_tunnel_ports(bp);
8554         }
8555 }
8556
8557 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8558 {
8559         struct hwrm_func_cfg_input *req;
8560         u8 evb_mode;
8561         int rc;
8562
8563         if (br_mode == BRIDGE_MODE_VEB)
8564                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8565         else if (br_mode == BRIDGE_MODE_VEPA)
8566                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8567         else
8568                 return -EINVAL;
8569
8570         rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8571         if (rc)
8572                 return rc;
8573
8574         req->fid = cpu_to_le16(0xffff);
8575         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8576         req->evb_mode = evb_mode;
8577         return hwrm_req_send(bp, req);
8578 }
8579
8580 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8581 {
8582         struct hwrm_func_cfg_input *req;
8583         int rc;
8584
8585         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8586                 return 0;
8587
8588         rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8589         if (rc)
8590                 return rc;
8591
8592         req->fid = cpu_to_le16(0xffff);
8593         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8594         req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8595         if (size == 128)
8596                 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8597
8598         return hwrm_req_send(bp, req);
8599 }
8600
8601 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8602 {
8603         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8604         int rc;
8605
8606         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8607                 goto skip_rss_ctx;
8608
8609         /* allocate context for vnic */
8610         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8611         if (rc) {
8612                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8613                            vnic_id, rc);
8614                 goto vnic_setup_err;
8615         }
8616         bp->rsscos_nr_ctxs++;
8617
8618         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8619                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8620                 if (rc) {
8621                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8622                                    vnic_id, rc);
8623                         goto vnic_setup_err;
8624                 }
8625                 bp->rsscos_nr_ctxs++;
8626         }
8627
8628 skip_rss_ctx:
8629         /* configure default vnic, ring grp */
8630         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8631         if (rc) {
8632                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8633                            vnic_id, rc);
8634                 goto vnic_setup_err;
8635         }
8636
8637         /* Enable RSS hashing on vnic */
8638         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8639         if (rc) {
8640                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8641                            vnic_id, rc);
8642                 goto vnic_setup_err;
8643         }
8644
8645         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8646                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8647                 if (rc) {
8648                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8649                                    vnic_id, rc);
8650                 }
8651         }
8652
8653 vnic_setup_err:
8654         return rc;
8655 }
8656
8657 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8658 {
8659         int rc, i, nr_ctxs;
8660
8661         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8662         for (i = 0; i < nr_ctxs; i++) {
8663                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8664                 if (rc) {
8665                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8666                                    vnic_id, i, rc);
8667                         break;
8668                 }
8669                 bp->rsscos_nr_ctxs++;
8670         }
8671         if (i < nr_ctxs)
8672                 return -ENOMEM;
8673
8674         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8675         if (rc) {
8676                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8677                            vnic_id, rc);
8678                 return rc;
8679         }
8680         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8681         if (rc) {
8682                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8683                            vnic_id, rc);
8684                 return rc;
8685         }
8686         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8687                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8688                 if (rc) {
8689                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8690                                    vnic_id, rc);
8691                 }
8692         }
8693         return rc;
8694 }
8695
8696 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8697 {
8698         if (bp->flags & BNXT_FLAG_CHIP_P5)
8699                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8700         else
8701                 return __bnxt_setup_vnic(bp, vnic_id);
8702 }
8703
8704 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8705 {
8706 #ifdef CONFIG_RFS_ACCEL
8707         int i, rc = 0;
8708
8709         if (bp->flags & BNXT_FLAG_CHIP_P5)
8710                 return 0;
8711
8712         for (i = 0; i < bp->rx_nr_rings; i++) {
8713                 struct bnxt_vnic_info *vnic;
8714                 u16 vnic_id = i + 1;
8715                 u16 ring_id = i;
8716
8717                 if (vnic_id >= bp->nr_vnics)
8718                         break;
8719
8720                 vnic = &bp->vnic_info[vnic_id];
8721                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8722                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8723                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8724                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8725                 if (rc) {
8726                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8727                                    vnic_id, rc);
8728                         break;
8729                 }
8730                 rc = bnxt_setup_vnic(bp, vnic_id);
8731                 if (rc)
8732                         break;
8733         }
8734         return rc;
8735 #else
8736         return 0;
8737 #endif
8738 }
8739
8740 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
8741 static bool bnxt_promisc_ok(struct bnxt *bp)
8742 {
8743 #ifdef CONFIG_BNXT_SRIOV
8744         if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
8745                 return false;
8746 #endif
8747         return true;
8748 }
8749
8750 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8751 {
8752         unsigned int rc = 0;
8753
8754         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8755         if (rc) {
8756                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8757                            rc);
8758                 return rc;
8759         }
8760
8761         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8762         if (rc) {
8763                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8764                            rc);
8765                 return rc;
8766         }
8767         return rc;
8768 }
8769
8770 static int bnxt_cfg_rx_mode(struct bnxt *);
8771 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8772
8773 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8774 {
8775         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8776         int rc = 0;
8777         unsigned int rx_nr_rings = bp->rx_nr_rings;
8778
8779         if (irq_re_init) {
8780                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8781                 if (rc) {
8782                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8783                                    rc);
8784                         goto err_out;
8785                 }
8786         }
8787
8788         rc = bnxt_hwrm_ring_alloc(bp);
8789         if (rc) {
8790                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8791                 goto err_out;
8792         }
8793
8794         rc = bnxt_hwrm_ring_grp_alloc(bp);
8795         if (rc) {
8796                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8797                 goto err_out;
8798         }
8799
8800         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8801                 rx_nr_rings--;
8802
8803         /* default vnic 0 */
8804         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8805         if (rc) {
8806                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8807                 goto err_out;
8808         }
8809
8810         rc = bnxt_setup_vnic(bp, 0);
8811         if (rc)
8812                 goto err_out;
8813
8814         if (bp->flags & BNXT_FLAG_RFS) {
8815                 rc = bnxt_alloc_rfs_vnics(bp);
8816                 if (rc)
8817                         goto err_out;
8818         }
8819
8820         if (bp->flags & BNXT_FLAG_TPA) {
8821                 rc = bnxt_set_tpa(bp, true);
8822                 if (rc)
8823                         goto err_out;
8824         }
8825
8826         if (BNXT_VF(bp))
8827                 bnxt_update_vf_mac(bp);
8828
8829         /* Filter for default vnic 0 */
8830         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8831         if (rc) {
8832                 if (BNXT_VF(bp) && rc == -ENODEV)
8833                         netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
8834                 else
8835                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8836                 goto err_out;
8837         }
8838         vnic->uc_filter_count = 1;
8839
8840         vnic->rx_mask = 0;
8841         if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
8842                 goto skip_rx_mask;
8843
8844         if (bp->dev->flags & IFF_BROADCAST)
8845                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8846
8847         if (bp->dev->flags & IFF_PROMISC)
8848                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8849
8850         if (bp->dev->flags & IFF_ALLMULTI) {
8851                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8852                 vnic->mc_list_count = 0;
8853         } else if (bp->dev->flags & IFF_MULTICAST) {
8854                 u32 mask = 0;
8855
8856                 bnxt_mc_list_updated(bp, &mask);
8857                 vnic->rx_mask |= mask;
8858         }
8859
8860         rc = bnxt_cfg_rx_mode(bp);
8861         if (rc)
8862                 goto err_out;
8863
8864 skip_rx_mask:
8865         rc = bnxt_hwrm_set_coal(bp);
8866         if (rc)
8867                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8868                                 rc);
8869
8870         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8871                 rc = bnxt_setup_nitroa0_vnic(bp);
8872                 if (rc)
8873                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8874                                    rc);
8875         }
8876
8877         if (BNXT_VF(bp)) {
8878                 bnxt_hwrm_func_qcfg(bp);
8879                 netdev_update_features(bp->dev);
8880         }
8881
8882         return 0;
8883
8884 err_out:
8885         bnxt_hwrm_resource_free(bp, 0, true);
8886
8887         return rc;
8888 }
8889
8890 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8891 {
8892         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8893         return 0;
8894 }
8895
8896 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8897 {
8898         bnxt_init_cp_rings(bp);
8899         bnxt_init_rx_rings(bp);
8900         bnxt_init_tx_rings(bp);
8901         bnxt_init_ring_grps(bp, irq_re_init);
8902         bnxt_init_vnics(bp);
8903
8904         return bnxt_init_chip(bp, irq_re_init);
8905 }
8906
8907 static int bnxt_set_real_num_queues(struct bnxt *bp)
8908 {
8909         int rc;
8910         struct net_device *dev = bp->dev;
8911
8912         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8913                                           bp->tx_nr_rings_xdp);
8914         if (rc)
8915                 return rc;
8916
8917         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8918         if (rc)
8919                 return rc;
8920
8921 #ifdef CONFIG_RFS_ACCEL
8922         if (bp->flags & BNXT_FLAG_RFS)
8923                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8924 #endif
8925
8926         return rc;
8927 }
8928
8929 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8930                            bool shared)
8931 {
8932         int _rx = *rx, _tx = *tx;
8933
8934         if (shared) {
8935                 *rx = min_t(int, _rx, max);
8936                 *tx = min_t(int, _tx, max);
8937         } else {
8938                 if (max < 2)
8939                         return -ENOMEM;
8940
8941                 while (_rx + _tx > max) {
8942                         if (_rx > _tx && _rx > 1)
8943                                 _rx--;
8944                         else if (_tx > 1)
8945                                 _tx--;
8946                 }
8947                 *rx = _rx;
8948                 *tx = _tx;
8949         }
8950         return 0;
8951 }
8952
8953 static void bnxt_setup_msix(struct bnxt *bp)
8954 {
8955         const int len = sizeof(bp->irq_tbl[0].name);
8956         struct net_device *dev = bp->dev;
8957         int tcs, i;
8958
8959         tcs = netdev_get_num_tc(dev);
8960         if (tcs) {
8961                 int i, off, count;
8962
8963                 for (i = 0; i < tcs; i++) {
8964                         count = bp->tx_nr_rings_per_tc;
8965                         off = i * count;
8966                         netdev_set_tc_queue(dev, i, count, off);
8967                 }
8968         }
8969
8970         for (i = 0; i < bp->cp_nr_rings; i++) {
8971                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8972                 char *attr;
8973
8974                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8975                         attr = "TxRx";
8976                 else if (i < bp->rx_nr_rings)
8977                         attr = "rx";
8978                 else
8979                         attr = "tx";
8980
8981                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8982                          attr, i);
8983                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8984         }
8985 }
8986
8987 static void bnxt_setup_inta(struct bnxt *bp)
8988 {
8989         const int len = sizeof(bp->irq_tbl[0].name);
8990
8991         if (netdev_get_num_tc(bp->dev))
8992                 netdev_reset_tc(bp->dev);
8993
8994         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8995                  0);
8996         bp->irq_tbl[0].handler = bnxt_inta;
8997 }
8998
8999 static int bnxt_init_int_mode(struct bnxt *bp);
9000
9001 static int bnxt_setup_int_mode(struct bnxt *bp)
9002 {
9003         int rc;
9004
9005         if (!bp->irq_tbl) {
9006                 rc = bnxt_init_int_mode(bp);
9007                 if (rc || !bp->irq_tbl)
9008                         return rc ?: -ENODEV;
9009         }
9010
9011         if (bp->flags & BNXT_FLAG_USING_MSIX)
9012                 bnxt_setup_msix(bp);
9013         else
9014                 bnxt_setup_inta(bp);
9015
9016         rc = bnxt_set_real_num_queues(bp);
9017         return rc;
9018 }
9019
9020 #ifdef CONFIG_RFS_ACCEL
9021 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
9022 {
9023         return bp->hw_resc.max_rsscos_ctxs;
9024 }
9025
9026 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
9027 {
9028         return bp->hw_resc.max_vnics;
9029 }
9030 #endif
9031
9032 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
9033 {
9034         return bp->hw_resc.max_stat_ctxs;
9035 }
9036
9037 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
9038 {
9039         return bp->hw_resc.max_cp_rings;
9040 }
9041
9042 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
9043 {
9044         unsigned int cp = bp->hw_resc.max_cp_rings;
9045
9046         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9047                 cp -= bnxt_get_ulp_msix_num(bp);
9048
9049         return cp;
9050 }
9051
9052 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
9053 {
9054         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9055
9056         if (bp->flags & BNXT_FLAG_CHIP_P5)
9057                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
9058
9059         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
9060 }
9061
9062 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
9063 {
9064         bp->hw_resc.max_irqs = max_irqs;
9065 }
9066
9067 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
9068 {
9069         unsigned int cp;
9070
9071         cp = bnxt_get_max_func_cp_rings_for_en(bp);
9072         if (bp->flags & BNXT_FLAG_CHIP_P5)
9073                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
9074         else
9075                 return cp - bp->cp_nr_rings;
9076 }
9077
9078 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
9079 {
9080         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
9081 }
9082
9083 int bnxt_get_avail_msix(struct bnxt *bp, int num)
9084 {
9085         int max_cp = bnxt_get_max_func_cp_rings(bp);
9086         int max_irq = bnxt_get_max_func_irqs(bp);
9087         int total_req = bp->cp_nr_rings + num;
9088         int max_idx, avail_msix;
9089
9090         max_idx = bp->total_irqs;
9091         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9092                 max_idx = min_t(int, bp->total_irqs, max_cp);
9093         avail_msix = max_idx - bp->cp_nr_rings;
9094         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
9095                 return avail_msix;
9096
9097         if (max_irq < total_req) {
9098                 num = max_irq - bp->cp_nr_rings;
9099                 if (num <= 0)
9100                         return 0;
9101         }
9102         return num;
9103 }
9104
9105 static int bnxt_get_num_msix(struct bnxt *bp)
9106 {
9107         if (!BNXT_NEW_RM(bp))
9108                 return bnxt_get_max_func_irqs(bp);
9109
9110         return bnxt_nq_rings_in_use(bp);
9111 }
9112
9113 static int bnxt_init_msix(struct bnxt *bp)
9114 {
9115         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
9116         struct msix_entry *msix_ent;
9117
9118         total_vecs = bnxt_get_num_msix(bp);
9119         max = bnxt_get_max_func_irqs(bp);
9120         if (total_vecs > max)
9121                 total_vecs = max;
9122
9123         if (!total_vecs)
9124                 return 0;
9125
9126         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
9127         if (!msix_ent)
9128                 return -ENOMEM;
9129
9130         for (i = 0; i < total_vecs; i++) {
9131                 msix_ent[i].entry = i;
9132                 msix_ent[i].vector = 0;
9133         }
9134
9135         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
9136                 min = 2;
9137
9138         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
9139         ulp_msix = bnxt_get_ulp_msix_num(bp);
9140         if (total_vecs < 0 || total_vecs < ulp_msix) {
9141                 rc = -ENODEV;
9142                 goto msix_setup_exit;
9143         }
9144
9145         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
9146         if (bp->irq_tbl) {
9147                 for (i = 0; i < total_vecs; i++)
9148                         bp->irq_tbl[i].vector = msix_ent[i].vector;
9149
9150                 bp->total_irqs = total_vecs;
9151                 /* Trim rings based upon num of vectors allocated */
9152                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
9153                                      total_vecs - ulp_msix, min == 1);
9154                 if (rc)
9155                         goto msix_setup_exit;
9156
9157                 bp->cp_nr_rings = (min == 1) ?
9158                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9159                                   bp->tx_nr_rings + bp->rx_nr_rings;
9160
9161         } else {
9162                 rc = -ENOMEM;
9163                 goto msix_setup_exit;
9164         }
9165         bp->flags |= BNXT_FLAG_USING_MSIX;
9166         kfree(msix_ent);
9167         return 0;
9168
9169 msix_setup_exit:
9170         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
9171         kfree(bp->irq_tbl);
9172         bp->irq_tbl = NULL;
9173         pci_disable_msix(bp->pdev);
9174         kfree(msix_ent);
9175         return rc;
9176 }
9177
9178 static int bnxt_init_inta(struct bnxt *bp)
9179 {
9180         bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
9181         if (!bp->irq_tbl)
9182                 return -ENOMEM;
9183
9184         bp->total_irqs = 1;
9185         bp->rx_nr_rings = 1;
9186         bp->tx_nr_rings = 1;
9187         bp->cp_nr_rings = 1;
9188         bp->flags |= BNXT_FLAG_SHARED_RINGS;
9189         bp->irq_tbl[0].vector = bp->pdev->irq;
9190         return 0;
9191 }
9192
9193 static int bnxt_init_int_mode(struct bnxt *bp)
9194 {
9195         int rc = -ENODEV;
9196
9197         if (bp->flags & BNXT_FLAG_MSIX_CAP)
9198                 rc = bnxt_init_msix(bp);
9199
9200         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
9201                 /* fallback to INTA */
9202                 rc = bnxt_init_inta(bp);
9203         }
9204         return rc;
9205 }
9206
9207 static void bnxt_clear_int_mode(struct bnxt *bp)
9208 {
9209         if (bp->flags & BNXT_FLAG_USING_MSIX)
9210                 pci_disable_msix(bp->pdev);
9211
9212         kfree(bp->irq_tbl);
9213         bp->irq_tbl = NULL;
9214         bp->flags &= ~BNXT_FLAG_USING_MSIX;
9215 }
9216
9217 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
9218 {
9219         int tcs = netdev_get_num_tc(bp->dev);
9220         bool irq_cleared = false;
9221         int rc;
9222
9223         if (!bnxt_need_reserve_rings(bp))
9224                 return 0;
9225
9226         if (irq_re_init && BNXT_NEW_RM(bp) &&
9227             bnxt_get_num_msix(bp) != bp->total_irqs) {
9228                 bnxt_ulp_irq_stop(bp);
9229                 bnxt_clear_int_mode(bp);
9230                 irq_cleared = true;
9231         }
9232         rc = __bnxt_reserve_rings(bp);
9233         if (irq_cleared) {
9234                 if (!rc)
9235                         rc = bnxt_init_int_mode(bp);
9236                 bnxt_ulp_irq_restart(bp, rc);
9237         }
9238         if (rc) {
9239                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
9240                 return rc;
9241         }
9242         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
9243                 netdev_err(bp->dev, "tx ring reservation failure\n");
9244                 netdev_reset_tc(bp->dev);
9245                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9246                 return -ENOMEM;
9247         }
9248         return 0;
9249 }
9250
9251 static void bnxt_free_irq(struct bnxt *bp)
9252 {
9253         struct bnxt_irq *irq;
9254         int i;
9255
9256 #ifdef CONFIG_RFS_ACCEL
9257         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
9258         bp->dev->rx_cpu_rmap = NULL;
9259 #endif
9260         if (!bp->irq_tbl || !bp->bnapi)
9261                 return;
9262
9263         for (i = 0; i < bp->cp_nr_rings; i++) {
9264                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9265
9266                 irq = &bp->irq_tbl[map_idx];
9267                 if (irq->requested) {
9268                         if (irq->have_cpumask) {
9269                                 irq_set_affinity_hint(irq->vector, NULL);
9270                                 free_cpumask_var(irq->cpu_mask);
9271                                 irq->have_cpumask = 0;
9272                         }
9273                         free_irq(irq->vector, bp->bnapi[i]);
9274                 }
9275
9276                 irq->requested = 0;
9277         }
9278 }
9279
9280 static int bnxt_request_irq(struct bnxt *bp)
9281 {
9282         int i, j, rc = 0;
9283         unsigned long flags = 0;
9284 #ifdef CONFIG_RFS_ACCEL
9285         struct cpu_rmap *rmap;
9286 #endif
9287
9288         rc = bnxt_setup_int_mode(bp);
9289         if (rc) {
9290                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
9291                            rc);
9292                 return rc;
9293         }
9294 #ifdef CONFIG_RFS_ACCEL
9295         rmap = bp->dev->rx_cpu_rmap;
9296 #endif
9297         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
9298                 flags = IRQF_SHARED;
9299
9300         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
9301                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9302                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
9303
9304 #ifdef CONFIG_RFS_ACCEL
9305                 if (rmap && bp->bnapi[i]->rx_ring) {
9306                         rc = irq_cpu_rmap_add(rmap, irq->vector);
9307                         if (rc)
9308                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
9309                                             j);
9310                         j++;
9311                 }
9312 #endif
9313                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
9314                                  bp->bnapi[i]);
9315                 if (rc)
9316                         break;
9317
9318                 irq->requested = 1;
9319
9320                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
9321                         int numa_node = dev_to_node(&bp->pdev->dev);
9322
9323                         irq->have_cpumask = 1;
9324                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
9325                                         irq->cpu_mask);
9326                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
9327                         if (rc) {
9328                                 netdev_warn(bp->dev,
9329                                             "Set affinity failed, IRQ = %d\n",
9330                                             irq->vector);
9331                                 break;
9332                         }
9333                 }
9334         }
9335         return rc;
9336 }
9337
9338 static void bnxt_del_napi(struct bnxt *bp)
9339 {
9340         int i;
9341
9342         if (!bp->bnapi)
9343                 return;
9344
9345         for (i = 0; i < bp->cp_nr_rings; i++) {
9346                 struct bnxt_napi *bnapi = bp->bnapi[i];
9347
9348                 __netif_napi_del(&bnapi->napi);
9349         }
9350         /* We called __netif_napi_del(), we need
9351          * to respect an RCU grace period before freeing napi structures.
9352          */
9353         synchronize_net();
9354 }
9355
9356 static void bnxt_init_napi(struct bnxt *bp)
9357 {
9358         int i;
9359         unsigned int cp_nr_rings = bp->cp_nr_rings;
9360         struct bnxt_napi *bnapi;
9361
9362         if (bp->flags & BNXT_FLAG_USING_MSIX) {
9363                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
9364
9365                 if (bp->flags & BNXT_FLAG_CHIP_P5)
9366                         poll_fn = bnxt_poll_p5;
9367                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9368                         cp_nr_rings--;
9369                 for (i = 0; i < cp_nr_rings; i++) {
9370                         bnapi = bp->bnapi[i];
9371                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
9372                 }
9373                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9374                         bnapi = bp->bnapi[cp_nr_rings];
9375                         netif_napi_add(bp->dev, &bnapi->napi,
9376                                        bnxt_poll_nitroa0, 64);
9377                 }
9378         } else {
9379                 bnapi = bp->bnapi[0];
9380                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
9381         }
9382 }
9383
9384 static void bnxt_disable_napi(struct bnxt *bp)
9385 {
9386         int i;
9387
9388         if (!bp->bnapi ||
9389             test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
9390                 return;
9391
9392         for (i = 0; i < bp->cp_nr_rings; i++) {
9393                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
9394
9395                 napi_disable(&bp->bnapi[i]->napi);
9396                 if (bp->bnapi[i]->rx_ring)
9397                         cancel_work_sync(&cpr->dim.work);
9398         }
9399 }
9400
9401 static void bnxt_enable_napi(struct bnxt *bp)
9402 {
9403         int i;
9404
9405         clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
9406         for (i = 0; i < bp->cp_nr_rings; i++) {
9407                 struct bnxt_napi *bnapi = bp->bnapi[i];
9408                 struct bnxt_cp_ring_info *cpr;
9409
9410                 cpr = &bnapi->cp_ring;
9411                 if (bnapi->in_reset)
9412                         cpr->sw_stats.rx.rx_resets++;
9413                 bnapi->in_reset = false;
9414
9415                 if (bnapi->rx_ring) {
9416                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
9417                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
9418                 }
9419                 napi_enable(&bnapi->napi);
9420         }
9421 }
9422
9423 void bnxt_tx_disable(struct bnxt *bp)
9424 {
9425         int i;
9426         struct bnxt_tx_ring_info *txr;
9427
9428         if (bp->tx_ring) {
9429                 for (i = 0; i < bp->tx_nr_rings; i++) {
9430                         txr = &bp->tx_ring[i];
9431                         WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
9432                 }
9433         }
9434         /* Make sure napi polls see @dev_state change */
9435         synchronize_net();
9436         /* Drop carrier first to prevent TX timeout */
9437         netif_carrier_off(bp->dev);
9438         /* Stop all TX queues */
9439         netif_tx_disable(bp->dev);
9440 }
9441
9442 void bnxt_tx_enable(struct bnxt *bp)
9443 {
9444         int i;
9445         struct bnxt_tx_ring_info *txr;
9446
9447         for (i = 0; i < bp->tx_nr_rings; i++) {
9448                 txr = &bp->tx_ring[i];
9449                 WRITE_ONCE(txr->dev_state, 0);
9450         }
9451         /* Make sure napi polls see @dev_state change */
9452         synchronize_net();
9453         netif_tx_wake_all_queues(bp->dev);
9454         if (BNXT_LINK_IS_UP(bp))
9455                 netif_carrier_on(bp->dev);
9456 }
9457
9458 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9459 {
9460         u8 active_fec = link_info->active_fec_sig_mode &
9461                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9462
9463         switch (active_fec) {
9464         default:
9465         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9466                 return "None";
9467         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9468                 return "Clause 74 BaseR";
9469         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9470                 return "Clause 91 RS(528,514)";
9471         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9472                 return "Clause 91 RS544_1XN";
9473         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9474                 return "Clause 91 RS(544,514)";
9475         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9476                 return "Clause 91 RS272_1XN";
9477         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9478                 return "Clause 91 RS(272,257)";
9479         }
9480 }
9481
9482 void bnxt_report_link(struct bnxt *bp)
9483 {
9484         if (BNXT_LINK_IS_UP(bp)) {
9485                 const char *signal = "";
9486                 const char *flow_ctrl;
9487                 const char *duplex;
9488                 u32 speed;
9489                 u16 fec;
9490
9491                 netif_carrier_on(bp->dev);
9492                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9493                 if (speed == SPEED_UNKNOWN) {
9494                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9495                         return;
9496                 }
9497                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9498                         duplex = "full";
9499                 else
9500                         duplex = "half";
9501                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9502                         flow_ctrl = "ON - receive & transmit";
9503                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9504                         flow_ctrl = "ON - transmit";
9505                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9506                         flow_ctrl = "ON - receive";
9507                 else
9508                         flow_ctrl = "none";
9509                 if (bp->link_info.phy_qcfg_resp.option_flags &
9510                     PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
9511                         u8 sig_mode = bp->link_info.active_fec_sig_mode &
9512                                       PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
9513                         switch (sig_mode) {
9514                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
9515                                 signal = "(NRZ) ";
9516                                 break;
9517                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
9518                                 signal = "(PAM4) ";
9519                                 break;
9520                         default:
9521                                 break;
9522                         }
9523                 }
9524                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
9525                             speed, signal, duplex, flow_ctrl);
9526                 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
9527                         netdev_info(bp->dev, "EEE is %s\n",
9528                                     bp->eee.eee_active ? "active" :
9529                                                          "not active");
9530                 fec = bp->link_info.fec_cfg;
9531                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9532                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9533                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9534                                     bnxt_report_fec(&bp->link_info));
9535         } else {
9536                 netif_carrier_off(bp->dev);
9537                 netdev_err(bp->dev, "NIC Link is Down\n");
9538         }
9539 }
9540
9541 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9542 {
9543         if (!resp->supported_speeds_auto_mode &&
9544             !resp->supported_speeds_force_mode &&
9545             !resp->supported_pam4_speeds_auto_mode &&
9546             !resp->supported_pam4_speeds_force_mode)
9547                 return true;
9548         return false;
9549 }
9550
9551 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9552 {
9553         struct bnxt_link_info *link_info = &bp->link_info;
9554         struct hwrm_port_phy_qcaps_output *resp;
9555         struct hwrm_port_phy_qcaps_input *req;
9556         int rc = 0;
9557
9558         if (bp->hwrm_spec_code < 0x10201)
9559                 return 0;
9560
9561         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
9562         if (rc)
9563                 return rc;
9564
9565         resp = hwrm_req_hold(bp, req);
9566         rc = hwrm_req_send(bp, req);
9567         if (rc)
9568                 goto hwrm_phy_qcaps_exit;
9569
9570         bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
9571         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9572                 struct ethtool_eee *eee = &bp->eee;
9573                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9574
9575                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9576                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9577                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9578                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9579                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9580         }
9581
9582         if (bp->hwrm_spec_code >= 0x10a01) {
9583                 if (bnxt_phy_qcaps_no_speed(resp)) {
9584                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9585                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9586                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9587                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9588                         netdev_info(bp->dev, "Ethernet link enabled\n");
9589                         /* Phy re-enabled, reprobe the speeds */
9590                         link_info->support_auto_speeds = 0;
9591                         link_info->support_pam4_auto_speeds = 0;
9592                 }
9593         }
9594         if (resp->supported_speeds_auto_mode)
9595                 link_info->support_auto_speeds =
9596                         le16_to_cpu(resp->supported_speeds_auto_mode);
9597         if (resp->supported_pam4_speeds_auto_mode)
9598                 link_info->support_pam4_auto_speeds =
9599                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9600
9601         bp->port_count = resp->port_cnt;
9602
9603 hwrm_phy_qcaps_exit:
9604         hwrm_req_drop(bp, req);
9605         return rc;
9606 }
9607
9608 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9609 {
9610         u16 diff = advertising ^ supported;
9611
9612         return ((supported | diff) != supported);
9613 }
9614
9615 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9616 {
9617         struct bnxt_link_info *link_info = &bp->link_info;
9618         struct hwrm_port_phy_qcfg_output *resp;
9619         struct hwrm_port_phy_qcfg_input *req;
9620         u8 link_state = link_info->link_state;
9621         bool support_changed = false;
9622         int rc;
9623
9624         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
9625         if (rc)
9626                 return rc;
9627
9628         resp = hwrm_req_hold(bp, req);
9629         rc = hwrm_req_send(bp, req);
9630         if (rc) {
9631                 hwrm_req_drop(bp, req);
9632                 if (BNXT_VF(bp) && rc == -ENODEV) {
9633                         netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
9634                         rc = 0;
9635                 }
9636                 return rc;
9637         }
9638
9639         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9640         link_info->phy_link_status = resp->link;
9641         link_info->duplex = resp->duplex_cfg;
9642         if (bp->hwrm_spec_code >= 0x10800)
9643                 link_info->duplex = resp->duplex_state;
9644         link_info->pause = resp->pause;
9645         link_info->auto_mode = resp->auto_mode;
9646         link_info->auto_pause_setting = resp->auto_pause;
9647         link_info->lp_pause = resp->link_partner_adv_pause;
9648         link_info->force_pause_setting = resp->force_pause;
9649         link_info->duplex_setting = resp->duplex_cfg;
9650         if (link_info->phy_link_status == BNXT_LINK_LINK)
9651                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9652         else
9653                 link_info->link_speed = 0;
9654         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9655         link_info->force_pam4_link_speed =
9656                 le16_to_cpu(resp->force_pam4_link_speed);
9657         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9658         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9659         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9660         link_info->auto_pam4_link_speeds =
9661                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9662         link_info->lp_auto_link_speeds =
9663                 le16_to_cpu(resp->link_partner_adv_speeds);
9664         link_info->lp_auto_pam4_link_speeds =
9665                 resp->link_partner_pam4_adv_speeds;
9666         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9667         link_info->phy_ver[0] = resp->phy_maj;
9668         link_info->phy_ver[1] = resp->phy_min;
9669         link_info->phy_ver[2] = resp->phy_bld;
9670         link_info->media_type = resp->media_type;
9671         link_info->phy_type = resp->phy_type;
9672         link_info->transceiver = resp->xcvr_pkg_type;
9673         link_info->phy_addr = resp->eee_config_phy_addr &
9674                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9675         link_info->module_status = resp->module_status;
9676
9677         if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
9678                 struct ethtool_eee *eee = &bp->eee;
9679                 u16 fw_speeds;
9680
9681                 eee->eee_active = 0;
9682                 if (resp->eee_config_phy_addr &
9683                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9684                         eee->eee_active = 1;
9685                         fw_speeds = le16_to_cpu(
9686                                 resp->link_partner_adv_eee_link_speed_mask);
9687                         eee->lp_advertised =
9688                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9689                 }
9690
9691                 /* Pull initial EEE config */
9692                 if (!chng_link_state) {
9693                         if (resp->eee_config_phy_addr &
9694                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9695                                 eee->eee_enabled = 1;
9696
9697                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9698                         eee->advertised =
9699                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9700
9701                         if (resp->eee_config_phy_addr &
9702                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9703                                 __le32 tmr;
9704
9705                                 eee->tx_lpi_enabled = 1;
9706                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9707                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9708                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9709                         }
9710                 }
9711         }
9712
9713         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9714         if (bp->hwrm_spec_code >= 0x10504) {
9715                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9716                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9717         }
9718         /* TODO: need to add more logic to report VF link */
9719         if (chng_link_state) {
9720                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9721                         link_info->link_state = BNXT_LINK_STATE_UP;
9722                 else
9723                         link_info->link_state = BNXT_LINK_STATE_DOWN;
9724                 if (link_state != link_info->link_state)
9725                         bnxt_report_link(bp);
9726         } else {
9727                 /* always link down if not require to update link state */
9728                 link_info->link_state = BNXT_LINK_STATE_DOWN;
9729         }
9730         hwrm_req_drop(bp, req);
9731
9732         if (!BNXT_PHY_CFG_ABLE(bp))
9733                 return 0;
9734
9735         /* Check if any advertised speeds are no longer supported. The caller
9736          * holds the link_lock mutex, so we can modify link_info settings.
9737          */
9738         if (bnxt_support_dropped(link_info->advertising,
9739                                  link_info->support_auto_speeds)) {
9740                 link_info->advertising = link_info->support_auto_speeds;
9741                 support_changed = true;
9742         }
9743         if (bnxt_support_dropped(link_info->advertising_pam4,
9744                                  link_info->support_pam4_auto_speeds)) {
9745                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9746                 support_changed = true;
9747         }
9748         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9749                 bnxt_hwrm_set_link_setting(bp, true, false);
9750         return 0;
9751 }
9752
9753 static void bnxt_get_port_module_status(struct bnxt *bp)
9754 {
9755         struct bnxt_link_info *link_info = &bp->link_info;
9756         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9757         u8 module_status;
9758
9759         if (bnxt_update_link(bp, true))
9760                 return;
9761
9762         module_status = link_info->module_status;
9763         switch (module_status) {
9764         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9765         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9766         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9767                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9768                             bp->pf.port_id);
9769                 if (bp->hwrm_spec_code >= 0x10201) {
9770                         netdev_warn(bp->dev, "Module part number %s\n",
9771                                     resp->phy_vendor_partnumber);
9772                 }
9773                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9774                         netdev_warn(bp->dev, "TX is disabled\n");
9775                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9776                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9777         }
9778 }
9779
9780 static void
9781 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9782 {
9783         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9784                 if (bp->hwrm_spec_code >= 0x10201)
9785                         req->auto_pause =
9786                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9787                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9788                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9789                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9790                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9791                 req->enables |=
9792                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9793         } else {
9794                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9795                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9796                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9797                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9798                 req->enables |=
9799                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9800                 if (bp->hwrm_spec_code >= 0x10201) {
9801                         req->auto_pause = req->force_pause;
9802                         req->enables |= cpu_to_le32(
9803                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9804                 }
9805         }
9806 }
9807
9808 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9809 {
9810         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9811                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9812                 if (bp->link_info.advertising) {
9813                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9814                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9815                 }
9816                 if (bp->link_info.advertising_pam4) {
9817                         req->enables |=
9818                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9819                         req->auto_link_pam4_speed_mask =
9820                                 cpu_to_le16(bp->link_info.advertising_pam4);
9821                 }
9822                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9823                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9824         } else {
9825                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9826                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9827                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9828                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9829                 } else {
9830                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9831                 }
9832         }
9833
9834         /* tell chimp that the setting takes effect immediately */
9835         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9836 }
9837
9838 int bnxt_hwrm_set_pause(struct bnxt *bp)
9839 {
9840         struct hwrm_port_phy_cfg_input *req;
9841         int rc;
9842
9843         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9844         if (rc)
9845                 return rc;
9846
9847         bnxt_hwrm_set_pause_common(bp, req);
9848
9849         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9850             bp->link_info.force_link_chng)
9851                 bnxt_hwrm_set_link_common(bp, req);
9852
9853         rc = hwrm_req_send(bp, req);
9854         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9855                 /* since changing of pause setting doesn't trigger any link
9856                  * change event, the driver needs to update the current pause
9857                  * result upon successfully return of the phy_cfg command
9858                  */
9859                 bp->link_info.pause =
9860                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9861                 bp->link_info.auto_pause_setting = 0;
9862                 if (!bp->link_info.force_link_chng)
9863                         bnxt_report_link(bp);
9864         }
9865         bp->link_info.force_link_chng = false;
9866         return rc;
9867 }
9868
9869 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9870                               struct hwrm_port_phy_cfg_input *req)
9871 {
9872         struct ethtool_eee *eee = &bp->eee;
9873
9874         if (eee->eee_enabled) {
9875                 u16 eee_speeds;
9876                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9877
9878                 if (eee->tx_lpi_enabled)
9879                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9880                 else
9881                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9882
9883                 req->flags |= cpu_to_le32(flags);
9884                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9885                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9886                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9887         } else {
9888                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9889         }
9890 }
9891
9892 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9893 {
9894         struct hwrm_port_phy_cfg_input *req;
9895         int rc;
9896
9897         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9898         if (rc)
9899                 return rc;
9900
9901         if (set_pause)
9902                 bnxt_hwrm_set_pause_common(bp, req);
9903
9904         bnxt_hwrm_set_link_common(bp, req);
9905
9906         if (set_eee)
9907                 bnxt_hwrm_set_eee(bp, req);
9908         return hwrm_req_send(bp, req);
9909 }
9910
9911 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9912 {
9913         struct hwrm_port_phy_cfg_input *req;
9914         int rc;
9915
9916         if (!BNXT_SINGLE_PF(bp))
9917                 return 0;
9918
9919         if (pci_num_vf(bp->pdev) &&
9920             !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
9921                 return 0;
9922
9923         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9924         if (rc)
9925                 return rc;
9926
9927         req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9928         rc = hwrm_req_send(bp, req);
9929         if (!rc) {
9930                 mutex_lock(&bp->link_lock);
9931                 /* Device is not obliged link down in certain scenarios, even
9932                  * when forced. Setting the state unknown is consistent with
9933                  * driver startup and will force link state to be reported
9934                  * during subsequent open based on PORT_PHY_QCFG.
9935                  */
9936                 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
9937                 mutex_unlock(&bp->link_lock);
9938         }
9939         return rc;
9940 }
9941
9942 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9943 {
9944 #ifdef CONFIG_TEE_BNXT_FW
9945         int rc = tee_bnxt_fw_load();
9946
9947         if (rc)
9948                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9949
9950         return rc;
9951 #else
9952         netdev_err(bp->dev, "OP-TEE not supported\n");
9953         return -ENODEV;
9954 #endif
9955 }
9956
9957 static int bnxt_try_recover_fw(struct bnxt *bp)
9958 {
9959         if (bp->fw_health && bp->fw_health->status_reliable) {
9960                 int retry = 0, rc;
9961                 u32 sts;
9962
9963                 do {
9964                         sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9965                         rc = bnxt_hwrm_poll(bp);
9966                         if (!BNXT_FW_IS_BOOTING(sts) &&
9967                             !BNXT_FW_IS_RECOVERING(sts))
9968                                 break;
9969                         retry++;
9970                 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9971
9972                 if (!BNXT_FW_IS_HEALTHY(sts)) {
9973                         netdev_err(bp->dev,
9974                                    "Firmware not responding, status: 0x%x\n",
9975                                    sts);
9976                         rc = -ENODEV;
9977                 }
9978                 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
9979                         netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
9980                         return bnxt_fw_reset_via_optee(bp);
9981                 }
9982                 return rc;
9983         }
9984
9985         return -ENODEV;
9986 }
9987
9988 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
9989 {
9990         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9991         int rc;
9992
9993         if (!BNXT_NEW_RM(bp))
9994                 return 0; /* no resource reservations required */
9995
9996         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9997         if (rc)
9998                 netdev_err(bp->dev, "resc_qcaps failed\n");
9999
10000         hw_resc->resv_cp_rings = 0;
10001         hw_resc->resv_stat_ctxs = 0;
10002         hw_resc->resv_irqs = 0;
10003         hw_resc->resv_tx_rings = 0;
10004         hw_resc->resv_rx_rings = 0;
10005         hw_resc->resv_hw_ring_grps = 0;
10006         hw_resc->resv_vnics = 0;
10007         if (!fw_reset) {
10008                 bp->tx_nr_rings = 0;
10009                 bp->rx_nr_rings = 0;
10010         }
10011
10012         return rc;
10013 }
10014
10015 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
10016 {
10017         struct hwrm_func_drv_if_change_output *resp;
10018         struct hwrm_func_drv_if_change_input *req;
10019         bool fw_reset = !bp->irq_tbl;
10020         bool resc_reinit = false;
10021         int rc, retry = 0;
10022         u32 flags = 0;
10023
10024         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
10025                 return 0;
10026
10027         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
10028         if (rc)
10029                 return rc;
10030
10031         if (up)
10032                 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
10033         resp = hwrm_req_hold(bp, req);
10034
10035         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
10036         while (retry < BNXT_FW_IF_RETRY) {
10037                 rc = hwrm_req_send(bp, req);
10038                 if (rc != -EAGAIN)
10039                         break;
10040
10041                 msleep(50);
10042                 retry++;
10043         }
10044
10045         if (rc == -EAGAIN) {
10046                 hwrm_req_drop(bp, req);
10047                 return rc;
10048         } else if (!rc) {
10049                 flags = le32_to_cpu(resp->flags);
10050         } else if (up) {
10051                 rc = bnxt_try_recover_fw(bp);
10052                 fw_reset = true;
10053         }
10054         hwrm_req_drop(bp, req);
10055         if (rc)
10056                 return rc;
10057
10058         if (!up) {
10059                 bnxt_inv_fw_health_reg(bp);
10060                 return 0;
10061         }
10062
10063         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
10064                 resc_reinit = true;
10065         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
10066                 fw_reset = true;
10067         else
10068                 bnxt_remap_fw_health_regs(bp);
10069
10070         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
10071                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
10072                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10073                 return -ENODEV;
10074         }
10075         if (resc_reinit || fw_reset) {
10076                 if (fw_reset) {
10077                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10078                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10079                                 bnxt_ulp_stop(bp);
10080                         bnxt_free_ctx_mem(bp);
10081                         kfree(bp->ctx);
10082                         bp->ctx = NULL;
10083                         bnxt_dcb_free(bp);
10084                         rc = bnxt_fw_init_one(bp);
10085                         if (rc) {
10086                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10087                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10088                                 return rc;
10089                         }
10090                         bnxt_clear_int_mode(bp);
10091                         rc = bnxt_init_int_mode(bp);
10092                         if (rc) {
10093                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10094                                 netdev_err(bp->dev, "init int mode failed\n");
10095                                 return rc;
10096                         }
10097                 }
10098                 rc = bnxt_cancel_reservations(bp, fw_reset);
10099         }
10100         return rc;
10101 }
10102
10103 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
10104 {
10105         struct hwrm_port_led_qcaps_output *resp;
10106         struct hwrm_port_led_qcaps_input *req;
10107         struct bnxt_pf_info *pf = &bp->pf;
10108         int rc;
10109
10110         bp->num_leds = 0;
10111         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
10112                 return 0;
10113
10114         rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
10115         if (rc)
10116                 return rc;
10117
10118         req->port_id = cpu_to_le16(pf->port_id);
10119         resp = hwrm_req_hold(bp, req);
10120         rc = hwrm_req_send(bp, req);
10121         if (rc) {
10122                 hwrm_req_drop(bp, req);
10123                 return rc;
10124         }
10125         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
10126                 int i;
10127
10128                 bp->num_leds = resp->num_leds;
10129                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
10130                                                  bp->num_leds);
10131                 for (i = 0; i < bp->num_leds; i++) {
10132                         struct bnxt_led_info *led = &bp->leds[i];
10133                         __le16 caps = led->led_state_caps;
10134
10135                         if (!led->led_group_id ||
10136                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
10137                                 bp->num_leds = 0;
10138                                 break;
10139                         }
10140                 }
10141         }
10142         hwrm_req_drop(bp, req);
10143         return 0;
10144 }
10145
10146 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
10147 {
10148         struct hwrm_wol_filter_alloc_output *resp;
10149         struct hwrm_wol_filter_alloc_input *req;
10150         int rc;
10151
10152         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
10153         if (rc)
10154                 return rc;
10155
10156         req->port_id = cpu_to_le16(bp->pf.port_id);
10157         req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
10158         req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
10159         memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
10160
10161         resp = hwrm_req_hold(bp, req);
10162         rc = hwrm_req_send(bp, req);
10163         if (!rc)
10164                 bp->wol_filter_id = resp->wol_filter_id;
10165         hwrm_req_drop(bp, req);
10166         return rc;
10167 }
10168
10169 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
10170 {
10171         struct hwrm_wol_filter_free_input *req;
10172         int rc;
10173
10174         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
10175         if (rc)
10176                 return rc;
10177
10178         req->port_id = cpu_to_le16(bp->pf.port_id);
10179         req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
10180         req->wol_filter_id = bp->wol_filter_id;
10181
10182         return hwrm_req_send(bp, req);
10183 }
10184
10185 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
10186 {
10187         struct hwrm_wol_filter_qcfg_output *resp;
10188         struct hwrm_wol_filter_qcfg_input *req;
10189         u16 next_handle = 0;
10190         int rc;
10191
10192         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
10193         if (rc)
10194                 return rc;
10195
10196         req->port_id = cpu_to_le16(bp->pf.port_id);
10197         req->handle = cpu_to_le16(handle);
10198         resp = hwrm_req_hold(bp, req);
10199         rc = hwrm_req_send(bp, req);
10200         if (!rc) {
10201                 next_handle = le16_to_cpu(resp->next_handle);
10202                 if (next_handle != 0) {
10203                         if (resp->wol_type ==
10204                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
10205                                 bp->wol = 1;
10206                                 bp->wol_filter_id = resp->wol_filter_id;
10207                         }
10208                 }
10209         }
10210         hwrm_req_drop(bp, req);
10211         return next_handle;
10212 }
10213
10214 static void bnxt_get_wol_settings(struct bnxt *bp)
10215 {
10216         u16 handle = 0;
10217
10218         bp->wol = 0;
10219         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
10220                 return;
10221
10222         do {
10223                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
10224         } while (handle && handle != 0xffff);
10225 }
10226
10227 #ifdef CONFIG_BNXT_HWMON
10228 static ssize_t bnxt_show_temp(struct device *dev,
10229                               struct device_attribute *devattr, char *buf)
10230 {
10231         struct hwrm_temp_monitor_query_output *resp;
10232         struct hwrm_temp_monitor_query_input *req;
10233         struct bnxt *bp = dev_get_drvdata(dev);
10234         u32 len = 0;
10235         int rc;
10236
10237         rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10238         if (rc)
10239                 return rc;
10240         resp = hwrm_req_hold(bp, req);
10241         rc = hwrm_req_send(bp, req);
10242         if (!rc)
10243                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
10244         hwrm_req_drop(bp, req);
10245         if (rc)
10246                 return rc;
10247         return len;
10248 }
10249 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
10250
10251 static struct attribute *bnxt_attrs[] = {
10252         &sensor_dev_attr_temp1_input.dev_attr.attr,
10253         NULL
10254 };
10255 ATTRIBUTE_GROUPS(bnxt);
10256
10257 static void bnxt_hwmon_close(struct bnxt *bp)
10258 {
10259         if (bp->hwmon_dev) {
10260                 hwmon_device_unregister(bp->hwmon_dev);
10261                 bp->hwmon_dev = NULL;
10262         }
10263 }
10264
10265 static void bnxt_hwmon_open(struct bnxt *bp)
10266 {
10267         struct hwrm_temp_monitor_query_input *req;
10268         struct pci_dev *pdev = bp->pdev;
10269         int rc;
10270
10271         rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10272         if (!rc)
10273                 rc = hwrm_req_send_silent(bp, req);
10274         if (rc == -EACCES || rc == -EOPNOTSUPP) {
10275                 bnxt_hwmon_close(bp);
10276                 return;
10277         }
10278
10279         if (bp->hwmon_dev)
10280                 return;
10281
10282         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
10283                                                           DRV_MODULE_NAME, bp,
10284                                                           bnxt_groups);
10285         if (IS_ERR(bp->hwmon_dev)) {
10286                 bp->hwmon_dev = NULL;
10287                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
10288         }
10289 }
10290 #else
10291 static void bnxt_hwmon_close(struct bnxt *bp)
10292 {
10293 }
10294
10295 static void bnxt_hwmon_open(struct bnxt *bp)
10296 {
10297 }
10298 #endif
10299
10300 static bool bnxt_eee_config_ok(struct bnxt *bp)
10301 {
10302         struct ethtool_eee *eee = &bp->eee;
10303         struct bnxt_link_info *link_info = &bp->link_info;
10304
10305         if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
10306                 return true;
10307
10308         if (eee->eee_enabled) {
10309                 u32 advertising =
10310                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
10311
10312                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10313                         eee->eee_enabled = 0;
10314                         return false;
10315                 }
10316                 if (eee->advertised & ~advertising) {
10317                         eee->advertised = advertising & eee->supported;
10318                         return false;
10319                 }
10320         }
10321         return true;
10322 }
10323
10324 static int bnxt_update_phy_setting(struct bnxt *bp)
10325 {
10326         int rc;
10327         bool update_link = false;
10328         bool update_pause = false;
10329         bool update_eee = false;
10330         struct bnxt_link_info *link_info = &bp->link_info;
10331
10332         rc = bnxt_update_link(bp, true);
10333         if (rc) {
10334                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
10335                            rc);
10336                 return rc;
10337         }
10338         if (!BNXT_SINGLE_PF(bp))
10339                 return 0;
10340
10341         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10342             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
10343             link_info->req_flow_ctrl)
10344                 update_pause = true;
10345         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10346             link_info->force_pause_setting != link_info->req_flow_ctrl)
10347                 update_pause = true;
10348         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10349                 if (BNXT_AUTO_MODE(link_info->auto_mode))
10350                         update_link = true;
10351                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
10352                     link_info->req_link_speed != link_info->force_link_speed)
10353                         update_link = true;
10354                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
10355                          link_info->req_link_speed != link_info->force_pam4_link_speed)
10356                         update_link = true;
10357                 if (link_info->req_duplex != link_info->duplex_setting)
10358                         update_link = true;
10359         } else {
10360                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
10361                         update_link = true;
10362                 if (link_info->advertising != link_info->auto_link_speeds ||
10363                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
10364                         update_link = true;
10365         }
10366
10367         /* The last close may have shutdown the link, so need to call
10368          * PHY_CFG to bring it back up.
10369          */
10370         if (!BNXT_LINK_IS_UP(bp))
10371                 update_link = true;
10372
10373         if (!bnxt_eee_config_ok(bp))
10374                 update_eee = true;
10375
10376         if (update_link)
10377                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
10378         else if (update_pause)
10379                 rc = bnxt_hwrm_set_pause(bp);
10380         if (rc) {
10381                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
10382                            rc);
10383                 return rc;
10384         }
10385
10386         return rc;
10387 }
10388
10389 /* Common routine to pre-map certain register block to different GRC window.
10390  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
10391  * in PF and 3 windows in VF that can be customized to map in different
10392  * register blocks.
10393  */
10394 static void bnxt_preset_reg_win(struct bnxt *bp)
10395 {
10396         if (BNXT_PF(bp)) {
10397                 /* CAG registers map to GRC window #4 */
10398                 writel(BNXT_CAG_REG_BASE,
10399                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
10400         }
10401 }
10402
10403 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
10404
10405 static int bnxt_reinit_after_abort(struct bnxt *bp)
10406 {
10407         int rc;
10408
10409         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10410                 return -EBUSY;
10411
10412         if (bp->dev->reg_state == NETREG_UNREGISTERED)
10413                 return -ENODEV;
10414
10415         rc = bnxt_fw_init_one(bp);
10416         if (!rc) {
10417                 bnxt_clear_int_mode(bp);
10418                 rc = bnxt_init_int_mode(bp);
10419                 if (!rc) {
10420                         clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10421                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10422                 }
10423         }
10424         return rc;
10425 }
10426
10427 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10428 {
10429         int rc = 0;
10430
10431         bnxt_preset_reg_win(bp);
10432         netif_carrier_off(bp->dev);
10433         if (irq_re_init) {
10434                 /* Reserve rings now if none were reserved at driver probe. */
10435                 rc = bnxt_init_dflt_ring_mode(bp);
10436                 if (rc) {
10437                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
10438                         return rc;
10439                 }
10440         }
10441         rc = bnxt_reserve_rings(bp, irq_re_init);
10442         if (rc)
10443                 return rc;
10444         if ((bp->flags & BNXT_FLAG_RFS) &&
10445             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
10446                 /* disable RFS if falling back to INTA */
10447                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
10448                 bp->flags &= ~BNXT_FLAG_RFS;
10449         }
10450
10451         rc = bnxt_alloc_mem(bp, irq_re_init);
10452         if (rc) {
10453                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10454                 goto open_err_free_mem;
10455         }
10456
10457         if (irq_re_init) {
10458                 bnxt_init_napi(bp);
10459                 rc = bnxt_request_irq(bp);
10460                 if (rc) {
10461                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
10462                         goto open_err_irq;
10463                 }
10464         }
10465
10466         rc = bnxt_init_nic(bp, irq_re_init);
10467         if (rc) {
10468                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10469                 goto open_err_irq;
10470         }
10471
10472         bnxt_enable_napi(bp);
10473         bnxt_debug_dev_init(bp);
10474
10475         if (link_re_init) {
10476                 mutex_lock(&bp->link_lock);
10477                 rc = bnxt_update_phy_setting(bp);
10478                 mutex_unlock(&bp->link_lock);
10479                 if (rc) {
10480                         netdev_warn(bp->dev, "failed to update phy settings\n");
10481                         if (BNXT_SINGLE_PF(bp)) {
10482                                 bp->link_info.phy_retry = true;
10483                                 bp->link_info.phy_retry_expires =
10484                                         jiffies + 5 * HZ;
10485                         }
10486                 }
10487         }
10488
10489         if (irq_re_init)
10490                 udp_tunnel_nic_reset_ntf(bp->dev);
10491
10492         if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
10493                 if (!static_key_enabled(&bnxt_xdp_locking_key))
10494                         static_branch_enable(&bnxt_xdp_locking_key);
10495         } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
10496                 static_branch_disable(&bnxt_xdp_locking_key);
10497         }
10498         set_bit(BNXT_STATE_OPEN, &bp->state);
10499         bnxt_enable_int(bp);
10500         /* Enable TX queues */
10501         bnxt_tx_enable(bp);
10502         mod_timer(&bp->timer, jiffies + bp->current_interval);
10503         /* Poll link status and check for SFP+ module status */
10504         mutex_lock(&bp->link_lock);
10505         bnxt_get_port_module_status(bp);
10506         mutex_unlock(&bp->link_lock);
10507
10508         /* VF-reps may need to be re-opened after the PF is re-opened */
10509         if (BNXT_PF(bp))
10510                 bnxt_vf_reps_open(bp);
10511         bnxt_ptp_init_rtc(bp, true);
10512         bnxt_ptp_cfg_tstamp_filters(bp);
10513         return 0;
10514
10515 open_err_irq:
10516         bnxt_del_napi(bp);
10517
10518 open_err_free_mem:
10519         bnxt_free_skbs(bp);
10520         bnxt_free_irq(bp);
10521         bnxt_free_mem(bp, true);
10522         return rc;
10523 }
10524
10525 /* rtnl_lock held */
10526 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10527 {
10528         int rc = 0;
10529
10530         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
10531                 rc = -EIO;
10532         if (!rc)
10533                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10534         if (rc) {
10535                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10536                 dev_close(bp->dev);
10537         }
10538         return rc;
10539 }
10540
10541 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10542  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10543  * self tests.
10544  */
10545 int bnxt_half_open_nic(struct bnxt *bp)
10546 {
10547         int rc = 0;
10548
10549         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10550                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
10551                 rc = -ENODEV;
10552                 goto half_open_err;
10553         }
10554
10555         rc = bnxt_alloc_mem(bp, true);
10556         if (rc) {
10557                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10558                 goto half_open_err;
10559         }
10560         set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10561         rc = bnxt_init_nic(bp, true);
10562         if (rc) {
10563                 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10564                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10565                 goto half_open_err;
10566         }
10567         return 0;
10568
10569 half_open_err:
10570         bnxt_free_skbs(bp);
10571         bnxt_free_mem(bp, true);
10572         dev_close(bp->dev);
10573         return rc;
10574 }
10575
10576 /* rtnl_lock held, this call can only be made after a previous successful
10577  * call to bnxt_half_open_nic().
10578  */
10579 void bnxt_half_close_nic(struct bnxt *bp)
10580 {
10581         bnxt_hwrm_resource_free(bp, false, true);
10582         bnxt_free_skbs(bp);
10583         bnxt_free_mem(bp, true);
10584         clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10585 }
10586
10587 void bnxt_reenable_sriov(struct bnxt *bp)
10588 {
10589         if (BNXT_PF(bp)) {
10590                 struct bnxt_pf_info *pf = &bp->pf;
10591                 int n = pf->active_vfs;
10592
10593                 if (n)
10594                         bnxt_cfg_hw_sriov(bp, &n, true);
10595         }
10596 }
10597
10598 static int bnxt_open(struct net_device *dev)
10599 {
10600         struct bnxt *bp = netdev_priv(dev);
10601         int rc;
10602
10603         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10604                 rc = bnxt_reinit_after_abort(bp);
10605                 if (rc) {
10606                         if (rc == -EBUSY)
10607                                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10608                         else
10609                                 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10610                         return -ENODEV;
10611                 }
10612         }
10613
10614         rc = bnxt_hwrm_if_change(bp, true);
10615         if (rc)
10616                 return rc;
10617
10618         rc = __bnxt_open_nic(bp, true, true);
10619         if (rc) {
10620                 bnxt_hwrm_if_change(bp, false);
10621         } else {
10622                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10623                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10624                                 bnxt_ulp_start(bp, 0);
10625                                 bnxt_reenable_sriov(bp);
10626                         }
10627                 }
10628                 bnxt_hwmon_open(bp);
10629         }
10630
10631         return rc;
10632 }
10633
10634 static bool bnxt_drv_busy(struct bnxt *bp)
10635 {
10636         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10637                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
10638 }
10639
10640 static void bnxt_get_ring_stats(struct bnxt *bp,
10641                                 struct rtnl_link_stats64 *stats);
10642
10643 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10644                              bool link_re_init)
10645 {
10646         /* Close the VF-reps before closing PF */
10647         if (BNXT_PF(bp))
10648                 bnxt_vf_reps_close(bp);
10649
10650         /* Change device state to avoid TX queue wake up's */
10651         bnxt_tx_disable(bp);
10652
10653         clear_bit(BNXT_STATE_OPEN, &bp->state);
10654         smp_mb__after_atomic();
10655         while (bnxt_drv_busy(bp))
10656                 msleep(20);
10657
10658         /* Flush rings and disable interrupts */
10659         bnxt_shutdown_nic(bp, irq_re_init);
10660
10661         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10662
10663         bnxt_debug_dev_exit(bp);
10664         bnxt_disable_napi(bp);
10665         del_timer_sync(&bp->timer);
10666         bnxt_free_skbs(bp);
10667
10668         /* Save ring stats before shutdown */
10669         if (bp->bnapi && irq_re_init)
10670                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10671         if (irq_re_init) {
10672                 bnxt_free_irq(bp);
10673                 bnxt_del_napi(bp);
10674         }
10675         bnxt_free_mem(bp, irq_re_init);
10676 }
10677
10678 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10679 {
10680         int rc = 0;
10681
10682         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10683                 /* If we get here, it means firmware reset is in progress
10684                  * while we are trying to close.  We can safely proceed with
10685                  * the close because we are holding rtnl_lock().  Some firmware
10686                  * messages may fail as we proceed to close.  We set the
10687                  * ABORT_ERR flag here so that the FW reset thread will later
10688                  * abort when it gets the rtnl_lock() and sees the flag.
10689                  */
10690                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10691                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10692         }
10693
10694 #ifdef CONFIG_BNXT_SRIOV
10695         if (bp->sriov_cfg) {
10696                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10697                                                       !bp->sriov_cfg,
10698                                                       BNXT_SRIOV_CFG_WAIT_TMO);
10699                 if (rc)
10700                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
10701         }
10702 #endif
10703         __bnxt_close_nic(bp, irq_re_init, link_re_init);
10704         return rc;
10705 }
10706
10707 static int bnxt_close(struct net_device *dev)
10708 {
10709         struct bnxt *bp = netdev_priv(dev);
10710
10711         bnxt_hwmon_close(bp);
10712         bnxt_close_nic(bp, true, true);
10713         bnxt_hwrm_shutdown_link(bp);
10714         bnxt_hwrm_if_change(bp, false);
10715         return 0;
10716 }
10717
10718 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10719                                    u16 *val)
10720 {
10721         struct hwrm_port_phy_mdio_read_output *resp;
10722         struct hwrm_port_phy_mdio_read_input *req;
10723         int rc;
10724
10725         if (bp->hwrm_spec_code < 0x10a00)
10726                 return -EOPNOTSUPP;
10727
10728         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
10729         if (rc)
10730                 return rc;
10731
10732         req->port_id = cpu_to_le16(bp->pf.port_id);
10733         req->phy_addr = phy_addr;
10734         req->reg_addr = cpu_to_le16(reg & 0x1f);
10735         if (mdio_phy_id_is_c45(phy_addr)) {
10736                 req->cl45_mdio = 1;
10737                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
10738                 req->dev_addr = mdio_phy_id_devad(phy_addr);
10739                 req->reg_addr = cpu_to_le16(reg);
10740         }
10741
10742         resp = hwrm_req_hold(bp, req);
10743         rc = hwrm_req_send(bp, req);
10744         if (!rc)
10745                 *val = le16_to_cpu(resp->reg_data);
10746         hwrm_req_drop(bp, req);
10747         return rc;
10748 }
10749
10750 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10751                                     u16 val)
10752 {
10753         struct hwrm_port_phy_mdio_write_input *req;
10754         int rc;
10755
10756         if (bp->hwrm_spec_code < 0x10a00)
10757                 return -EOPNOTSUPP;
10758
10759         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
10760         if (rc)
10761                 return rc;
10762
10763         req->port_id = cpu_to_le16(bp->pf.port_id);
10764         req->phy_addr = phy_addr;
10765         req->reg_addr = cpu_to_le16(reg & 0x1f);
10766         if (mdio_phy_id_is_c45(phy_addr)) {
10767                 req->cl45_mdio = 1;
10768                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
10769                 req->dev_addr = mdio_phy_id_devad(phy_addr);
10770                 req->reg_addr = cpu_to_le16(reg);
10771         }
10772         req->reg_data = cpu_to_le16(val);
10773
10774         return hwrm_req_send(bp, req);
10775 }
10776
10777 /* rtnl_lock held */
10778 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10779 {
10780         struct mii_ioctl_data *mdio = if_mii(ifr);
10781         struct bnxt *bp = netdev_priv(dev);
10782         int rc;
10783
10784         switch (cmd) {
10785         case SIOCGMIIPHY:
10786                 mdio->phy_id = bp->link_info.phy_addr;
10787
10788                 fallthrough;
10789         case SIOCGMIIREG: {
10790                 u16 mii_regval = 0;
10791
10792                 if (!netif_running(dev))
10793                         return -EAGAIN;
10794
10795                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10796                                              &mii_regval);
10797                 mdio->val_out = mii_regval;
10798                 return rc;
10799         }
10800
10801         case SIOCSMIIREG:
10802                 if (!netif_running(dev))
10803                         return -EAGAIN;
10804
10805                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10806                                                 mdio->val_in);
10807
10808         case SIOCSHWTSTAMP:
10809                 return bnxt_hwtstamp_set(dev, ifr);
10810
10811         case SIOCGHWTSTAMP:
10812                 return bnxt_hwtstamp_get(dev, ifr);
10813
10814         default:
10815                 /* do nothing */
10816                 break;
10817         }
10818         return -EOPNOTSUPP;
10819 }
10820
10821 static void bnxt_get_ring_stats(struct bnxt *bp,
10822                                 struct rtnl_link_stats64 *stats)
10823 {
10824         int i;
10825
10826         for (i = 0; i < bp->cp_nr_rings; i++) {
10827                 struct bnxt_napi *bnapi = bp->bnapi[i];
10828                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10829                 u64 *sw = cpr->stats.sw_stats;
10830
10831                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10832                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10833                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10834
10835                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10836                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10837                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10838
10839                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10840                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10841                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10842
10843                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10844                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10845                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10846
10847                 stats->rx_missed_errors +=
10848                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10849
10850                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10851
10852                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10853
10854                 stats->rx_dropped +=
10855                         cpr->sw_stats.rx.rx_netpoll_discards +
10856                         cpr->sw_stats.rx.rx_oom_discards;
10857         }
10858 }
10859
10860 static void bnxt_add_prev_stats(struct bnxt *bp,
10861                                 struct rtnl_link_stats64 *stats)
10862 {
10863         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10864
10865         stats->rx_packets += prev_stats->rx_packets;
10866         stats->tx_packets += prev_stats->tx_packets;
10867         stats->rx_bytes += prev_stats->rx_bytes;
10868         stats->tx_bytes += prev_stats->tx_bytes;
10869         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10870         stats->multicast += prev_stats->multicast;
10871         stats->rx_dropped += prev_stats->rx_dropped;
10872         stats->tx_dropped += prev_stats->tx_dropped;
10873 }
10874
10875 static void
10876 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10877 {
10878         struct bnxt *bp = netdev_priv(dev);
10879
10880         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10881         /* Make sure bnxt_close_nic() sees that we are reading stats before
10882          * we check the BNXT_STATE_OPEN flag.
10883          */
10884         smp_mb__after_atomic();
10885         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10886                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10887                 *stats = bp->net_stats_prev;
10888                 return;
10889         }
10890
10891         bnxt_get_ring_stats(bp, stats);
10892         bnxt_add_prev_stats(bp, stats);
10893
10894         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10895                 u64 *rx = bp->port_stats.sw_stats;
10896                 u64 *tx = bp->port_stats.sw_stats +
10897                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10898
10899                 stats->rx_crc_errors =
10900                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10901                 stats->rx_frame_errors =
10902                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10903                 stats->rx_length_errors =
10904                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10905                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10906                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10907                 stats->rx_errors =
10908                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10909                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10910                 stats->collisions =
10911                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10912                 stats->tx_fifo_errors =
10913                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10914                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10915         }
10916         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10917 }
10918
10919 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10920 {
10921         struct net_device *dev = bp->dev;
10922         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10923         struct netdev_hw_addr *ha;
10924         u8 *haddr;
10925         int mc_count = 0;
10926         bool update = false;
10927         int off = 0;
10928
10929         netdev_for_each_mc_addr(ha, dev) {
10930                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10931                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10932                         vnic->mc_list_count = 0;
10933                         return false;
10934                 }
10935                 haddr = ha->addr;
10936                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10937                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10938                         update = true;
10939                 }
10940                 off += ETH_ALEN;
10941                 mc_count++;
10942         }
10943         if (mc_count)
10944                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10945
10946         if (mc_count != vnic->mc_list_count) {
10947                 vnic->mc_list_count = mc_count;
10948                 update = true;
10949         }
10950         return update;
10951 }
10952
10953 static bool bnxt_uc_list_updated(struct bnxt *bp)
10954 {
10955         struct net_device *dev = bp->dev;
10956         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10957         struct netdev_hw_addr *ha;
10958         int off = 0;
10959
10960         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10961                 return true;
10962
10963         netdev_for_each_uc_addr(ha, dev) {
10964                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10965                         return true;
10966
10967                 off += ETH_ALEN;
10968         }
10969         return false;
10970 }
10971
10972 static void bnxt_set_rx_mode(struct net_device *dev)
10973 {
10974         struct bnxt *bp = netdev_priv(dev);
10975         struct bnxt_vnic_info *vnic;
10976         bool mc_update = false;
10977         bool uc_update;
10978         u32 mask;
10979
10980         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10981                 return;
10982
10983         vnic = &bp->vnic_info[0];
10984         mask = vnic->rx_mask;
10985         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10986                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10987                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10988                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10989
10990         if (dev->flags & IFF_PROMISC)
10991                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10992
10993         uc_update = bnxt_uc_list_updated(bp);
10994
10995         if (dev->flags & IFF_BROADCAST)
10996                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10997         if (dev->flags & IFF_ALLMULTI) {
10998                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10999                 vnic->mc_list_count = 0;
11000         } else if (dev->flags & IFF_MULTICAST) {
11001                 mc_update = bnxt_mc_list_updated(bp, &mask);
11002         }
11003
11004         if (mask != vnic->rx_mask || uc_update || mc_update) {
11005                 vnic->rx_mask = mask;
11006
11007                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
11008                 bnxt_queue_sp_work(bp);
11009         }
11010 }
11011
11012 static int bnxt_cfg_rx_mode(struct bnxt *bp)
11013 {
11014         struct net_device *dev = bp->dev;
11015         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11016         struct hwrm_cfa_l2_filter_free_input *req;
11017         struct netdev_hw_addr *ha;
11018         int i, off = 0, rc;
11019         bool uc_update;
11020
11021         netif_addr_lock_bh(dev);
11022         uc_update = bnxt_uc_list_updated(bp);
11023         netif_addr_unlock_bh(dev);
11024
11025         if (!uc_update)
11026                 goto skip_uc;
11027
11028         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
11029         if (rc)
11030                 return rc;
11031         hwrm_req_hold(bp, req);
11032         for (i = 1; i < vnic->uc_filter_count; i++) {
11033                 req->l2_filter_id = vnic->fw_l2_filter_id[i];
11034
11035                 rc = hwrm_req_send(bp, req);
11036         }
11037         hwrm_req_drop(bp, req);
11038
11039         vnic->uc_filter_count = 1;
11040
11041         netif_addr_lock_bh(dev);
11042         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
11043                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11044         } else {
11045                 netdev_for_each_uc_addr(ha, dev) {
11046                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
11047                         off += ETH_ALEN;
11048                         vnic->uc_filter_count++;
11049                 }
11050         }
11051         netif_addr_unlock_bh(dev);
11052
11053         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
11054                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
11055                 if (rc) {
11056                         if (BNXT_VF(bp) && rc == -ENODEV) {
11057                                 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11058                                         netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
11059                                 else
11060                                         netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
11061                                 rc = 0;
11062                         } else {
11063                                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
11064                         }
11065                         vnic->uc_filter_count = i;
11066                         return rc;
11067                 }
11068         }
11069         if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11070                 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
11071
11072 skip_uc:
11073         if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
11074             !bnxt_promisc_ok(bp))
11075                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11076         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11077         if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
11078                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
11079                             rc);
11080                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11081                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11082                 vnic->mc_list_count = 0;
11083                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11084         }
11085         if (rc)
11086                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
11087                            rc);
11088
11089         return rc;
11090 }
11091
11092 static bool bnxt_can_reserve_rings(struct bnxt *bp)
11093 {
11094 #ifdef CONFIG_BNXT_SRIOV
11095         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
11096                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11097
11098                 /* No minimum rings were provisioned by the PF.  Don't
11099                  * reserve rings by default when device is down.
11100                  */
11101                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
11102                         return true;
11103
11104                 if (!netif_running(bp->dev))
11105                         return false;
11106         }
11107 #endif
11108         return true;
11109 }
11110
11111 /* If the chip and firmware supports RFS */
11112 static bool bnxt_rfs_supported(struct bnxt *bp)
11113 {
11114         if (bp->flags & BNXT_FLAG_CHIP_P5) {
11115                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
11116                         return true;
11117                 return false;
11118         }
11119         /* 212 firmware is broken for aRFS */
11120         if (BNXT_FW_MAJ(bp) == 212)
11121                 return false;
11122         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
11123                 return true;
11124         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11125                 return true;
11126         return false;
11127 }
11128
11129 /* If runtime conditions support RFS */
11130 static bool bnxt_rfs_capable(struct bnxt *bp)
11131 {
11132 #ifdef CONFIG_RFS_ACCEL
11133         int vnics, max_vnics, max_rss_ctxs;
11134
11135         if (bp->flags & BNXT_FLAG_CHIP_P5)
11136                 return bnxt_rfs_supported(bp);
11137         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
11138                 return false;
11139
11140         vnics = 1 + bp->rx_nr_rings;
11141         max_vnics = bnxt_get_max_func_vnics(bp);
11142         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
11143
11144         /* RSS contexts not a limiting factor */
11145         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11146                 max_rss_ctxs = max_vnics;
11147         if (vnics > max_vnics || vnics > max_rss_ctxs) {
11148                 if (bp->rx_nr_rings > 1)
11149                         netdev_warn(bp->dev,
11150                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
11151                                     min(max_rss_ctxs - 1, max_vnics - 1));
11152                 return false;
11153         }
11154
11155         if (!BNXT_NEW_RM(bp))
11156                 return true;
11157
11158         if (vnics == bp->hw_resc.resv_vnics)
11159                 return true;
11160
11161         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
11162         if (vnics <= bp->hw_resc.resv_vnics)
11163                 return true;
11164
11165         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
11166         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
11167         return false;
11168 #else
11169         return false;
11170 #endif
11171 }
11172
11173 static netdev_features_t bnxt_fix_features(struct net_device *dev,
11174                                            netdev_features_t features)
11175 {
11176         struct bnxt *bp = netdev_priv(dev);
11177         netdev_features_t vlan_features;
11178
11179         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
11180                 features &= ~NETIF_F_NTUPLE;
11181
11182         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
11183                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11184
11185         if (!(bp->flags & BNXT_FLAG_TPA))
11186                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11187
11188         if (!(features & NETIF_F_GRO))
11189                 features &= ~NETIF_F_GRO_HW;
11190
11191         if (features & NETIF_F_GRO_HW)
11192                 features &= ~NETIF_F_LRO;
11193
11194         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
11195          * turned on or off together.
11196          */
11197         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
11198         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
11199                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11200                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11201                 else if (vlan_features)
11202                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
11203         }
11204 #ifdef CONFIG_BNXT_SRIOV
11205         if (BNXT_VF(bp) && bp->vf.vlan)
11206                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11207 #endif
11208         return features;
11209 }
11210
11211 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
11212 {
11213         struct bnxt *bp = netdev_priv(dev);
11214         u32 flags = bp->flags;
11215         u32 changes;
11216         int rc = 0;
11217         bool re_init = false;
11218         bool update_tpa = false;
11219
11220         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
11221         if (features & NETIF_F_GRO_HW)
11222                 flags |= BNXT_FLAG_GRO;
11223         else if (features & NETIF_F_LRO)
11224                 flags |= BNXT_FLAG_LRO;
11225
11226         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
11227                 flags &= ~BNXT_FLAG_TPA;
11228
11229         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11230                 flags |= BNXT_FLAG_STRIP_VLAN;
11231
11232         if (features & NETIF_F_NTUPLE)
11233                 flags |= BNXT_FLAG_RFS;
11234
11235         changes = flags ^ bp->flags;
11236         if (changes & BNXT_FLAG_TPA) {
11237                 update_tpa = true;
11238                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
11239                     (flags & BNXT_FLAG_TPA) == 0 ||
11240                     (bp->flags & BNXT_FLAG_CHIP_P5))
11241                         re_init = true;
11242         }
11243
11244         if (changes & ~BNXT_FLAG_TPA)
11245                 re_init = true;
11246
11247         if (flags != bp->flags) {
11248                 u32 old_flags = bp->flags;
11249
11250                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11251                         bp->flags = flags;
11252                         if (update_tpa)
11253                                 bnxt_set_ring_params(bp);
11254                         return rc;
11255                 }
11256
11257                 if (re_init) {
11258                         bnxt_close_nic(bp, false, false);
11259                         bp->flags = flags;
11260                         if (update_tpa)
11261                                 bnxt_set_ring_params(bp);
11262
11263                         return bnxt_open_nic(bp, false, false);
11264                 }
11265                 if (update_tpa) {
11266                         bp->flags = flags;
11267                         rc = bnxt_set_tpa(bp,
11268                                           (flags & BNXT_FLAG_TPA) ?
11269                                           true : false);
11270                         if (rc)
11271                                 bp->flags = old_flags;
11272                 }
11273         }
11274         return rc;
11275 }
11276
11277 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
11278                               u8 **nextp)
11279 {
11280         struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
11281         int hdr_count = 0;
11282         u8 *nexthdr;
11283         int start;
11284
11285         /* Check that there are at most 2 IPv6 extension headers, no
11286          * fragment header, and each is <= 64 bytes.
11287          */
11288         start = nw_off + sizeof(*ip6h);
11289         nexthdr = &ip6h->nexthdr;
11290         while (ipv6_ext_hdr(*nexthdr)) {
11291                 struct ipv6_opt_hdr *hp;
11292                 int hdrlen;
11293
11294                 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
11295                     *nexthdr == NEXTHDR_FRAGMENT)
11296                         return false;
11297                 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
11298                                           skb_headlen(skb), NULL);
11299                 if (!hp)
11300                         return false;
11301                 if (*nexthdr == NEXTHDR_AUTH)
11302                         hdrlen = ipv6_authlen(hp);
11303                 else
11304                         hdrlen = ipv6_optlen(hp);
11305
11306                 if (hdrlen > 64)
11307                         return false;
11308                 nexthdr = &hp->nexthdr;
11309                 start += hdrlen;
11310                 hdr_count++;
11311         }
11312         if (nextp) {
11313                 /* Caller will check inner protocol */
11314                 if (skb->encapsulation) {
11315                         *nextp = nexthdr;
11316                         return true;
11317                 }
11318                 *nextp = NULL;
11319         }
11320         /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
11321         return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
11322 }
11323
11324 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
11325 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
11326 {
11327         struct udphdr *uh = udp_hdr(skb);
11328         __be16 udp_port = uh->dest;
11329
11330         if (udp_port != bp->vxlan_port && udp_port != bp->nge_port)
11331                 return false;
11332         if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) {
11333                 struct ethhdr *eh = inner_eth_hdr(skb);
11334
11335                 switch (eh->h_proto) {
11336                 case htons(ETH_P_IP):
11337                         return true;
11338                 case htons(ETH_P_IPV6):
11339                         return bnxt_exthdr_check(bp, skb,
11340                                                  skb_inner_network_offset(skb),
11341                                                  NULL);
11342                 }
11343         }
11344         return false;
11345 }
11346
11347 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
11348 {
11349         switch (l4_proto) {
11350         case IPPROTO_UDP:
11351                 return bnxt_udp_tunl_check(bp, skb);
11352         case IPPROTO_IPIP:
11353                 return true;
11354         case IPPROTO_GRE: {
11355                 switch (skb->inner_protocol) {
11356                 default:
11357                         return false;
11358                 case htons(ETH_P_IP):
11359                         return true;
11360                 case htons(ETH_P_IPV6):
11361                         fallthrough;
11362                 }
11363         }
11364         case IPPROTO_IPV6:
11365                 /* Check ext headers of inner ipv6 */
11366                 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
11367                                          NULL);
11368         }
11369         return false;
11370 }
11371
11372 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
11373                                              struct net_device *dev,
11374                                              netdev_features_t features)
11375 {
11376         struct bnxt *bp = netdev_priv(dev);
11377         u8 *l4_proto;
11378
11379         features = vlan_features_check(skb, features);
11380         switch (vlan_get_protocol(skb)) {
11381         case htons(ETH_P_IP):
11382                 if (!skb->encapsulation)
11383                         return features;
11384                 l4_proto = &ip_hdr(skb)->protocol;
11385                 if (bnxt_tunl_check(bp, skb, *l4_proto))
11386                         return features;
11387                 break;
11388         case htons(ETH_P_IPV6):
11389                 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
11390                                        &l4_proto))
11391                         break;
11392                 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
11393                         return features;
11394                 break;
11395         }
11396         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11397 }
11398
11399 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
11400                          u32 *reg_buf)
11401 {
11402         struct hwrm_dbg_read_direct_output *resp;
11403         struct hwrm_dbg_read_direct_input *req;
11404         __le32 *dbg_reg_buf;
11405         dma_addr_t mapping;
11406         int rc, i;
11407
11408         rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
11409         if (rc)
11410                 return rc;
11411
11412         dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
11413                                          &mapping);
11414         if (!dbg_reg_buf) {
11415                 rc = -ENOMEM;
11416                 goto dbg_rd_reg_exit;
11417         }
11418
11419         req->host_dest_addr = cpu_to_le64(mapping);
11420
11421         resp = hwrm_req_hold(bp, req);
11422         req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
11423         req->read_len32 = cpu_to_le32(num_words);
11424
11425         rc = hwrm_req_send(bp, req);
11426         if (rc || resp->error_code) {
11427                 rc = -EIO;
11428                 goto dbg_rd_reg_exit;
11429         }
11430         for (i = 0; i < num_words; i++)
11431                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
11432
11433 dbg_rd_reg_exit:
11434         hwrm_req_drop(bp, req);
11435         return rc;
11436 }
11437
11438 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
11439                                        u32 ring_id, u32 *prod, u32 *cons)
11440 {
11441         struct hwrm_dbg_ring_info_get_output *resp;
11442         struct hwrm_dbg_ring_info_get_input *req;
11443         int rc;
11444
11445         rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
11446         if (rc)
11447                 return rc;
11448
11449         req->ring_type = ring_type;
11450         req->fw_ring_id = cpu_to_le32(ring_id);
11451         resp = hwrm_req_hold(bp, req);
11452         rc = hwrm_req_send(bp, req);
11453         if (!rc) {
11454                 *prod = le32_to_cpu(resp->producer_index);
11455                 *cons = le32_to_cpu(resp->consumer_index);
11456         }
11457         hwrm_req_drop(bp, req);
11458         return rc;
11459 }
11460
11461 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
11462 {
11463         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
11464         int i = bnapi->index;
11465
11466         if (!txr)
11467                 return;
11468
11469         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
11470                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
11471                     txr->tx_cons);
11472 }
11473
11474 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
11475 {
11476         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
11477         int i = bnapi->index;
11478
11479         if (!rxr)
11480                 return;
11481
11482         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
11483                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
11484                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
11485                     rxr->rx_sw_agg_prod);
11486 }
11487
11488 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
11489 {
11490         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11491         int i = bnapi->index;
11492
11493         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
11494                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
11495 }
11496
11497 static void bnxt_dbg_dump_states(struct bnxt *bp)
11498 {
11499         int i;
11500         struct bnxt_napi *bnapi;
11501
11502         for (i = 0; i < bp->cp_nr_rings; i++) {
11503                 bnapi = bp->bnapi[i];
11504                 if (netif_msg_drv(bp)) {
11505                         bnxt_dump_tx_sw_state(bnapi);
11506                         bnxt_dump_rx_sw_state(bnapi);
11507                         bnxt_dump_cp_sw_state(bnapi);
11508                 }
11509         }
11510 }
11511
11512 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
11513 {
11514         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
11515         struct hwrm_ring_reset_input *req;
11516         struct bnxt_napi *bnapi = rxr->bnapi;
11517         struct bnxt_cp_ring_info *cpr;
11518         u16 cp_ring_id;
11519         int rc;
11520
11521         rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
11522         if (rc)
11523                 return rc;
11524
11525         cpr = &bnapi->cp_ring;
11526         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
11527         req->cmpl_ring = cpu_to_le16(cp_ring_id);
11528         req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
11529         req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
11530         return hwrm_req_send_silent(bp, req);
11531 }
11532
11533 static void bnxt_reset_task(struct bnxt *bp, bool silent)
11534 {
11535         if (!silent)
11536                 bnxt_dbg_dump_states(bp);
11537         if (netif_running(bp->dev)) {
11538                 int rc;
11539
11540                 if (silent) {
11541                         bnxt_close_nic(bp, false, false);
11542                         bnxt_open_nic(bp, false, false);
11543                 } else {
11544                         bnxt_ulp_stop(bp);
11545                         bnxt_close_nic(bp, true, false);
11546                         rc = bnxt_open_nic(bp, true, false);
11547                         bnxt_ulp_start(bp, rc);
11548                 }
11549         }
11550 }
11551
11552 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
11553 {
11554         struct bnxt *bp = netdev_priv(dev);
11555
11556         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
11557         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
11558         bnxt_queue_sp_work(bp);
11559 }
11560
11561 static void bnxt_fw_health_check(struct bnxt *bp)
11562 {
11563         struct bnxt_fw_health *fw_health = bp->fw_health;
11564         u32 val;
11565
11566         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11567                 return;
11568
11569         /* Make sure it is enabled before checking the tmr_counter. */
11570         smp_rmb();
11571         if (fw_health->tmr_counter) {
11572                 fw_health->tmr_counter--;
11573                 return;
11574         }
11575
11576         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11577         if (val == fw_health->last_fw_heartbeat) {
11578                 fw_health->arrests++;
11579                 goto fw_reset;
11580         }
11581
11582         fw_health->last_fw_heartbeat = val;
11583
11584         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11585         if (val != fw_health->last_fw_reset_cnt) {
11586                 fw_health->discoveries++;
11587                 goto fw_reset;
11588         }
11589
11590         fw_health->tmr_counter = fw_health->tmr_multiplier;
11591         return;
11592
11593 fw_reset:
11594         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
11595         bnxt_queue_sp_work(bp);
11596 }
11597
11598 static void bnxt_timer(struct timer_list *t)
11599 {
11600         struct bnxt *bp = from_timer(bp, t, timer);
11601         struct net_device *dev = bp->dev;
11602
11603         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
11604                 return;
11605
11606         if (atomic_read(&bp->intr_sem) != 0)
11607                 goto bnxt_restart_timer;
11608
11609         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
11610                 bnxt_fw_health_check(bp);
11611
11612         if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) {
11613                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
11614                 bnxt_queue_sp_work(bp);
11615         }
11616
11617         if (bnxt_tc_flower_enabled(bp)) {
11618                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
11619                 bnxt_queue_sp_work(bp);
11620         }
11621
11622 #ifdef CONFIG_RFS_ACCEL
11623         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
11624                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11625                 bnxt_queue_sp_work(bp);
11626         }
11627 #endif /*CONFIG_RFS_ACCEL*/
11628
11629         if (bp->link_info.phy_retry) {
11630                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
11631                         bp->link_info.phy_retry = false;
11632                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
11633                 } else {
11634                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
11635                         bnxt_queue_sp_work(bp);
11636                 }
11637         }
11638
11639         if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) {
11640                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
11641                 bnxt_queue_sp_work(bp);
11642         }
11643
11644         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
11645             netif_carrier_ok(dev)) {
11646                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
11647                 bnxt_queue_sp_work(bp);
11648         }
11649 bnxt_restart_timer:
11650         mod_timer(&bp->timer, jiffies + bp->current_interval);
11651 }
11652
11653 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
11654 {
11655         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
11656          * set.  If the device is being closed, bnxt_close() may be holding
11657          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
11658          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
11659          */
11660         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11661         rtnl_lock();
11662 }
11663
11664 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
11665 {
11666         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11667         rtnl_unlock();
11668 }
11669
11670 /* Only called from bnxt_sp_task() */
11671 static void bnxt_reset(struct bnxt *bp, bool silent)
11672 {
11673         bnxt_rtnl_lock_sp(bp);
11674         if (test_bit(BNXT_STATE_OPEN, &bp->state))
11675                 bnxt_reset_task(bp, silent);
11676         bnxt_rtnl_unlock_sp(bp);
11677 }
11678
11679 /* Only called from bnxt_sp_task() */
11680 static void bnxt_rx_ring_reset(struct bnxt *bp)
11681 {
11682         int i;
11683
11684         bnxt_rtnl_lock_sp(bp);
11685         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11686                 bnxt_rtnl_unlock_sp(bp);
11687                 return;
11688         }
11689         /* Disable and flush TPA before resetting the RX ring */
11690         if (bp->flags & BNXT_FLAG_TPA)
11691                 bnxt_set_tpa(bp, false);
11692         for (i = 0; i < bp->rx_nr_rings; i++) {
11693                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
11694                 struct bnxt_cp_ring_info *cpr;
11695                 int rc;
11696
11697                 if (!rxr->bnapi->in_reset)
11698                         continue;
11699
11700                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
11701                 if (rc) {
11702                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
11703                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
11704                         else
11705                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
11706                                             rc);
11707                         bnxt_reset_task(bp, true);
11708                         break;
11709                 }
11710                 bnxt_free_one_rx_ring_skbs(bp, i);
11711                 rxr->rx_prod = 0;
11712                 rxr->rx_agg_prod = 0;
11713                 rxr->rx_sw_agg_prod = 0;
11714                 rxr->rx_next_cons = 0;
11715                 rxr->bnapi->in_reset = false;
11716                 bnxt_alloc_one_rx_ring(bp, i);
11717                 cpr = &rxr->bnapi->cp_ring;
11718                 cpr->sw_stats.rx.rx_resets++;
11719                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
11720                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
11721                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
11722         }
11723         if (bp->flags & BNXT_FLAG_TPA)
11724                 bnxt_set_tpa(bp, true);
11725         bnxt_rtnl_unlock_sp(bp);
11726 }
11727
11728 static void bnxt_fw_reset_close(struct bnxt *bp)
11729 {
11730         bnxt_ulp_stop(bp);
11731         /* When firmware is in fatal state, quiesce device and disable
11732          * bus master to prevent any potential bad DMAs before freeing
11733          * kernel memory.
11734          */
11735         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11736                 u16 val = 0;
11737
11738                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11739                 if (val == 0xffff)
11740                         bp->fw_reset_min_dsecs = 0;
11741                 bnxt_tx_disable(bp);
11742                 bnxt_disable_napi(bp);
11743                 bnxt_disable_int_sync(bp);
11744                 bnxt_free_irq(bp);
11745                 bnxt_clear_int_mode(bp);
11746                 pci_disable_device(bp->pdev);
11747         }
11748         __bnxt_close_nic(bp, true, false);
11749         bnxt_vf_reps_free(bp);
11750         bnxt_clear_int_mode(bp);
11751         bnxt_hwrm_func_drv_unrgtr(bp);
11752         if (pci_is_enabled(bp->pdev))
11753                 pci_disable_device(bp->pdev);
11754         bnxt_free_ctx_mem(bp);
11755         kfree(bp->ctx);
11756         bp->ctx = NULL;
11757 }
11758
11759 static bool is_bnxt_fw_ok(struct bnxt *bp)
11760 {
11761         struct bnxt_fw_health *fw_health = bp->fw_health;
11762         bool no_heartbeat = false, has_reset = false;
11763         u32 val;
11764
11765         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11766         if (val == fw_health->last_fw_heartbeat)
11767                 no_heartbeat = true;
11768
11769         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11770         if (val != fw_health->last_fw_reset_cnt)
11771                 has_reset = true;
11772
11773         if (!no_heartbeat && has_reset)
11774                 return true;
11775
11776         return false;
11777 }
11778
11779 /* rtnl_lock is acquired before calling this function */
11780 static void bnxt_force_fw_reset(struct bnxt *bp)
11781 {
11782         struct bnxt_fw_health *fw_health = bp->fw_health;
11783         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11784         u32 wait_dsecs;
11785
11786         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11787             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11788                 return;
11789
11790         if (ptp) {
11791                 spin_lock_bh(&ptp->ptp_lock);
11792                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11793                 spin_unlock_bh(&ptp->ptp_lock);
11794         } else {
11795                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11796         }
11797         bnxt_fw_reset_close(bp);
11798         wait_dsecs = fw_health->master_func_wait_dsecs;
11799         if (fw_health->primary) {
11800                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11801                         wait_dsecs = 0;
11802                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11803         } else {
11804                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11805                 wait_dsecs = fw_health->normal_func_wait_dsecs;
11806                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11807         }
11808
11809         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11810         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11811         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11812 }
11813
11814 void bnxt_fw_exception(struct bnxt *bp)
11815 {
11816         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11817         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11818         bnxt_rtnl_lock_sp(bp);
11819         bnxt_force_fw_reset(bp);
11820         bnxt_rtnl_unlock_sp(bp);
11821 }
11822
11823 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11824  * < 0 on error.
11825  */
11826 static int bnxt_get_registered_vfs(struct bnxt *bp)
11827 {
11828 #ifdef CONFIG_BNXT_SRIOV
11829         int rc;
11830
11831         if (!BNXT_PF(bp))
11832                 return 0;
11833
11834         rc = bnxt_hwrm_func_qcfg(bp);
11835         if (rc) {
11836                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11837                 return rc;
11838         }
11839         if (bp->pf.registered_vfs)
11840                 return bp->pf.registered_vfs;
11841         if (bp->sriov_cfg)
11842                 return 1;
11843 #endif
11844         return 0;
11845 }
11846
11847 void bnxt_fw_reset(struct bnxt *bp)
11848 {
11849         bnxt_rtnl_lock_sp(bp);
11850         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11851             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11852                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11853                 int n = 0, tmo;
11854
11855                 if (ptp) {
11856                         spin_lock_bh(&ptp->ptp_lock);
11857                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11858                         spin_unlock_bh(&ptp->ptp_lock);
11859                 } else {
11860                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11861                 }
11862                 if (bp->pf.active_vfs &&
11863                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11864                         n = bnxt_get_registered_vfs(bp);
11865                 if (n < 0) {
11866                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11867                                    n);
11868                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11869                         dev_close(bp->dev);
11870                         goto fw_reset_exit;
11871                 } else if (n > 0) {
11872                         u16 vf_tmo_dsecs = n * 10;
11873
11874                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11875                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11876                         bp->fw_reset_state =
11877                                 BNXT_FW_RESET_STATE_POLL_VF;
11878                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11879                         goto fw_reset_exit;
11880                 }
11881                 bnxt_fw_reset_close(bp);
11882                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11883                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11884                         tmo = HZ / 10;
11885                 } else {
11886                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11887                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11888                 }
11889                 bnxt_queue_fw_reset_work(bp, tmo);
11890         }
11891 fw_reset_exit:
11892         bnxt_rtnl_unlock_sp(bp);
11893 }
11894
11895 static void bnxt_chk_missed_irq(struct bnxt *bp)
11896 {
11897         int i;
11898
11899         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11900                 return;
11901
11902         for (i = 0; i < bp->cp_nr_rings; i++) {
11903                 struct bnxt_napi *bnapi = bp->bnapi[i];
11904                 struct bnxt_cp_ring_info *cpr;
11905                 u32 fw_ring_id;
11906                 int j;
11907
11908                 if (!bnapi)
11909                         continue;
11910
11911                 cpr = &bnapi->cp_ring;
11912                 for (j = 0; j < 2; j++) {
11913                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11914                         u32 val[2];
11915
11916                         if (!cpr2 || cpr2->has_more_work ||
11917                             !bnxt_has_work(bp, cpr2))
11918                                 continue;
11919
11920                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
11921                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
11922                                 continue;
11923                         }
11924                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
11925                         bnxt_dbg_hwrm_ring_info_get(bp,
11926                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
11927                                 fw_ring_id, &val[0], &val[1]);
11928                         cpr->sw_stats.cmn.missed_irqs++;
11929                 }
11930         }
11931 }
11932
11933 static void bnxt_cfg_ntp_filters(struct bnxt *);
11934
11935 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
11936 {
11937         struct bnxt_link_info *link_info = &bp->link_info;
11938
11939         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
11940                 link_info->autoneg = BNXT_AUTONEG_SPEED;
11941                 if (bp->hwrm_spec_code >= 0x10201) {
11942                         if (link_info->auto_pause_setting &
11943                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
11944                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11945                 } else {
11946                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11947                 }
11948                 link_info->advertising = link_info->auto_link_speeds;
11949                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
11950         } else {
11951                 link_info->req_link_speed = link_info->force_link_speed;
11952                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
11953                 if (link_info->force_pam4_link_speed) {
11954                         link_info->req_link_speed =
11955                                 link_info->force_pam4_link_speed;
11956                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
11957                 }
11958                 link_info->req_duplex = link_info->duplex_setting;
11959         }
11960         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
11961                 link_info->req_flow_ctrl =
11962                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
11963         else
11964                 link_info->req_flow_ctrl = link_info->force_pause_setting;
11965 }
11966
11967 static void bnxt_fw_echo_reply(struct bnxt *bp)
11968 {
11969         struct bnxt_fw_health *fw_health = bp->fw_health;
11970         struct hwrm_func_echo_response_input *req;
11971         int rc;
11972
11973         rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
11974         if (rc)
11975                 return;
11976         req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
11977         req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
11978         hwrm_req_send(bp, req);
11979 }
11980
11981 static void bnxt_sp_task(struct work_struct *work)
11982 {
11983         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
11984
11985         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11986         smp_mb__after_atomic();
11987         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11988                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11989                 return;
11990         }
11991
11992         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11993                 bnxt_cfg_rx_mode(bp);
11994
11995         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11996                 bnxt_cfg_ntp_filters(bp);
11997         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11998                 bnxt_hwrm_exec_fwd_req(bp);
11999         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
12000                 bnxt_hwrm_port_qstats(bp, 0);
12001                 bnxt_hwrm_port_qstats_ext(bp, 0);
12002                 bnxt_accumulate_all_stats(bp);
12003         }
12004
12005         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
12006                 int rc;
12007
12008                 mutex_lock(&bp->link_lock);
12009                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
12010                                        &bp->sp_event))
12011                         bnxt_hwrm_phy_qcaps(bp);
12012
12013                 rc = bnxt_update_link(bp, true);
12014                 if (rc)
12015                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
12016                                    rc);
12017
12018                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
12019                                        &bp->sp_event))
12020                         bnxt_init_ethtool_link_settings(bp);
12021                 mutex_unlock(&bp->link_lock);
12022         }
12023         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
12024                 int rc;
12025
12026                 mutex_lock(&bp->link_lock);
12027                 rc = bnxt_update_phy_setting(bp);
12028                 mutex_unlock(&bp->link_lock);
12029                 if (rc) {
12030                         netdev_warn(bp->dev, "update phy settings retry failed\n");
12031                 } else {
12032                         bp->link_info.phy_retry = false;
12033                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
12034                 }
12035         }
12036         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
12037                 mutex_lock(&bp->link_lock);
12038                 bnxt_get_port_module_status(bp);
12039                 mutex_unlock(&bp->link_lock);
12040         }
12041
12042         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
12043                 bnxt_tc_flow_stats_work(bp);
12044
12045         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
12046                 bnxt_chk_missed_irq(bp);
12047
12048         if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
12049                 bnxt_fw_echo_reply(bp);
12050
12051         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
12052          * must be the last functions to be called before exiting.
12053          */
12054         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
12055                 bnxt_reset(bp, false);
12056
12057         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
12058                 bnxt_reset(bp, true);
12059
12060         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
12061                 bnxt_rx_ring_reset(bp);
12062
12063         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
12064                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
12065                     test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
12066                         bnxt_devlink_health_fw_report(bp);
12067                 else
12068                         bnxt_fw_reset(bp);
12069         }
12070
12071         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
12072                 if (!is_bnxt_fw_ok(bp))
12073                         bnxt_devlink_health_fw_report(bp);
12074         }
12075
12076         smp_mb__before_atomic();
12077         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12078 }
12079
12080 /* Under rtnl_lock */
12081 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
12082                      int tx_xdp)
12083 {
12084         int max_rx, max_tx, tx_sets = 1;
12085         int tx_rings_needed, stats;
12086         int rx_rings = rx;
12087         int cp, vnics, rc;
12088
12089         if (tcs)
12090                 tx_sets = tcs;
12091
12092         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
12093         if (rc)
12094                 return rc;
12095
12096         if (max_rx < rx)
12097                 return -ENOMEM;
12098
12099         tx_rings_needed = tx * tx_sets + tx_xdp;
12100         if (max_tx < tx_rings_needed)
12101                 return -ENOMEM;
12102
12103         vnics = 1;
12104         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
12105                 vnics += rx_rings;
12106
12107         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12108                 rx_rings <<= 1;
12109         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
12110         stats = cp;
12111         if (BNXT_NEW_RM(bp)) {
12112                 cp += bnxt_get_ulp_msix_num(bp);
12113                 stats += bnxt_get_ulp_stat_ctxs(bp);
12114         }
12115         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
12116                                      stats, vnics);
12117 }
12118
12119 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
12120 {
12121         if (bp->bar2) {
12122                 pci_iounmap(pdev, bp->bar2);
12123                 bp->bar2 = NULL;
12124         }
12125
12126         if (bp->bar1) {
12127                 pci_iounmap(pdev, bp->bar1);
12128                 bp->bar1 = NULL;
12129         }
12130
12131         if (bp->bar0) {
12132                 pci_iounmap(pdev, bp->bar0);
12133                 bp->bar0 = NULL;
12134         }
12135 }
12136
12137 static void bnxt_cleanup_pci(struct bnxt *bp)
12138 {
12139         bnxt_unmap_bars(bp, bp->pdev);
12140         pci_release_regions(bp->pdev);
12141         if (pci_is_enabled(bp->pdev))
12142                 pci_disable_device(bp->pdev);
12143 }
12144
12145 static void bnxt_init_dflt_coal(struct bnxt *bp)
12146 {
12147         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
12148         struct bnxt_coal *coal;
12149         u16 flags = 0;
12150
12151         if (coal_cap->cmpl_params &
12152             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
12153                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
12154
12155         /* Tick values in micro seconds.
12156          * 1 coal_buf x bufs_per_record = 1 completion record.
12157          */
12158         coal = &bp->rx_coal;
12159         coal->coal_ticks = 10;
12160         coal->coal_bufs = 30;
12161         coal->coal_ticks_irq = 1;
12162         coal->coal_bufs_irq = 2;
12163         coal->idle_thresh = 50;
12164         coal->bufs_per_record = 2;
12165         coal->budget = 64;              /* NAPI budget */
12166         coal->flags = flags;
12167
12168         coal = &bp->tx_coal;
12169         coal->coal_ticks = 28;
12170         coal->coal_bufs = 30;
12171         coal->coal_ticks_irq = 2;
12172         coal->coal_bufs_irq = 2;
12173         coal->bufs_per_record = 1;
12174         coal->flags = flags;
12175
12176         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
12177 }
12178
12179 static int bnxt_fw_init_one_p1(struct bnxt *bp)
12180 {
12181         int rc;
12182
12183         bp->fw_cap = 0;
12184         rc = bnxt_hwrm_ver_get(bp);
12185         bnxt_try_map_fw_health_reg(bp);
12186         if (rc) {
12187                 rc = bnxt_try_recover_fw(bp);
12188                 if (rc)
12189                         return rc;
12190                 rc = bnxt_hwrm_ver_get(bp);
12191                 if (rc)
12192                         return rc;
12193         }
12194
12195         bnxt_nvm_cfg_ver_get(bp);
12196
12197         rc = bnxt_hwrm_func_reset(bp);
12198         if (rc)
12199                 return -ENODEV;
12200
12201         bnxt_hwrm_fw_set_time(bp);
12202         return 0;
12203 }
12204
12205 static int bnxt_fw_init_one_p2(struct bnxt *bp)
12206 {
12207         int rc;
12208
12209         /* Get the MAX capabilities for this function */
12210         rc = bnxt_hwrm_func_qcaps(bp);
12211         if (rc) {
12212                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
12213                            rc);
12214                 return -ENODEV;
12215         }
12216
12217         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
12218         if (rc)
12219                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
12220                             rc);
12221
12222         if (bnxt_alloc_fw_health(bp)) {
12223                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
12224         } else {
12225                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
12226                 if (rc)
12227                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
12228                                     rc);
12229         }
12230
12231         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
12232         if (rc)
12233                 return -ENODEV;
12234
12235         bnxt_hwrm_func_qcfg(bp);
12236         bnxt_hwrm_vnic_qcaps(bp);
12237         bnxt_hwrm_port_led_qcaps(bp);
12238         bnxt_ethtool_init(bp);
12239         bnxt_dcb_init(bp);
12240         return 0;
12241 }
12242
12243 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
12244 {
12245         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
12246         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
12247                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
12248                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
12249                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
12250         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
12251                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
12252                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
12253                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
12254         }
12255 }
12256
12257 static void bnxt_set_dflt_rfs(struct bnxt *bp)
12258 {
12259         struct net_device *dev = bp->dev;
12260
12261         dev->hw_features &= ~NETIF_F_NTUPLE;
12262         dev->features &= ~NETIF_F_NTUPLE;
12263         bp->flags &= ~BNXT_FLAG_RFS;
12264         if (bnxt_rfs_supported(bp)) {
12265                 dev->hw_features |= NETIF_F_NTUPLE;
12266                 if (bnxt_rfs_capable(bp)) {
12267                         bp->flags |= BNXT_FLAG_RFS;
12268                         dev->features |= NETIF_F_NTUPLE;
12269                 }
12270         }
12271 }
12272
12273 static void bnxt_fw_init_one_p3(struct bnxt *bp)
12274 {
12275         struct pci_dev *pdev = bp->pdev;
12276
12277         bnxt_set_dflt_rss_hash_type(bp);
12278         bnxt_set_dflt_rfs(bp);
12279
12280         bnxt_get_wol_settings(bp);
12281         if (bp->flags & BNXT_FLAG_WOL_CAP)
12282                 device_set_wakeup_enable(&pdev->dev, bp->wol);
12283         else
12284                 device_set_wakeup_capable(&pdev->dev, false);
12285
12286         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
12287         bnxt_hwrm_coal_params_qcaps(bp);
12288 }
12289
12290 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
12291
12292 int bnxt_fw_init_one(struct bnxt *bp)
12293 {
12294         int rc;
12295
12296         rc = bnxt_fw_init_one_p1(bp);
12297         if (rc) {
12298                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
12299                 return rc;
12300         }
12301         rc = bnxt_fw_init_one_p2(bp);
12302         if (rc) {
12303                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
12304                 return rc;
12305         }
12306         rc = bnxt_probe_phy(bp, false);
12307         if (rc)
12308                 return rc;
12309         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
12310         if (rc)
12311                 return rc;
12312
12313         bnxt_fw_init_one_p3(bp);
12314         return 0;
12315 }
12316
12317 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
12318 {
12319         struct bnxt_fw_health *fw_health = bp->fw_health;
12320         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
12321         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
12322         u32 reg_type, reg_off, delay_msecs;
12323
12324         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
12325         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
12326         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
12327         switch (reg_type) {
12328         case BNXT_FW_HEALTH_REG_TYPE_CFG:
12329                 pci_write_config_dword(bp->pdev, reg_off, val);
12330                 break;
12331         case BNXT_FW_HEALTH_REG_TYPE_GRC:
12332                 writel(reg_off & BNXT_GRC_BASE_MASK,
12333                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
12334                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
12335                 fallthrough;
12336         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
12337                 writel(val, bp->bar0 + reg_off);
12338                 break;
12339         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
12340                 writel(val, bp->bar1 + reg_off);
12341                 break;
12342         }
12343         if (delay_msecs) {
12344                 pci_read_config_dword(bp->pdev, 0, &val);
12345                 msleep(delay_msecs);
12346         }
12347 }
12348
12349 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
12350 {
12351         struct hwrm_func_qcfg_output *resp;
12352         struct hwrm_func_qcfg_input *req;
12353         bool result = true; /* firmware will enforce if unknown */
12354
12355         if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
12356                 return result;
12357
12358         if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
12359                 return result;
12360
12361         req->fid = cpu_to_le16(0xffff);
12362         resp = hwrm_req_hold(bp, req);
12363         if (!hwrm_req_send(bp, req))
12364                 result = !!(le16_to_cpu(resp->flags) &
12365                             FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
12366         hwrm_req_drop(bp, req);
12367         return result;
12368 }
12369
12370 static void bnxt_reset_all(struct bnxt *bp)
12371 {
12372         struct bnxt_fw_health *fw_health = bp->fw_health;
12373         int i, rc;
12374
12375         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12376                 bnxt_fw_reset_via_optee(bp);
12377                 bp->fw_reset_timestamp = jiffies;
12378                 return;
12379         }
12380
12381         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
12382                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
12383                         bnxt_fw_reset_writel(bp, i);
12384         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
12385                 struct hwrm_fw_reset_input *req;
12386
12387                 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
12388                 if (!rc) {
12389                         req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
12390                         req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
12391                         req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
12392                         req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
12393                         rc = hwrm_req_send(bp, req);
12394                 }
12395                 if (rc != -ENODEV)
12396                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
12397         }
12398         bp->fw_reset_timestamp = jiffies;
12399 }
12400
12401 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
12402 {
12403         return time_after(jiffies, bp->fw_reset_timestamp +
12404                           (bp->fw_reset_max_dsecs * HZ / 10));
12405 }
12406
12407 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
12408 {
12409         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12410         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
12411                 bnxt_ulp_start(bp, rc);
12412                 bnxt_dl_health_fw_status_update(bp, false);
12413         }
12414         bp->fw_reset_state = 0;
12415         dev_close(bp->dev);
12416 }
12417
12418 static void bnxt_fw_reset_task(struct work_struct *work)
12419 {
12420         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
12421         int rc = 0;
12422
12423         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12424                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
12425                 return;
12426         }
12427
12428         switch (bp->fw_reset_state) {
12429         case BNXT_FW_RESET_STATE_POLL_VF: {
12430                 int n = bnxt_get_registered_vfs(bp);
12431                 int tmo;
12432
12433                 if (n < 0) {
12434                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
12435                                    n, jiffies_to_msecs(jiffies -
12436                                    bp->fw_reset_timestamp));
12437                         goto fw_reset_abort;
12438                 } else if (n > 0) {
12439                         if (bnxt_fw_reset_timeout(bp)) {
12440                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12441                                 bp->fw_reset_state = 0;
12442                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
12443                                            n);
12444                                 return;
12445                         }
12446                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12447                         return;
12448                 }
12449                 bp->fw_reset_timestamp = jiffies;
12450                 rtnl_lock();
12451                 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12452                         bnxt_fw_reset_abort(bp, rc);
12453                         rtnl_unlock();
12454                         return;
12455                 }
12456                 bnxt_fw_reset_close(bp);
12457                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12458                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12459                         tmo = HZ / 10;
12460                 } else {
12461                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12462                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
12463                 }
12464                 rtnl_unlock();
12465                 bnxt_queue_fw_reset_work(bp, tmo);
12466                 return;
12467         }
12468         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
12469                 u32 val;
12470
12471                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12472                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
12473                     !bnxt_fw_reset_timeout(bp)) {
12474                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12475                         return;
12476                 }
12477
12478                 if (!bp->fw_health->primary) {
12479                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
12480
12481                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12482                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12483                         return;
12484                 }
12485                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12486         }
12487                 fallthrough;
12488         case BNXT_FW_RESET_STATE_RESET_FW:
12489                 bnxt_reset_all(bp);
12490                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12491                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
12492                 return;
12493         case BNXT_FW_RESET_STATE_ENABLE_DEV:
12494                 bnxt_inv_fw_health_reg(bp);
12495                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
12496                     !bp->fw_reset_min_dsecs) {
12497                         u16 val;
12498
12499                         pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12500                         if (val == 0xffff) {
12501                                 if (bnxt_fw_reset_timeout(bp)) {
12502                                         netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
12503                                         rc = -ETIMEDOUT;
12504                                         goto fw_reset_abort;
12505                                 }
12506                                 bnxt_queue_fw_reset_work(bp, HZ / 1000);
12507                                 return;
12508                         }
12509                 }
12510                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12511                 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
12512                 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
12513                     !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
12514                         bnxt_dl_remote_reload(bp);
12515                 if (pci_enable_device(bp->pdev)) {
12516                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
12517                         rc = -ENODEV;
12518                         goto fw_reset_abort;
12519                 }
12520                 pci_set_master(bp->pdev);
12521                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
12522                 fallthrough;
12523         case BNXT_FW_RESET_STATE_POLL_FW:
12524                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
12525                 rc = bnxt_hwrm_poll(bp);
12526                 if (rc) {
12527                         if (bnxt_fw_reset_timeout(bp)) {
12528                                 netdev_err(bp->dev, "Firmware reset aborted\n");
12529                                 goto fw_reset_abort_status;
12530                         }
12531                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12532                         return;
12533                 }
12534                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
12535                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
12536                 fallthrough;
12537         case BNXT_FW_RESET_STATE_OPENING:
12538                 while (!rtnl_trylock()) {
12539                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12540                         return;
12541                 }
12542                 rc = bnxt_open(bp->dev);
12543                 if (rc) {
12544                         netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
12545                         bnxt_fw_reset_abort(bp, rc);
12546                         rtnl_unlock();
12547                         return;
12548                 }
12549
12550                 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
12551                     bp->fw_health->enabled) {
12552                         bp->fw_health->last_fw_reset_cnt =
12553                                 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12554                 }
12555                 bp->fw_reset_state = 0;
12556                 /* Make sure fw_reset_state is 0 before clearing the flag */
12557                 smp_mb__before_atomic();
12558                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12559                 bnxt_ulp_start(bp, 0);
12560                 bnxt_reenable_sriov(bp);
12561                 bnxt_vf_reps_alloc(bp);
12562                 bnxt_vf_reps_open(bp);
12563                 bnxt_ptp_reapply_pps(bp);
12564                 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
12565                 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
12566                         bnxt_dl_health_fw_recovery_done(bp);
12567                         bnxt_dl_health_fw_status_update(bp, true);
12568                 }
12569                 rtnl_unlock();
12570                 break;
12571         }
12572         return;
12573
12574 fw_reset_abort_status:
12575         if (bp->fw_health->status_reliable ||
12576             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
12577                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12578
12579                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
12580         }
12581 fw_reset_abort:
12582         rtnl_lock();
12583         bnxt_fw_reset_abort(bp, rc);
12584         rtnl_unlock();
12585 }
12586
12587 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
12588 {
12589         int rc;
12590         struct bnxt *bp = netdev_priv(dev);
12591
12592         SET_NETDEV_DEV(dev, &pdev->dev);
12593
12594         /* enable device (incl. PCI PM wakeup), and bus-mastering */
12595         rc = pci_enable_device(pdev);
12596         if (rc) {
12597                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
12598                 goto init_err;
12599         }
12600
12601         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12602                 dev_err(&pdev->dev,
12603                         "Cannot find PCI device base address, aborting\n");
12604                 rc = -ENODEV;
12605                 goto init_err_disable;
12606         }
12607
12608         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12609         if (rc) {
12610                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
12611                 goto init_err_disable;
12612         }
12613
12614         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
12615             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12616                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
12617                 rc = -EIO;
12618                 goto init_err_release;
12619         }
12620
12621         pci_set_master(pdev);
12622
12623         bp->dev = dev;
12624         bp->pdev = pdev;
12625
12626         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
12627          * determines the BAR size.
12628          */
12629         bp->bar0 = pci_ioremap_bar(pdev, 0);
12630         if (!bp->bar0) {
12631                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
12632                 rc = -ENOMEM;
12633                 goto init_err_release;
12634         }
12635
12636         bp->bar2 = pci_ioremap_bar(pdev, 4);
12637         if (!bp->bar2) {
12638                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
12639                 rc = -ENOMEM;
12640                 goto init_err_release;
12641         }
12642
12643         pci_enable_pcie_error_reporting(pdev);
12644
12645         INIT_WORK(&bp->sp_task, bnxt_sp_task);
12646         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
12647
12648         spin_lock_init(&bp->ntp_fltr_lock);
12649 #if BITS_PER_LONG == 32
12650         spin_lock_init(&bp->db_lock);
12651 #endif
12652
12653         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
12654         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
12655
12656         timer_setup(&bp->timer, bnxt_timer, 0);
12657         bp->current_interval = BNXT_TIMER_INTERVAL;
12658
12659         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
12660         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
12661
12662         clear_bit(BNXT_STATE_OPEN, &bp->state);
12663         return 0;
12664
12665 init_err_release:
12666         bnxt_unmap_bars(bp, pdev);
12667         pci_release_regions(pdev);
12668
12669 init_err_disable:
12670         pci_disable_device(pdev);
12671
12672 init_err:
12673         return rc;
12674 }
12675
12676 /* rtnl_lock held */
12677 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
12678 {
12679         struct sockaddr *addr = p;
12680         struct bnxt *bp = netdev_priv(dev);
12681         int rc = 0;
12682
12683         if (!is_valid_ether_addr(addr->sa_data))
12684                 return -EADDRNOTAVAIL;
12685
12686         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
12687                 return 0;
12688
12689         rc = bnxt_approve_mac(bp, addr->sa_data, true);
12690         if (rc)
12691                 return rc;
12692
12693         eth_hw_addr_set(dev, addr->sa_data);
12694         if (netif_running(dev)) {
12695                 bnxt_close_nic(bp, false, false);
12696                 rc = bnxt_open_nic(bp, false, false);
12697         }
12698
12699         return rc;
12700 }
12701
12702 /* rtnl_lock held */
12703 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
12704 {
12705         struct bnxt *bp = netdev_priv(dev);
12706
12707         if (netif_running(dev))
12708                 bnxt_close_nic(bp, true, false);
12709
12710         dev->mtu = new_mtu;
12711         bnxt_set_ring_params(bp);
12712
12713         if (netif_running(dev))
12714                 return bnxt_open_nic(bp, true, false);
12715
12716         return 0;
12717 }
12718
12719 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
12720 {
12721         struct bnxt *bp = netdev_priv(dev);
12722         bool sh = false;
12723         int rc;
12724
12725         if (tc > bp->max_tc) {
12726                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
12727                            tc, bp->max_tc);
12728                 return -EINVAL;
12729         }
12730
12731         if (netdev_get_num_tc(dev) == tc)
12732                 return 0;
12733
12734         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
12735                 sh = true;
12736
12737         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
12738                               sh, tc, bp->tx_nr_rings_xdp);
12739         if (rc)
12740                 return rc;
12741
12742         /* Needs to close the device and do hw resource re-allocations */
12743         if (netif_running(bp->dev))
12744                 bnxt_close_nic(bp, true, false);
12745
12746         if (tc) {
12747                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
12748                 netdev_set_num_tc(dev, tc);
12749         } else {
12750                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12751                 netdev_reset_tc(dev);
12752         }
12753         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
12754         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
12755                                bp->tx_nr_rings + bp->rx_nr_rings;
12756
12757         if (netif_running(bp->dev))
12758                 return bnxt_open_nic(bp, true, false);
12759
12760         return 0;
12761 }
12762
12763 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
12764                                   void *cb_priv)
12765 {
12766         struct bnxt *bp = cb_priv;
12767
12768         if (!bnxt_tc_flower_enabled(bp) ||
12769             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
12770                 return -EOPNOTSUPP;
12771
12772         switch (type) {
12773         case TC_SETUP_CLSFLOWER:
12774                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
12775         default:
12776                 return -EOPNOTSUPP;
12777         }
12778 }
12779
12780 LIST_HEAD(bnxt_block_cb_list);
12781
12782 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
12783                          void *type_data)
12784 {
12785         struct bnxt *bp = netdev_priv(dev);
12786
12787         switch (type) {
12788         case TC_SETUP_BLOCK:
12789                 return flow_block_cb_setup_simple(type_data,
12790                                                   &bnxt_block_cb_list,
12791                                                   bnxt_setup_tc_block_cb,
12792                                                   bp, bp, true);
12793         case TC_SETUP_QDISC_MQPRIO: {
12794                 struct tc_mqprio_qopt *mqprio = type_data;
12795
12796                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12797
12798                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12799         }
12800         default:
12801                 return -EOPNOTSUPP;
12802         }
12803 }
12804
12805 #ifdef CONFIG_RFS_ACCEL
12806 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12807                             struct bnxt_ntuple_filter *f2)
12808 {
12809         struct flow_keys *keys1 = &f1->fkeys;
12810         struct flow_keys *keys2 = &f2->fkeys;
12811
12812         if (keys1->basic.n_proto != keys2->basic.n_proto ||
12813             keys1->basic.ip_proto != keys2->basic.ip_proto)
12814                 return false;
12815
12816         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12817                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12818                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12819                         return false;
12820         } else {
12821                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12822                            sizeof(keys1->addrs.v6addrs.src)) ||
12823                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12824                            sizeof(keys1->addrs.v6addrs.dst)))
12825                         return false;
12826         }
12827
12828         if (keys1->ports.ports == keys2->ports.ports &&
12829             keys1->control.flags == keys2->control.flags &&
12830             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12831             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12832                 return true;
12833
12834         return false;
12835 }
12836
12837 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12838                               u16 rxq_index, u32 flow_id)
12839 {
12840         struct bnxt *bp = netdev_priv(dev);
12841         struct bnxt_ntuple_filter *fltr, *new_fltr;
12842         struct flow_keys *fkeys;
12843         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12844         int rc = 0, idx, bit_id, l2_idx = 0;
12845         struct hlist_head *head;
12846         u32 flags;
12847
12848         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12849                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12850                 int off = 0, j;
12851
12852                 netif_addr_lock_bh(dev);
12853                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12854                         if (ether_addr_equal(eth->h_dest,
12855                                              vnic->uc_list + off)) {
12856                                 l2_idx = j + 1;
12857                                 break;
12858                         }
12859                 }
12860                 netif_addr_unlock_bh(dev);
12861                 if (!l2_idx)
12862                         return -EINVAL;
12863         }
12864         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12865         if (!new_fltr)
12866                 return -ENOMEM;
12867
12868         fkeys = &new_fltr->fkeys;
12869         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12870                 rc = -EPROTONOSUPPORT;
12871                 goto err_free;
12872         }
12873
12874         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12875              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12876             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12877              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12878                 rc = -EPROTONOSUPPORT;
12879                 goto err_free;
12880         }
12881         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12882             bp->hwrm_spec_code < 0x10601) {
12883                 rc = -EPROTONOSUPPORT;
12884                 goto err_free;
12885         }
12886         flags = fkeys->control.flags;
12887         if (((flags & FLOW_DIS_ENCAPSULATION) &&
12888              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12889                 rc = -EPROTONOSUPPORT;
12890                 goto err_free;
12891         }
12892
12893         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12894         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12895
12896         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12897         head = &bp->ntp_fltr_hash_tbl[idx];
12898         rcu_read_lock();
12899         hlist_for_each_entry_rcu(fltr, head, hash) {
12900                 if (bnxt_fltr_match(fltr, new_fltr)) {
12901                         rcu_read_unlock();
12902                         rc = 0;
12903                         goto err_free;
12904                 }
12905         }
12906         rcu_read_unlock();
12907
12908         spin_lock_bh(&bp->ntp_fltr_lock);
12909         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12910                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
12911         if (bit_id < 0) {
12912                 spin_unlock_bh(&bp->ntp_fltr_lock);
12913                 rc = -ENOMEM;
12914                 goto err_free;
12915         }
12916
12917         new_fltr->sw_id = (u16)bit_id;
12918         new_fltr->flow_id = flow_id;
12919         new_fltr->l2_fltr_idx = l2_idx;
12920         new_fltr->rxq = rxq_index;
12921         hlist_add_head_rcu(&new_fltr->hash, head);
12922         bp->ntp_fltr_count++;
12923         spin_unlock_bh(&bp->ntp_fltr_lock);
12924
12925         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
12926         bnxt_queue_sp_work(bp);
12927
12928         return new_fltr->sw_id;
12929
12930 err_free:
12931         kfree(new_fltr);
12932         return rc;
12933 }
12934
12935 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12936 {
12937         int i;
12938
12939         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
12940                 struct hlist_head *head;
12941                 struct hlist_node *tmp;
12942                 struct bnxt_ntuple_filter *fltr;
12943                 int rc;
12944
12945                 head = &bp->ntp_fltr_hash_tbl[i];
12946                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
12947                         bool del = false;
12948
12949                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
12950                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
12951                                                         fltr->flow_id,
12952                                                         fltr->sw_id)) {
12953                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
12954                                                                          fltr);
12955                                         del = true;
12956                                 }
12957                         } else {
12958                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
12959                                                                        fltr);
12960                                 if (rc)
12961                                         del = true;
12962                                 else
12963                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
12964                         }
12965
12966                         if (del) {
12967                                 spin_lock_bh(&bp->ntp_fltr_lock);
12968                                 hlist_del_rcu(&fltr->hash);
12969                                 bp->ntp_fltr_count--;
12970                                 spin_unlock_bh(&bp->ntp_fltr_lock);
12971                                 synchronize_rcu();
12972                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
12973                                 kfree(fltr);
12974                         }
12975                 }
12976         }
12977         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
12978                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
12979 }
12980
12981 #else
12982
12983 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12984 {
12985 }
12986
12987 #endif /* CONFIG_RFS_ACCEL */
12988
12989 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
12990 {
12991         struct bnxt *bp = netdev_priv(netdev);
12992         struct udp_tunnel_info ti;
12993         unsigned int cmd;
12994
12995         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
12996         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
12997                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
12998         else
12999                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13000
13001         if (ti.port)
13002                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
13003
13004         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
13005 }
13006
13007 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
13008         .sync_table     = bnxt_udp_tunnel_sync,
13009         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
13010                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
13011         .tables         = {
13012                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
13013                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
13014         },
13015 };
13016
13017 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13018                                struct net_device *dev, u32 filter_mask,
13019                                int nlflags)
13020 {
13021         struct bnxt *bp = netdev_priv(dev);
13022
13023         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
13024                                        nlflags, filter_mask, NULL);
13025 }
13026
13027 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
13028                                u16 flags, struct netlink_ext_ack *extack)
13029 {
13030         struct bnxt *bp = netdev_priv(dev);
13031         struct nlattr *attr, *br_spec;
13032         int rem, rc = 0;
13033
13034         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
13035                 return -EOPNOTSUPP;
13036
13037         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13038         if (!br_spec)
13039                 return -EINVAL;
13040
13041         nla_for_each_nested(attr, br_spec, rem) {
13042                 u16 mode;
13043
13044                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
13045                         continue;
13046
13047                 if (nla_len(attr) < sizeof(mode))
13048                         return -EINVAL;
13049
13050                 mode = nla_get_u16(attr);
13051                 if (mode == bp->br_mode)
13052                         break;
13053
13054                 rc = bnxt_hwrm_set_br_mode(bp, mode);
13055                 if (!rc)
13056                         bp->br_mode = mode;
13057                 break;
13058         }
13059         return rc;
13060 }
13061
13062 int bnxt_get_port_parent_id(struct net_device *dev,
13063                             struct netdev_phys_item_id *ppid)
13064 {
13065         struct bnxt *bp = netdev_priv(dev);
13066
13067         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
13068                 return -EOPNOTSUPP;
13069
13070         /* The PF and it's VF-reps only support the switchdev framework */
13071         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
13072                 return -EOPNOTSUPP;
13073
13074         ppid->id_len = sizeof(bp->dsn);
13075         memcpy(ppid->id, bp->dsn, ppid->id_len);
13076
13077         return 0;
13078 }
13079
13080 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
13081 {
13082         struct bnxt *bp = netdev_priv(dev);
13083
13084         return &bp->dl_port;
13085 }
13086
13087 static const struct net_device_ops bnxt_netdev_ops = {
13088         .ndo_open               = bnxt_open,
13089         .ndo_start_xmit         = bnxt_start_xmit,
13090         .ndo_stop               = bnxt_close,
13091         .ndo_get_stats64        = bnxt_get_stats64,
13092         .ndo_set_rx_mode        = bnxt_set_rx_mode,
13093         .ndo_eth_ioctl          = bnxt_ioctl,
13094         .ndo_validate_addr      = eth_validate_addr,
13095         .ndo_set_mac_address    = bnxt_change_mac_addr,
13096         .ndo_change_mtu         = bnxt_change_mtu,
13097         .ndo_fix_features       = bnxt_fix_features,
13098         .ndo_set_features       = bnxt_set_features,
13099         .ndo_features_check     = bnxt_features_check,
13100         .ndo_tx_timeout         = bnxt_tx_timeout,
13101 #ifdef CONFIG_BNXT_SRIOV
13102         .ndo_get_vf_config      = bnxt_get_vf_config,
13103         .ndo_set_vf_mac         = bnxt_set_vf_mac,
13104         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
13105         .ndo_set_vf_rate        = bnxt_set_vf_bw,
13106         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
13107         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
13108         .ndo_set_vf_trust       = bnxt_set_vf_trust,
13109 #endif
13110         .ndo_setup_tc           = bnxt_setup_tc,
13111 #ifdef CONFIG_RFS_ACCEL
13112         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
13113 #endif
13114         .ndo_bpf                = bnxt_xdp,
13115         .ndo_xdp_xmit           = bnxt_xdp_xmit,
13116         .ndo_bridge_getlink     = bnxt_bridge_getlink,
13117         .ndo_bridge_setlink     = bnxt_bridge_setlink,
13118         .ndo_get_devlink_port   = bnxt_get_devlink_port,
13119 };
13120
13121 static void bnxt_remove_one(struct pci_dev *pdev)
13122 {
13123         struct net_device *dev = pci_get_drvdata(pdev);
13124         struct bnxt *bp = netdev_priv(dev);
13125
13126         if (BNXT_PF(bp))
13127                 bnxt_sriov_disable(bp);
13128
13129         if (BNXT_PF(bp))
13130                 devlink_port_type_clear(&bp->dl_port);
13131
13132         bnxt_ptp_clear(bp);
13133         pci_disable_pcie_error_reporting(pdev);
13134         unregister_netdev(dev);
13135         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13136         /* Flush any pending tasks */
13137         cancel_work_sync(&bp->sp_task);
13138         cancel_delayed_work_sync(&bp->fw_reset_task);
13139         bp->sp_event = 0;
13140
13141         bnxt_dl_fw_reporters_destroy(bp);
13142         bnxt_dl_unregister(bp);
13143         bnxt_shutdown_tc(bp);
13144
13145         bnxt_clear_int_mode(bp);
13146         bnxt_hwrm_func_drv_unrgtr(bp);
13147         bnxt_free_hwrm_resources(bp);
13148         bnxt_ethtool_free(bp);
13149         bnxt_dcb_free(bp);
13150         kfree(bp->edev);
13151         bp->edev = NULL;
13152         kfree(bp->ptp_cfg);
13153         bp->ptp_cfg = NULL;
13154         kfree(bp->fw_health);
13155         bp->fw_health = NULL;
13156         bnxt_cleanup_pci(bp);
13157         bnxt_free_ctx_mem(bp);
13158         kfree(bp->ctx);
13159         bp->ctx = NULL;
13160         kfree(bp->rss_indir_tbl);
13161         bp->rss_indir_tbl = NULL;
13162         bnxt_free_port_stats(bp);
13163         free_netdev(dev);
13164 }
13165
13166 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
13167 {
13168         int rc = 0;
13169         struct bnxt_link_info *link_info = &bp->link_info;
13170
13171         bp->phy_flags = 0;
13172         rc = bnxt_hwrm_phy_qcaps(bp);
13173         if (rc) {
13174                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
13175                            rc);
13176                 return rc;
13177         }
13178         if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
13179                 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
13180         else
13181                 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
13182         if (!fw_dflt)
13183                 return 0;
13184
13185         mutex_lock(&bp->link_lock);
13186         rc = bnxt_update_link(bp, false);
13187         if (rc) {
13188                 mutex_unlock(&bp->link_lock);
13189                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
13190                            rc);
13191                 return rc;
13192         }
13193
13194         /* Older firmware does not have supported_auto_speeds, so assume
13195          * that all supported speeds can be autonegotiated.
13196          */
13197         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
13198                 link_info->support_auto_speeds = link_info->support_speeds;
13199
13200         bnxt_init_ethtool_link_settings(bp);
13201         mutex_unlock(&bp->link_lock);
13202         return 0;
13203 }
13204
13205 static int bnxt_get_max_irq(struct pci_dev *pdev)
13206 {
13207         u16 ctrl;
13208
13209         if (!pdev->msix_cap)
13210                 return 1;
13211
13212         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
13213         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
13214 }
13215
13216 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13217                                 int *max_cp)
13218 {
13219         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
13220         int max_ring_grps = 0, max_irq;
13221
13222         *max_tx = hw_resc->max_tx_rings;
13223         *max_rx = hw_resc->max_rx_rings;
13224         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
13225         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
13226                         bnxt_get_ulp_msix_num(bp),
13227                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
13228         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
13229                 *max_cp = min_t(int, *max_cp, max_irq);
13230         max_ring_grps = hw_resc->max_hw_ring_grps;
13231         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
13232                 *max_cp -= 1;
13233                 *max_rx -= 2;
13234         }
13235         if (bp->flags & BNXT_FLAG_AGG_RINGS)
13236                 *max_rx >>= 1;
13237         if (bp->flags & BNXT_FLAG_CHIP_P5) {
13238                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
13239                 /* On P5 chips, max_cp output param should be available NQs */
13240                 *max_cp = max_irq;
13241         }
13242         *max_rx = min_t(int, *max_rx, max_ring_grps);
13243 }
13244
13245 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
13246 {
13247         int rx, tx, cp;
13248
13249         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
13250         *max_rx = rx;
13251         *max_tx = tx;
13252         if (!rx || !tx || !cp)
13253                 return -ENOMEM;
13254
13255         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
13256 }
13257
13258 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13259                                bool shared)
13260 {
13261         int rc;
13262
13263         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13264         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
13265                 /* Not enough rings, try disabling agg rings. */
13266                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
13267                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13268                 if (rc) {
13269                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
13270                         bp->flags |= BNXT_FLAG_AGG_RINGS;
13271                         return rc;
13272                 }
13273                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
13274                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13275                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13276                 bnxt_set_ring_params(bp);
13277         }
13278
13279         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
13280                 int max_cp, max_stat, max_irq;
13281
13282                 /* Reserve minimum resources for RoCE */
13283                 max_cp = bnxt_get_max_func_cp_rings(bp);
13284                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
13285                 max_irq = bnxt_get_max_func_irqs(bp);
13286                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
13287                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
13288                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
13289                         return 0;
13290
13291                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
13292                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
13293                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
13294                 max_cp = min_t(int, max_cp, max_irq);
13295                 max_cp = min_t(int, max_cp, max_stat);
13296                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
13297                 if (rc)
13298                         rc = 0;
13299         }
13300         return rc;
13301 }
13302
13303 /* In initial default shared ring setting, each shared ring must have a
13304  * RX/TX ring pair.
13305  */
13306 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
13307 {
13308         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
13309         bp->rx_nr_rings = bp->cp_nr_rings;
13310         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
13311         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13312 }
13313
13314 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
13315 {
13316         int dflt_rings, max_rx_rings, max_tx_rings, rc;
13317
13318         if (!bnxt_can_reserve_rings(bp))
13319                 return 0;
13320
13321         if (sh)
13322                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
13323         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
13324         /* Reduce default rings on multi-port cards so that total default
13325          * rings do not exceed CPU count.
13326          */
13327         if (bp->port_count > 1) {
13328                 int max_rings =
13329                         max_t(int, num_online_cpus() / bp->port_count, 1);
13330
13331                 dflt_rings = min_t(int, dflt_rings, max_rings);
13332         }
13333         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
13334         if (rc)
13335                 return rc;
13336         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
13337         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
13338         if (sh)
13339                 bnxt_trim_dflt_sh_rings(bp);
13340         else
13341                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
13342         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13343
13344         rc = __bnxt_reserve_rings(bp);
13345         if (rc && rc != -ENODEV)
13346                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
13347         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13348         if (sh)
13349                 bnxt_trim_dflt_sh_rings(bp);
13350
13351         /* Rings may have been trimmed, re-reserve the trimmed rings. */
13352         if (bnxt_need_reserve_rings(bp)) {
13353                 rc = __bnxt_reserve_rings(bp);
13354                 if (rc && rc != -ENODEV)
13355                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
13356                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13357         }
13358         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
13359                 bp->rx_nr_rings++;
13360                 bp->cp_nr_rings++;
13361         }
13362         if (rc) {
13363                 bp->tx_nr_rings = 0;
13364                 bp->rx_nr_rings = 0;
13365         }
13366         return rc;
13367 }
13368
13369 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
13370 {
13371         int rc;
13372
13373         if (bp->tx_nr_rings)
13374                 return 0;
13375
13376         bnxt_ulp_irq_stop(bp);
13377         bnxt_clear_int_mode(bp);
13378         rc = bnxt_set_dflt_rings(bp, true);
13379         if (rc) {
13380                 if (BNXT_VF(bp) && rc == -ENODEV)
13381                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13382                 else
13383                         netdev_err(bp->dev, "Not enough rings available.\n");
13384                 goto init_dflt_ring_err;
13385         }
13386         rc = bnxt_init_int_mode(bp);
13387         if (rc)
13388                 goto init_dflt_ring_err;
13389
13390         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13391
13392         bnxt_set_dflt_rfs(bp);
13393
13394 init_dflt_ring_err:
13395         bnxt_ulp_irq_restart(bp, rc);
13396         return rc;
13397 }
13398
13399 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
13400 {
13401         int rc;
13402
13403         ASSERT_RTNL();
13404         bnxt_hwrm_func_qcaps(bp);
13405
13406         if (netif_running(bp->dev))
13407                 __bnxt_close_nic(bp, true, false);
13408
13409         bnxt_ulp_irq_stop(bp);
13410         bnxt_clear_int_mode(bp);
13411         rc = bnxt_init_int_mode(bp);
13412         bnxt_ulp_irq_restart(bp, rc);
13413
13414         if (netif_running(bp->dev)) {
13415                 if (rc)
13416                         dev_close(bp->dev);
13417                 else
13418                         rc = bnxt_open_nic(bp, true, false);
13419         }
13420
13421         return rc;
13422 }
13423
13424 static int bnxt_init_mac_addr(struct bnxt *bp)
13425 {
13426         int rc = 0;
13427
13428         if (BNXT_PF(bp)) {
13429                 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
13430         } else {
13431 #ifdef CONFIG_BNXT_SRIOV
13432                 struct bnxt_vf_info *vf = &bp->vf;
13433                 bool strict_approval = true;
13434
13435                 if (is_valid_ether_addr(vf->mac_addr)) {
13436                         /* overwrite netdev dev_addr with admin VF MAC */
13437                         eth_hw_addr_set(bp->dev, vf->mac_addr);
13438                         /* Older PF driver or firmware may not approve this
13439                          * correctly.
13440                          */
13441                         strict_approval = false;
13442                 } else {
13443                         eth_hw_addr_random(bp->dev);
13444                 }
13445                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
13446 #endif
13447         }
13448         return rc;
13449 }
13450
13451 static void bnxt_vpd_read_info(struct bnxt *bp)
13452 {
13453         struct pci_dev *pdev = bp->pdev;
13454         unsigned int vpd_size, kw_len;
13455         int pos, size;
13456         u8 *vpd_data;
13457
13458         vpd_data = pci_vpd_alloc(pdev, &vpd_size);
13459         if (IS_ERR(vpd_data)) {
13460                 pci_warn(pdev, "Unable to read VPD\n");
13461                 return;
13462         }
13463
13464         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13465                                            PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
13466         if (pos < 0)
13467                 goto read_sn;
13468
13469         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13470         memcpy(bp->board_partno, &vpd_data[pos], size);
13471
13472 read_sn:
13473         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13474                                            PCI_VPD_RO_KEYWORD_SERIALNO,
13475                                            &kw_len);
13476         if (pos < 0)
13477                 goto exit;
13478
13479         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13480         memcpy(bp->board_serialno, &vpd_data[pos], size);
13481 exit:
13482         kfree(vpd_data);
13483 }
13484
13485 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
13486 {
13487         struct pci_dev *pdev = bp->pdev;
13488         u64 qword;
13489
13490         qword = pci_get_dsn(pdev);
13491         if (!qword) {
13492                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
13493                 return -EOPNOTSUPP;
13494         }
13495
13496         put_unaligned_le64(qword, dsn);
13497
13498         bp->flags |= BNXT_FLAG_DSN_VALID;
13499         return 0;
13500 }
13501
13502 static int bnxt_map_db_bar(struct bnxt *bp)
13503 {
13504         if (!bp->db_size)
13505                 return -ENODEV;
13506         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
13507         if (!bp->bar1)
13508                 return -ENOMEM;
13509         return 0;
13510 }
13511
13512 void bnxt_print_device_info(struct bnxt *bp)
13513 {
13514         netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
13515                     board_info[bp->board_idx].name,
13516                     (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
13517
13518         pcie_print_link_status(bp->pdev);
13519 }
13520
13521 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
13522 {
13523         struct net_device *dev;
13524         struct bnxt *bp;
13525         int rc, max_irqs;
13526
13527         if (pci_is_bridge(pdev))
13528                 return -ENODEV;
13529
13530         /* Clear any pending DMA transactions from crash kernel
13531          * while loading driver in capture kernel.
13532          */
13533         if (is_kdump_kernel()) {
13534                 pci_clear_master(pdev);
13535                 pcie_flr(pdev);
13536         }
13537
13538         max_irqs = bnxt_get_max_irq(pdev);
13539         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
13540         if (!dev)
13541                 return -ENOMEM;
13542
13543         bp = netdev_priv(dev);
13544         bp->board_idx = ent->driver_data;
13545         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
13546         bnxt_set_max_func_irqs(bp, max_irqs);
13547
13548         if (bnxt_vf_pciid(bp->board_idx))
13549                 bp->flags |= BNXT_FLAG_VF;
13550
13551         if (pdev->msix_cap)
13552                 bp->flags |= BNXT_FLAG_MSIX_CAP;
13553
13554         rc = bnxt_init_board(pdev, dev);
13555         if (rc < 0)
13556                 goto init_err_free;
13557
13558         dev->netdev_ops = &bnxt_netdev_ops;
13559         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
13560         dev->ethtool_ops = &bnxt_ethtool_ops;
13561         pci_set_drvdata(pdev, dev);
13562
13563         rc = bnxt_alloc_hwrm_resources(bp);
13564         if (rc)
13565                 goto init_err_pci_clean;
13566
13567         mutex_init(&bp->hwrm_cmd_lock);
13568         mutex_init(&bp->link_lock);
13569
13570         rc = bnxt_fw_init_one_p1(bp);
13571         if (rc)
13572                 goto init_err_pci_clean;
13573
13574         if (BNXT_PF(bp))
13575                 bnxt_vpd_read_info(bp);
13576
13577         if (BNXT_CHIP_P5(bp)) {
13578                 bp->flags |= BNXT_FLAG_CHIP_P5;
13579                 if (BNXT_CHIP_SR2(bp))
13580                         bp->flags |= BNXT_FLAG_CHIP_SR2;
13581         }
13582
13583         rc = bnxt_alloc_rss_indir_tbl(bp);
13584         if (rc)
13585                 goto init_err_pci_clean;
13586
13587         rc = bnxt_fw_init_one_p2(bp);
13588         if (rc)
13589                 goto init_err_pci_clean;
13590
13591         rc = bnxt_map_db_bar(bp);
13592         if (rc) {
13593                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
13594                         rc);
13595                 goto init_err_pci_clean;
13596         }
13597
13598         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13599                            NETIF_F_TSO | NETIF_F_TSO6 |
13600                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13601                            NETIF_F_GSO_IPXIP4 |
13602                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13603                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
13604                            NETIF_F_RXCSUM | NETIF_F_GRO;
13605
13606         if (BNXT_SUPPORTS_TPA(bp))
13607                 dev->hw_features |= NETIF_F_LRO;
13608
13609         dev->hw_enc_features =
13610                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13611                         NETIF_F_TSO | NETIF_F_TSO6 |
13612                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13613                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13614                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
13615         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
13616
13617         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
13618                                     NETIF_F_GSO_GRE_CSUM;
13619         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
13620         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
13621                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13622         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
13623                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
13624         if (BNXT_SUPPORTS_TPA(bp))
13625                 dev->hw_features |= NETIF_F_GRO_HW;
13626         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
13627         if (dev->features & NETIF_F_GRO_HW)
13628                 dev->features &= ~NETIF_F_LRO;
13629         dev->priv_flags |= IFF_UNICAST_FLT;
13630
13631 #ifdef CONFIG_BNXT_SRIOV
13632         init_waitqueue_head(&bp->sriov_cfg_wait);
13633 #endif
13634         if (BNXT_SUPPORTS_TPA(bp)) {
13635                 bp->gro_func = bnxt_gro_func_5730x;
13636                 if (BNXT_CHIP_P4(bp))
13637                         bp->gro_func = bnxt_gro_func_5731x;
13638                 else if (BNXT_CHIP_P5(bp))
13639                         bp->gro_func = bnxt_gro_func_5750x;
13640         }
13641         if (!BNXT_CHIP_P4_PLUS(bp))
13642                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
13643
13644         rc = bnxt_init_mac_addr(bp);
13645         if (rc) {
13646                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
13647                 rc = -EADDRNOTAVAIL;
13648                 goto init_err_pci_clean;
13649         }
13650
13651         if (BNXT_PF(bp)) {
13652                 /* Read the adapter's DSN to use as the eswitch switch_id */
13653                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
13654         }
13655
13656         /* MTU range: 60 - FW defined max */
13657         dev->min_mtu = ETH_ZLEN;
13658         dev->max_mtu = bp->max_mtu;
13659
13660         rc = bnxt_probe_phy(bp, true);
13661         if (rc)
13662                 goto init_err_pci_clean;
13663
13664         bnxt_set_rx_skb_mode(bp, false);
13665         bnxt_set_tpa_flags(bp);
13666         bnxt_set_ring_params(bp);
13667         rc = bnxt_set_dflt_rings(bp, true);
13668         if (rc) {
13669                 if (BNXT_VF(bp) && rc == -ENODEV) {
13670                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13671                 } else {
13672                         netdev_err(bp->dev, "Not enough rings available.\n");
13673                         rc = -ENOMEM;
13674                 }
13675                 goto init_err_pci_clean;
13676         }
13677
13678         bnxt_fw_init_one_p3(bp);
13679
13680         bnxt_init_dflt_coal(bp);
13681
13682         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13683                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
13684
13685         rc = bnxt_init_int_mode(bp);
13686         if (rc)
13687                 goto init_err_pci_clean;
13688
13689         /* No TC has been set yet and rings may have been trimmed due to
13690          * limited MSIX, so we re-initialize the TX rings per TC.
13691          */
13692         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13693
13694         if (BNXT_PF(bp)) {
13695                 if (!bnxt_pf_wq) {
13696                         bnxt_pf_wq =
13697                                 create_singlethread_workqueue("bnxt_pf_wq");
13698                         if (!bnxt_pf_wq) {
13699                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
13700                                 rc = -ENOMEM;
13701                                 goto init_err_pci_clean;
13702                         }
13703                 }
13704                 rc = bnxt_init_tc(bp);
13705                 if (rc)
13706                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
13707                                    rc);
13708         }
13709
13710         bnxt_inv_fw_health_reg(bp);
13711         rc = bnxt_dl_register(bp);
13712         if (rc)
13713                 goto init_err_dl;
13714
13715         rc = register_netdev(dev);
13716         if (rc)
13717                 goto init_err_cleanup;
13718
13719         if (BNXT_PF(bp))
13720                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
13721         bnxt_dl_fw_reporters_create(bp);
13722
13723         bnxt_print_device_info(bp);
13724
13725         pci_save_state(pdev);
13726         return 0;
13727
13728 init_err_cleanup:
13729         bnxt_dl_unregister(bp);
13730 init_err_dl:
13731         bnxt_shutdown_tc(bp);
13732         bnxt_clear_int_mode(bp);
13733
13734 init_err_pci_clean:
13735         bnxt_hwrm_func_drv_unrgtr(bp);
13736         bnxt_free_hwrm_resources(bp);
13737         bnxt_ethtool_free(bp);
13738         bnxt_ptp_clear(bp);
13739         kfree(bp->ptp_cfg);
13740         bp->ptp_cfg = NULL;
13741         kfree(bp->fw_health);
13742         bp->fw_health = NULL;
13743         bnxt_cleanup_pci(bp);
13744         bnxt_free_ctx_mem(bp);
13745         kfree(bp->ctx);
13746         bp->ctx = NULL;
13747         kfree(bp->rss_indir_tbl);
13748         bp->rss_indir_tbl = NULL;
13749
13750 init_err_free:
13751         free_netdev(dev);
13752         return rc;
13753 }
13754
13755 static void bnxt_shutdown(struct pci_dev *pdev)
13756 {
13757         struct net_device *dev = pci_get_drvdata(pdev);
13758         struct bnxt *bp;
13759
13760         if (!dev)
13761                 return;
13762
13763         rtnl_lock();
13764         bp = netdev_priv(dev);
13765         if (!bp)
13766                 goto shutdown_exit;
13767
13768         if (netif_running(dev))
13769                 dev_close(dev);
13770
13771         bnxt_ulp_shutdown(bp);
13772         bnxt_clear_int_mode(bp);
13773         pci_disable_device(pdev);
13774
13775         if (system_state == SYSTEM_POWER_OFF) {
13776                 pci_wake_from_d3(pdev, bp->wol);
13777                 pci_set_power_state(pdev, PCI_D3hot);
13778         }
13779
13780 shutdown_exit:
13781         rtnl_unlock();
13782 }
13783
13784 #ifdef CONFIG_PM_SLEEP
13785 static int bnxt_suspend(struct device *device)
13786 {
13787         struct net_device *dev = dev_get_drvdata(device);
13788         struct bnxt *bp = netdev_priv(dev);
13789         int rc = 0;
13790
13791         rtnl_lock();
13792         bnxt_ulp_stop(bp);
13793         if (netif_running(dev)) {
13794                 netif_device_detach(dev);
13795                 rc = bnxt_close(dev);
13796         }
13797         bnxt_hwrm_func_drv_unrgtr(bp);
13798         pci_disable_device(bp->pdev);
13799         bnxt_free_ctx_mem(bp);
13800         kfree(bp->ctx);
13801         bp->ctx = NULL;
13802         rtnl_unlock();
13803         return rc;
13804 }
13805
13806 static int bnxt_resume(struct device *device)
13807 {
13808         struct net_device *dev = dev_get_drvdata(device);
13809         struct bnxt *bp = netdev_priv(dev);
13810         int rc = 0;
13811
13812         rtnl_lock();
13813         rc = pci_enable_device(bp->pdev);
13814         if (rc) {
13815                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13816                            rc);
13817                 goto resume_exit;
13818         }
13819         pci_set_master(bp->pdev);
13820         if (bnxt_hwrm_ver_get(bp)) {
13821                 rc = -ENODEV;
13822                 goto resume_exit;
13823         }
13824         rc = bnxt_hwrm_func_reset(bp);
13825         if (rc) {
13826                 rc = -EBUSY;
13827                 goto resume_exit;
13828         }
13829
13830         rc = bnxt_hwrm_func_qcaps(bp);
13831         if (rc)
13832                 goto resume_exit;
13833
13834         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13835                 rc = -ENODEV;
13836                 goto resume_exit;
13837         }
13838
13839         bnxt_get_wol_settings(bp);
13840         if (netif_running(dev)) {
13841                 rc = bnxt_open(dev);
13842                 if (!rc)
13843                         netif_device_attach(dev);
13844         }
13845
13846 resume_exit:
13847         bnxt_ulp_start(bp, rc);
13848         if (!rc)
13849                 bnxt_reenable_sriov(bp);
13850         rtnl_unlock();
13851         return rc;
13852 }
13853
13854 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13855 #define BNXT_PM_OPS (&bnxt_pm_ops)
13856
13857 #else
13858
13859 #define BNXT_PM_OPS NULL
13860
13861 #endif /* CONFIG_PM_SLEEP */
13862
13863 /**
13864  * bnxt_io_error_detected - called when PCI error is detected
13865  * @pdev: Pointer to PCI device
13866  * @state: The current pci connection state
13867  *
13868  * This function is called after a PCI bus error affecting
13869  * this device has been detected.
13870  */
13871 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13872                                                pci_channel_state_t state)
13873 {
13874         struct net_device *netdev = pci_get_drvdata(pdev);
13875         struct bnxt *bp = netdev_priv(netdev);
13876
13877         netdev_info(netdev, "PCI I/O error detected\n");
13878
13879         rtnl_lock();
13880         netif_device_detach(netdev);
13881
13882         bnxt_ulp_stop(bp);
13883
13884         if (state == pci_channel_io_perm_failure) {
13885                 rtnl_unlock();
13886                 return PCI_ERS_RESULT_DISCONNECT;
13887         }
13888
13889         if (state == pci_channel_io_frozen)
13890                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13891
13892         if (netif_running(netdev))
13893                 bnxt_close(netdev);
13894
13895         if (pci_is_enabled(pdev))
13896                 pci_disable_device(pdev);
13897         bnxt_free_ctx_mem(bp);
13898         kfree(bp->ctx);
13899         bp->ctx = NULL;
13900         rtnl_unlock();
13901
13902         /* Request a slot slot reset. */
13903         return PCI_ERS_RESULT_NEED_RESET;
13904 }
13905
13906 /**
13907  * bnxt_io_slot_reset - called after the pci bus has been reset.
13908  * @pdev: Pointer to PCI device
13909  *
13910  * Restart the card from scratch, as if from a cold-boot.
13911  * At this point, the card has exprienced a hard reset,
13912  * followed by fixups by BIOS, and has its config space
13913  * set up identically to what it was at cold boot.
13914  */
13915 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
13916 {
13917         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
13918         struct net_device *netdev = pci_get_drvdata(pdev);
13919         struct bnxt *bp = netdev_priv(netdev);
13920         int err = 0, off;
13921
13922         netdev_info(bp->dev, "PCI Slot Reset\n");
13923
13924         rtnl_lock();
13925
13926         if (pci_enable_device(pdev)) {
13927                 dev_err(&pdev->dev,
13928                         "Cannot re-enable PCI device after reset.\n");
13929         } else {
13930                 pci_set_master(pdev);
13931                 /* Upon fatal error, our device internal logic that latches to
13932                  * BAR value is getting reset and will restore only upon
13933                  * rewritting the BARs.
13934                  *
13935                  * As pci_restore_state() does not re-write the BARs if the
13936                  * value is same as saved value earlier, driver needs to
13937                  * write the BARs to 0 to force restore, in case of fatal error.
13938                  */
13939                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
13940                                        &bp->state)) {
13941                         for (off = PCI_BASE_ADDRESS_0;
13942                              off <= PCI_BASE_ADDRESS_5; off += 4)
13943                                 pci_write_config_dword(bp->pdev, off, 0);
13944                 }
13945                 pci_restore_state(pdev);
13946                 pci_save_state(pdev);
13947
13948                 err = bnxt_hwrm_func_reset(bp);
13949                 if (!err)
13950                         result = PCI_ERS_RESULT_RECOVERED;
13951         }
13952
13953         rtnl_unlock();
13954
13955         return result;
13956 }
13957
13958 /**
13959  * bnxt_io_resume - called when traffic can start flowing again.
13960  * @pdev: Pointer to PCI device
13961  *
13962  * This callback is called when the error recovery driver tells
13963  * us that its OK to resume normal operation.
13964  */
13965 static void bnxt_io_resume(struct pci_dev *pdev)
13966 {
13967         struct net_device *netdev = pci_get_drvdata(pdev);
13968         struct bnxt *bp = netdev_priv(netdev);
13969         int err;
13970
13971         netdev_info(bp->dev, "PCI Slot Resume\n");
13972         rtnl_lock();
13973
13974         err = bnxt_hwrm_func_qcaps(bp);
13975         if (!err && netif_running(netdev))
13976                 err = bnxt_open(netdev);
13977
13978         bnxt_ulp_start(bp, err);
13979         if (!err) {
13980                 bnxt_reenable_sriov(bp);
13981                 netif_device_attach(netdev);
13982         }
13983
13984         rtnl_unlock();
13985 }
13986
13987 static const struct pci_error_handlers bnxt_err_handler = {
13988         .error_detected = bnxt_io_error_detected,
13989         .slot_reset     = bnxt_io_slot_reset,
13990         .resume         = bnxt_io_resume
13991 };
13992
13993 static struct pci_driver bnxt_pci_driver = {
13994         .name           = DRV_MODULE_NAME,
13995         .id_table       = bnxt_pci_tbl,
13996         .probe          = bnxt_init_one,
13997         .remove         = bnxt_remove_one,
13998         .shutdown       = bnxt_shutdown,
13999         .driver.pm      = BNXT_PM_OPS,
14000         .err_handler    = &bnxt_err_handler,
14001 #if defined(CONFIG_BNXT_SRIOV)
14002         .sriov_configure = bnxt_sriov_configure,
14003 #endif
14004 };
14005
14006 static int __init bnxt_init(void)
14007 {
14008         bnxt_debug_init();
14009         return pci_register_driver(&bnxt_pci_driver);
14010 }
14011
14012 static void __exit bnxt_exit(void)
14013 {
14014         pci_unregister_driver(&bnxt_pci_driver);
14015         if (bnxt_pf_wq)
14016                 destroy_workqueue(bnxt_pf_wq);
14017         bnxt_debug_exit();
14018 }
14019
14020 module_init(bnxt_init);
14021 module_exit(bnxt_exit);